From da33db5d92b401981a5ed2c57de6f5d7189b1d09 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 10 Apr 2018 15:10:52 +0200 Subject: [PATCH 1/5] Derive max composite buffers from max content len With this commit we determine the maximum number of buffers that Netty keeps while accumulating one HTTP request based on the maximum content length. Previously, we kept the default value of 1024 which is too small for bulk requests which leads to unecessary copies of byte buffers internally. --- .../netty4/Netty4HttpServerTransport.java | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index ab0c271f3ae4f..b0719abe442d7 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -122,8 +122,34 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem Netty4Utils.setup(); } + private static final ByteSizeValue MTU_ETHERNET = new ByteSizeValue(1500); + + private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components"; + public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = - Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope); + new Setting<>(SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, (s) -> { + ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(s); + // Netty accumulates buffers containing data from all incoming network packets that make up one HTTP request in an instance of + // io.netty.buffer.CompositeByteBuf (think of it as a buffer of buffers). Once its capacity is reached, the buffer will iterate + // over its individual entries and put them into larger buffers (see io.netty.buffer.CompositeByteBuf#consolidateIfNeeded() + // for implementation details). We want to to resize that buffer because this leads to additional garbage on the heap and also + // increases the application's native memory footprint (as direct byte buffers hold their contents off-heap). + // + // With this setting we control the CompositeByteBuf's capacity (which is by default 1024, see + // io.netty.handler.codec.MessageAggregator#DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS). To determine a proper default capacity for + // that buffer, we need to consider that the upper bound for the size of HTTP requests is determined by `maxContentLength`. The + // number of buffers that are needed depend on how often Netty reads network packets which depends on the network type (MTU). + // We assume here that Elasticsearch receives HTTP requests via an Ethernet connection which has a MTU of 1500 bytes. + // + // Note that we are *not* pre-allocating any memory based on this setting but rather determine the CompositeByteBuf's capacity. + // The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers. + // With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries. + long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MTU_ETHERNET.getBytes())); + // clamp value to the allowed range + long maxBufferComponents = Math.max(2, Math.min(maxBufferComponentsEstimate, Integer.MAX_VALUE)); + return String.valueOf(maxBufferComponents); + // Netty's CompositeByteBuf implementation does not allow less than two components. + }, s -> Setting.parseInt(s, 2, Integer.MAX_VALUE, SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS), Property.NodeScope); public static final Setting SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count", (s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2), @@ -236,8 +262,9 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic this.maxContentLength = maxContentLength; logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " + - "receive_predictor[{}], pipelining[{}], pipelining_max_events[{}]", - maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, receivePredictor, pipelining, pipeliningMaxEvents); + "receive_predictor[{}], max_composite_buffer_components[{}], pipelining[{}], pipelining_max_events[{}]", + maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, receivePredictor, maxCompositeBufferComponents, + pipelining, pipeliningMaxEvents); } public Settings settings() { @@ -532,9 +559,7 @@ protected void initChannel(Channel ch) throws Exception { ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor()); ch.pipeline().addLast("encoder", new HttpResponseEncoder()); final HttpObjectAggregator aggregator = new HttpObjectAggregator(Math.toIntExact(transport.maxContentLength.getBytes())); - if (transport.maxCompositeBufferComponents != -1) { - aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); - } + aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); ch.pipeline().addLast("aggregator", aggregator); if (transport.compression) { ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel)); From b5c14fd7af00803293c47d69877c8991a083c789 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 3 May 2018 08:07:16 +0200 Subject: [PATCH 2/5] Use C-style comment --- .../netty4/Netty4HttpServerTransport.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index b0719abe442d7..5964e9158c53e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -129,21 +129,23 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = new Setting<>(SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, (s) -> { ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(s); - // Netty accumulates buffers containing data from all incoming network packets that make up one HTTP request in an instance of - // io.netty.buffer.CompositeByteBuf (think of it as a buffer of buffers). Once its capacity is reached, the buffer will iterate - // over its individual entries and put them into larger buffers (see io.netty.buffer.CompositeByteBuf#consolidateIfNeeded() - // for implementation details). We want to to resize that buffer because this leads to additional garbage on the heap and also - // increases the application's native memory footprint (as direct byte buffers hold their contents off-heap). - // - // With this setting we control the CompositeByteBuf's capacity (which is by default 1024, see - // io.netty.handler.codec.MessageAggregator#DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS). To determine a proper default capacity for - // that buffer, we need to consider that the upper bound for the size of HTTP requests is determined by `maxContentLength`. The - // number of buffers that are needed depend on how often Netty reads network packets which depends on the network type (MTU). - // We assume here that Elasticsearch receives HTTP requests via an Ethernet connection which has a MTU of 1500 bytes. - // - // Note that we are *not* pre-allocating any memory based on this setting but rather determine the CompositeByteBuf's capacity. - // The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers. - // With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries. + /* + * Netty accumulates buffers containing data from all incoming network packets that make up one HTTP request in an instance of + * io.netty.buffer.CompositeByteBuf (think of it as a buffer of buffers). Once its capacity is reached, the buffer will iterate + * over its individual entries and put them into larger buffers (see io.netty.buffer.CompositeByteBuf#consolidateIfNeeded() + * for implementation details). We want to to resize that buffer because this leads to additional garbage on the heap and also + * increases the application's native memory footprint (as direct byte buffers hold their contents off-heap). + * + * With this setting we control the CompositeByteBuf's capacity (which is by default 1024, see + * io.netty.handler.codec.MessageAggregator#DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS). To determine a proper default capacity for + * that buffer, we need to consider that the upper bound for the size of HTTP requests is determined by `maxContentLength`. The + * number of buffers that are needed depend on how often Netty reads network packets which depends on the network type (MTU). + * We assume here that Elasticsearch receives HTTP requests via an Ethernet connection which has a MTU of 1500 bytes. + * + * Note that we are *not* pre-allocating any memory based on this setting but rather determine the CompositeByteBuf's capacity. + * The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers. + * With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries. + */ long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MTU_ETHERNET.getBytes())); // clamp value to the allowed range long maxBufferComponents = Math.max(2, Math.min(maxBufferComponentsEstimate, Integer.MAX_VALUE)); From e61e44a3b2e177d4a7df4d504381d18b9b302359 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 8 May 2018 11:14:54 +0200 Subject: [PATCH 3/5] Expose message size as system property --- .../http/netty4/Netty4HttpServerTransport.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index 5964e9158c53e..9a00f85a8f8a2 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -122,7 +122,14 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem Netty4Utils.setup(); } - private static final ByteSizeValue MTU_ETHERNET = new ByteSizeValue(1500); + /* + * Size in bytes of an individual message received by io.netty.handler.codec.MessageAggregator which accumulates the content for an + * HTTP request. This number is used for estimating the maximum number of allowed buffers before the MessageAggregator's internal + * collection of buffers is resized. + * + * By default we assume the Ethernet MTU (1500 bytes) as message size but users can override it with a system property. + */ + private static final ByteSizeValue MESSAGE_SIZE = new ByteSizeValue(Long.valueOf(System.getProperty("es.net.message_size", "1500"))); private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components"; @@ -146,7 +153,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem * The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers. * With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries. */ - long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MTU_ETHERNET.getBytes())); + long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MESSAGE_SIZE.getBytes())); // clamp value to the allowed range long maxBufferComponents = Math.max(2, Math.min(maxBufferComponentsEstimate, Integer.MAX_VALUE)); return String.valueOf(maxBufferComponents); From 2049b32d67b4b4136c39429d62f746e2c7b05e9c Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 9 May 2018 08:11:03 +0200 Subject: [PATCH 4/5] Rename system property to 'es.net.mtu' --- .../http/netty4/Netty4HttpServerTransport.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index 9a00f85a8f8a2..9ec4b3005efb1 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -127,9 +127,9 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem * HTTP request. This number is used for estimating the maximum number of allowed buffers before the MessageAggregator's internal * collection of buffers is resized. * - * By default we assume the Ethernet MTU (1500 bytes) as message size but users can override it with a system property. + * By default we assume the Ethernet MTU (1500 bytes) but users can override it with a system property. */ - private static final ByteSizeValue MESSAGE_SIZE = new ByteSizeValue(Long.valueOf(System.getProperty("es.net.message_size", "1500"))); + private static final ByteSizeValue MTU = new ByteSizeValue(Long.valueOf(System.getProperty("es.net.mtu", "1500"))); private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components"; @@ -153,7 +153,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem * The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers. * With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries. */ - long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MESSAGE_SIZE.getBytes())); + long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MTU.getBytes())); // clamp value to the allowed range long maxBufferComponents = Math.max(2, Math.min(maxBufferComponentsEstimate, Integer.MAX_VALUE)); return String.valueOf(maxBufferComponents); From 635c9fb39ae3e2c946ee1cef830ac9af59dafc5a Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 9 May 2018 16:47:39 +0200 Subject: [PATCH 5/5] Avoid boxing --- .../elasticsearch/http/netty4/Netty4HttpServerTransport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index 9ec4b3005efb1..c8c2c4829d2cf 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -129,7 +129,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem * * By default we assume the Ethernet MTU (1500 bytes) but users can override it with a system property. */ - private static final ByteSizeValue MTU = new ByteSizeValue(Long.valueOf(System.getProperty("es.net.mtu", "1500"))); + private static final ByteSizeValue MTU = new ByteSizeValue(Long.parseLong(System.getProperty("es.net.mtu", "1500"))); private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components";