diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index 769e883205de3..6c1579bc28362 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -152,7 +152,7 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport { public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool, NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) { super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher); - Netty4Utils.setAvailableProcessors(EsExecutors.PROCESSORS_SETTING.get(settings)); + Netty4Utils.setAvailableProcessors(EsExecutors.NODE_PROCESSORS_SETTING.get(settings)); this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings); this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java index af5e8b66fe70e..d3e43e16dd5f4 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java @@ -112,7 +112,7 @@ public Netty4Transport(Settings settings, Version version, ThreadPool threadPool PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) { super(settings, version, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService); - Netty4Utils.setAvailableProcessors(EsExecutors.PROCESSORS_SETTING.get(settings)); + Netty4Utils.setAvailableProcessors(EsExecutors.NODE_PROCESSORS_SETTING.get(settings)); this.workerCount = WORKER_COUNT.get(settings); // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one diff --git a/plugins/repository-s3/lib/aws-java-sdk-s3-1.11.562.jar b/plugins/repository-s3/lib/aws-java-sdk-s3-1.11.562.jar new file mode 100644 index 0000000000000..2df50f5c1c024 Binary files /dev/null and b/plugins/repository-s3/lib/aws-java-sdk-s3-1.11.562.jar differ diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index ce19c8c86466e..999643e0baf0b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -390,6 +390,7 @@ public void apply(Settings value, Settings current, Settings previous) { Client.CLIENT_TYPE_SETTING_S, ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING, EsExecutors.PROCESSORS_SETTING, + EsExecutors.NODE_PROCESSORS_SETTING, ThreadContext.DEFAULT_HEADERS_SETTING, Loggers.LOG_DEFAULT_LEVEL_SETTING, Loggers.LOG_LEVEL_SETTING, diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index d4164b474de0b..5cc9ed046916f 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1053,6 +1053,15 @@ public static Setting intSetting(String key, Setting fallbackS return new Setting<>(key, fallbackSetting, (s) -> parseInt(s, minValue, key), properties); } + public static Setting intSetting( + final String key, + final Setting fallbackSetting, + final int minValue, + final int maxValue, + final Property... properties) { + return new Setting<>(key, fallbackSetting, (s) -> parseInt(s, minValue, maxValue, key), properties); + } + public static Setting intSetting(String key, Setting fallbackSetting, int minValue, Validator validator, Property... properties) { return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, (s) -> parseInt(s, minValue, key),validator, diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index 1623ffdf82565..91fcce3a1a596 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -46,14 +46,23 @@ public class EsExecutors { - /** - * Setting to manually set the number of available processors. This setting is used to adjust thread pool sizes per node. - */ public static final Setting PROCESSORS_SETTING = Setting.intSetting( "processors", Runtime.getRuntime().availableProcessors(), 1, Runtime.getRuntime().availableProcessors(), + Property.Deprecated, + Property.NodeScope); + + /** + * Setting to manually set the number of available processors. This setting is used to adjust thread pool sizes per node. + */ + // TODO: when removing "processors" setting, the default value is Runtime.getRuntime().availableProcessors() + public static final Setting NODE_PROCESSORS_SETTING = Setting.intSetting( + "node.processors", + PROCESSORS_SETTING, + 1, + Runtime.getRuntime().availableProcessors(), Property.NodeScope); /** @@ -65,7 +74,7 @@ public class EsExecutors { * @return the number of available processors */ public static int numberOfProcessors(final Settings settings) { - return PROCESSORS_SETTING.get(settings); + return NODE_PROCESSORS_SETTING.get(settings); } public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing(String name, ThreadFactory threadFactory, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index ca3888aa7f807..e1ac076d2ad76 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -203,7 +203,7 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte public void testAllocatedProcessors() throws Exception { // start one node with 7 processors. - internalCluster().startNode(Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 7).build()); + internalCluster().startNode(Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 7).build()); waitForNodes(1); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java index 6a64a93227747..64eedf2020f80 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java @@ -19,10 +19,12 @@ package org.elasticsearch.common.util.concurrent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matcher; +import java.util.Locale; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ThreadPoolExecutor; @@ -389,15 +391,30 @@ public void testGetTasks() throws InterruptedException { } } + public void testNodeProcessorsBound() { + runProcessorsBoundTest(EsExecutors.NODE_PROCESSORS_SETTING); + } + public void testProcessorsBound() { + runProcessorsBoundTest(EsExecutors.PROCESSORS_SETTING); + } + + private void runProcessorsBoundTest(final Setting processorsSetting) { final int available = Runtime.getRuntime().availableProcessors(); final int processors = randomIntBetween(available + 1, Integer.MAX_VALUE); - final Settings settings = Settings.builder().put("processors", processors).build(); + final Settings settings = Settings.builder().put(processorsSetting.getKey(), processors).build(); final IllegalArgumentException e = - expectThrows(IllegalArgumentException.class, () -> EsExecutors.PROCESSORS_SETTING.get(settings)); - assertThat( - e, - hasToString(containsString("Failed to parse value [" + processors + "] for setting [processors] must be <= " + available))); + expectThrows(IllegalArgumentException.class, () -> processorsSetting.get(settings)); + final String expected = String.format( + Locale.ROOT, + "Failed to parse value [%d] for setting [%s] must be <= %d", + processors, + processorsSetting.getKey(), + available); + assertThat(e, hasToString(containsString(expected))); + if (processorsSetting.getProperties().contains(Setting.Property.Deprecated)) { + assertSettingDeprecationsAndWarnings(new Setting[]{processorsSetting}); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java index c3c2a8176e3eb..490c89485d317 100644 --- a/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.common.util.concurrent.EsExecutors.PROCESSORS_SETTING; +import static org.elasticsearch.common.util.concurrent.EsExecutors.NODE_PROCESSORS_SETTING; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.elasticsearch.index.MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING; import static org.elasticsearch.index.MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING; @@ -139,7 +139,7 @@ private static IndexMetaData createMetaData(int maxThreadCount, int maxMergeCoun builder.put(MAX_MERGE_COUNT_SETTING.getKey(), maxMergeCount); } if (numProc != -1) { - builder.put(PROCESSORS_SETTING.getKey(), numProc); + builder.put(NODE_PROCESSORS_SETTING.getKey(), numProc); } return newIndexMeta("index", builder.build()); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java index f597d2adc8042..e22acfca3ae71 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java @@ -70,7 +70,7 @@ private Node startNode() throws NodeValidationException { .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent()) .put(Node.NODE_NAME_SETTING.getKey(), nodeName) .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m") - .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created + .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created .put("transport.type", getTestTransportType()) .put(Node.NODE_DATA_SETTING.getKey(), true) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) diff --git a/server/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/server/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java index f1dba4e58c61e..615bf5431276e 100644 --- a/server/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java +++ b/server/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java @@ -114,8 +114,8 @@ public void testNodesInfosTotalIndexingBuffer() throws Exception { public void testAllocatedProcessors() throws Exception { List nodesIds = internalCluster().startNodes( - Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 3).build(), - Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 6).build() + Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 3).build(), + Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 6).build() ); final String node_1 = nodesIds.get(0); diff --git a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java index 171a1f66d29c0..c9a857180f3a2 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java @@ -54,7 +54,7 @@ public void testScalingThreadPoolConfiguration() throws InterruptedException { if (randomBoolean()) { processors = randomIntBetween(1, availableProcessors); maxBasedOnNumberOfProcessors = expectedSize(threadPoolName, processors); - builder.put("processors", processors); + builder.put("node.processors", processors); } else { maxBasedOnNumberOfProcessors = expectedSize(threadPoolName, availableProcessors); processors = availableProcessors; diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 9ae4b58ab987d..a167419b9db3d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -197,7 +197,7 @@ private Node newNode() { .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent()) .put(Node.NODE_NAME_SETTING.getKey(), nodeName) .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m") - .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created + .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created .put("transport.type", getTestTransportType()) .put(TransportSettings.PORT.getKey(), ESTestCase.getPortRange()) .put(Node.NODE_DATA_SETTING.getKey(), true) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 35f113cd75444..d07e129c7bdc6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -450,7 +450,9 @@ private static Settings getRandomNodeSettings(long seed) { builder.put(SearchService.DEFAULT_KEEPALIVE_SETTING.getKey(), timeValueSeconds(100 + random.nextInt(5 * 60)).getStringRep()); } - builder.put(EsExecutors.PROCESSORS_SETTING.getKey(), 1 + random.nextInt(Math.min(4, Runtime.getRuntime().availableProcessors()))); + builder.put( + EsExecutors.NODE_PROCESSORS_SETTING.getKey(), + 1 + random.nextInt(Math.min(4, Runtime.getRuntime().availableProcessors()))); if (random.nextBoolean()) { if (random.nextBoolean()) { builder.put("indices.fielddata.cache.size", 1 + random.nextInt(1000), ByteSizeUnit.MB);