Remove artificial default processors limit

Today Elasticsearch limits the number of processors used in computing
thread counts to 32. This was from a time when Elasticsearch created
more threads than it does now and users would run into out of memory
errors. It appears the real cause of these out of memory errors was not
well understood (it's often due to ulimit settings) and so users were
left hitting these out of memory errors on boxes with high core
counts. Today Elasticsearch creates less threads (but still a lot) and
we have a bootstrap check in place to ensure that the relevant ulimit is
not too low.

There are some caveats still to having too many concurrent indexing
threads as it can lead to too many little segments, and it's not a
magical go faster knob if indexing is already bottlenecked by disk, but
this limitation is artificial and surprising to users and so it should
be removed.

This commit also increases the lower bound of the max processes ulimit,
to prepare for a world where Elasticsearch instances might be running
with more the previous cap of 32 processors. With the current settings,
Elasticsearch wants to create roughly 576 + 25 * p / 2 threads, where p
is the number of processors. Add in roughly 7 * p / 8 threads for the GC
threads and a fudge factor, and 4096 should cover us pretty well up to
256 cores.

Relates #20874

Author: jasontedor

core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java

@@ -51,7 +51,7 @@ public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, Tran
         super(settings, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new);
         this.clusterService = clusterService;
         this.searchAction = searchAction;
-        this.availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
+        this.availableProcessors = EsExecutors.numberOfProcessors(settings);
     }
 
     // For testing only:

core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java

@@ -308,7 +308,8 @@ boolean isMemoryLocked() {
 
     static class MaxNumberOfThreadsCheck implements Check {
 
-        private final long maxNumberOfThreadsThreshold = 1 << 11;
+        // this should be plenty for machines up to 256 cores
+        private final long maxNumberOfThreadsThreshold = 1 << 12;
 
         @Override
         public boolean check() {

core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java

@@ -69,7 +69,7 @@ protected PageCacheRecycler(Settings settings) {
         super(settings);
         final Type type = TYPE_SETTING .get(settings);
         final long limit = LIMIT_HEAP_SETTING .get(settings).getBytes();
-        final int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
+        final int availableProcessors = EsExecutors.numberOfProcessors(settings);
 
         // We have a global amount of memory that we need to divide across data types.
         // Since some types are more useful than other ones we give them different weights.

core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java

@@ -40,16 +40,17 @@ public class EsExecutors {
      * This is used to adjust thread pools sizes etc. per node.
      */
     public static final Setting<Integer> PROCESSORS_SETTING =
-        Setting.intSetting("processors", Math.min(32, Runtime.getRuntime().availableProcessors()), 1, Property.NodeScope);
+        Setting.intSetting("processors", Runtime.getRuntime().availableProcessors(), 1, Property.NodeScope);
 
     /**
-     * Returns the number of processors available but at most <tt>32</tt>.
+     * Returns the number of available processors. Defaults to
+     * {@link Runtime#availableProcessors()} but can be overridden by passing a {@link Settings}
+     * instance with the key "processors" set to the desired value.
+     *
+     * @param settings a {@link Settings} instance from which to derive the available processors
+     * @return the number of available processors
      */
-    public static int boundedNumberOfProcessors(Settings settings) {
-        /* This relates to issues where machines with large number of cores
-         * ie. >= 48 create too many threads and run into OOM see #3478
-         * We just use an 32 core upper-bound here to not stress the system
-         * too much with too many created threads */
+    public static int numberOfProcessors(final Settings settings) {
         return PROCESSORS_SETTING.get(settings);
     }
 

core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java

@@ -54,7 +54,7 @@ public final class MergeSchedulerConfig {
 
     public static final Setting<Integer> MAX_THREAD_COUNT_SETTING =
         new Setting<>("index.merge.scheduler.max_thread_count",
-            (s) -> Integer.toString(Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(s) / 2))),
+            (s) -> Integer.toString(Math.max(1, Math.min(4, EsExecutors.numberOfProcessors(s) / 2))),
             (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_thread_count"), Property.Dynamic,
             Property.IndexScope);
     public static final Setting<Integer> MAX_MERGE_COUNT_SETTING =

core/src/main/java/org/elasticsearch/monitor/os/OsService.java

@@ -41,7 +41,7 @@ public OsService(Settings settings) {
         super(settings);
         this.probe = OsProbe.getInstance();
         TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
-        this.info = probe.osInfo(refreshInterval.millis(), EsExecutors.boundedNumberOfProcessors(settings));
+        this.info = probe.osInfo(refreshInterval.millis(), EsExecutors.numberOfProcessors(settings));
         this.osStatsCache = new OsStatsCache(refreshInterval, probe.osStats());
         logger.debug("using refresh_interval [{}]", refreshInterval);
     }

core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java

@@ -23,7 +23,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.node.Node;
 
@@ -79,7 +78,7 @@ public FixedExecutorBuilder(final Settings settings, final String name, final in
 
     private int applyHardSizeLimit(final Settings settings, final String name) {
         if (name.equals(ThreadPool.Names.BULK) || name.equals(ThreadPool.Names.INDEX)) {
-            return 1 + EsExecutors.boundedNumberOfProcessors(settings);
+            return 1 + EsExecutors.numberOfProcessors(settings);
         } else {
             return Integer.MAX_VALUE;
         }

core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java

@@ -163,7 +163,7 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
         assert Node.NODE_NAME_SETTING.exists(settings);
 
         final Map<String, ExecutorBuilder> builders = new HashMap<>();
-        final int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
+        final int availableProcessors = EsExecutors.numberOfProcessors(settings);
         final int halfProcMaxAt5 = halfNumberOfProcessorsMaxFive(availableProcessors);
         final int halfProcMaxAt10 = halfNumberOfProcessorsMaxTen(availableProcessors);
         final int genericThreadPoolMax = boundedBy(4 * availableProcessors, 128, 512);

core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java

@@ -33,7 +33,7 @@ public void testRejectedExecutionCounter() throws InterruptedException {
         final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED);
         // some of the fixed thread pool are bound by the number of
         // cores so we can not exceed that
-        final int size = randomIntBetween(1, EsExecutors.boundedNumberOfProcessors(Settings.EMPTY));
+        final int size = randomIntBetween(1, EsExecutors.numberOfProcessors(Settings.EMPTY));
         final int queueSize = randomIntBetween(1, 16);
         final long rejections = randomIntBetween(1, 16);
 

core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java

@@ -62,7 +62,7 @@ public void testCorrectThreadPoolTypePermittedInSettings() throws InterruptedExc
 
     public void testIndexingThreadPoolsMaxSize() throws InterruptedException {
         final String name = randomFrom(Names.BULK, Names.INDEX);
-        final int maxSize = 1 + EsExecutors.boundedNumberOfProcessors(Settings.EMPTY);
+        final int maxSize = 1 + EsExecutors.numberOfProcessors(Settings.EMPTY);
         final int tooBig = randomIntBetween(1 + maxSize, Integer.MAX_VALUE);
 
         // try to create a too big thread pool
@@ -89,7 +89,7 @@ public void testIndexingThreadPoolsMaxSize() throws InterruptedException {
 
     private static int getExpectedThreadPoolSize(Settings settings, String name, int size) {
         if (name.equals(ThreadPool.Names.BULK) || name.equals(ThreadPool.Names.INDEX)) {
-            return Math.min(size, EsExecutors.boundedNumberOfProcessors(settings));
+            return Math.min(size, EsExecutors.numberOfProcessors(settings));
         } else {
             return size;
         }
@@ -185,7 +185,7 @@ public void testCustomThreadPool() throws Exception {
                 new ScalingExecutorBuilder(
                     "my_pool1",
                     1,
-                    EsExecutors.boundedNumberOfProcessors(Settings.EMPTY),
+                    EsExecutors.numberOfProcessors(Settings.EMPTY),
                     TimeValue.timeValueMinutes(1));
 
             final FixedExecutorBuilder fixed = new FixedExecutorBuilder(Settings.EMPTY, "my_pool2", 1, 1);