Skip to content

Commit fbd3386

Browse files
jasontedorjasontedor
committed
Enable avoiding mmap bootstrap check (#32421)
The maximum map count boostrap check can be a hindrance to users that do not own the underlying platform on which they are executing Elasticsearch. This is because addressing it requires tuning the kernel and a platform provider might now allow this, especially on shared infrastructure. However, this bootstrap check is not needed if mmapfs is not in use. Today we do not have a way for the user to communicate that they are not going to use mmapfs. This commit therefore adds a setting that enables the user to disallow mmapfs. When mmapfs is disallowed, the maximum map count bootstrap check is not enforced. Additionally, we fallback to a different default index store and prevent the explicit use of mmapfs for an index.
1 parent 23035de commit fbd3386

File tree

11 files changed

+243
-64
lines changed

11 files changed

+243
-64
lines changed

‎docs/reference/index-modules/store.asciidoc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,13 @@ process equal to the size of the file being mapped. Before using this
6767
class, be sure you have allowed plenty of
6868
<<vm-max-map-count,virtual address space>>.
6969

70+
[[allow-mmapfs]]
71+
You can restrict the use of the `mmapfs` store type via the setting
72+
`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not
73+
`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful,
74+
for example, if you are in an environment where you can not control the ability
75+
to create a lot of memory maps so you need disable the ability to use `mmapfs`.
76+
7077
=== Pre-loading data into the file system cache
7178

7279
NOTE: This is an expert setting, the details of which may change in the future.

‎docs/reference/setup/bootstrap-checks.asciidoc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,11 @@ the kernel allows a process to have at least 262,144 memory-mapped areas
155155
and is enforced on Linux only. To pass the maximum map count check, you
156156
must configure `vm.max_map_count` via `sysctl` to be at least `262144`.
157157

158+
Alternatively, the maximum map count check is only needed if you are using
159+
`mmapfs` as the <<index-modules-store,store type>> for your indices. If you
160+
<<allow-mmapfs,do not allow>> the use of `mmapfs` then this bootstrap check will
161+
not be enforced.
162+
158163
=== Client JVM check
159164

160165
There are two different JVMs provided by OpenJDK-derived JVMs: the

‎server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.common.transport.BoundTransportAddress;
2929
import org.elasticsearch.common.transport.TransportAddress;
3030
import org.elasticsearch.discovery.DiscoveryModule;
31+
import org.elasticsearch.index.IndexModule;
3132
import org.elasticsearch.monitor.jvm.JvmInfo;
3233
import org.elasticsearch.monitor.process.ProcessProbe;
3334
import org.elasticsearch.node.NodeValidationException;
@@ -393,17 +394,22 @@ long getMaxFileSize() {
393394

394395
static class MaxMapCountCheck implements BootstrapCheck {
395396

396-
private static final long LIMIT = 1 << 18;
397+
static final long LIMIT = 1 << 18;
397398

398399
@Override
399-
public BootstrapCheckResult check(BootstrapContext context) {
400-
if (getMaxMapCount() != -1 && getMaxMapCount() < LIMIT) {
401-
final String message = String.format(
402-
Locale.ROOT,
403-
"max virtual memory areas vm.max_map_count [%d] is too low, increase to at least [%d]",
404-
getMaxMapCount(),
405-
LIMIT);
406-
return BootstrapCheckResult.failure(message);
400+
public BootstrapCheckResult check(final BootstrapContext context) {
401+
// we only enforce the check if mmapfs is an allowed store type
402+
if (IndexModule.NODE_STORE_ALLOW_MMAPFS.get(context.settings)) {
403+
if (getMaxMapCount() != -1 && getMaxMapCount() < LIMIT) {
404+
final String message = String.format(
405+
Locale.ROOT,
406+
"max virtual memory areas vm.max_map_count [%d] is too low, increase to at least [%d]",
407+
getMaxMapCount(),
408+
LIMIT);
409+
return BootstrapCheckResult.failure(message);
410+
} else {
411+
return BootstrapCheckResult.success();
412+
}
407413
} else {
408414
return BootstrapCheckResult.success();
409415
}

‎server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.elasticsearch.env.NodeEnvironment;
6565
import org.elasticsearch.gateway.GatewayService;
6666
import org.elasticsearch.http.HttpTransportSettings;
67+
import org.elasticsearch.index.IndexModule;
6768
import org.elasticsearch.index.IndexSettings;
6869
import org.elasticsearch.indices.IndexingMemoryController;
6970
import org.elasticsearch.indices.IndicesQueryCache;
@@ -266,6 +267,7 @@ public void apply(Settings value, Settings current, Settings previous) {
266267
HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
267268
HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_LIMIT_SETTING,
268269
HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_OVERHEAD_SETTING,
270+
IndexModule.NODE_STORE_ALLOW_MMAPFS,
269271
ClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
270272
SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
271273
SearchService.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS,

‎server/src/main/java/org/elasticsearch/index/IndexModule.java‎

Lines changed: 82 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@
2121

2222
import org.apache.lucene.search.similarities.BM25Similarity;
2323
import org.apache.lucene.search.similarities.Similarity;
24+
import org.apache.lucene.store.MMapDirectory;
25+
import org.apache.lucene.util.Constants;
2426
import org.apache.lucene.util.SetOnce;
2527
import org.elasticsearch.Version;
2628
import org.elasticsearch.client.Client;
27-
import org.elasticsearch.common.Strings;
2829
import org.elasticsearch.common.TriFunction;
2930
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
3031
import org.elasticsearch.common.settings.Setting;
@@ -59,7 +60,6 @@
5960
import java.util.HashMap;
6061
import java.util.HashSet;
6162
import java.util.List;
62-
import java.util.Locale;
6363
import java.util.Map;
6464
import java.util.Objects;
6565
import java.util.Set;
@@ -84,8 +84,10 @@
8484
*/
8585
public final class IndexModule {
8686

87+
public static final Setting<Boolean> NODE_STORE_ALLOW_MMAPFS = Setting.boolSetting("node.store.allow_mmapfs", true, Property.NodeScope);
88+
8789
public static final Setting<String> INDEX_STORE_TYPE_SETTING =
88-
new Setting<>("index.store.type", "", Function.identity(), Property.IndexScope, Property.NodeScope);
90+
new Setting<>("index.store.type", "", Function.identity(), Property.IndexScope, Property.NodeScope);
8991

9092
/** On which extensions to load data into the file-system cache upon opening of files.
9193
* This only works with the mmap directory, and even in that case is still
@@ -289,7 +291,7 @@ IndexEventListener freeze() { // pkg private for testing
289291
}
290292
}
291293

292-
private static boolean isBuiltinType(String storeType) {
294+
public static boolean isBuiltinType(String storeType) {
293295
for (Type type : Type.values()) {
294296
if (type.match(storeType)) {
295297
return true;
@@ -298,21 +300,48 @@ private static boolean isBuiltinType(String storeType) {
298300
return false;
299301
}
300302

303+
301304
public enum Type {
302-
NIOFS,
303-
MMAPFS,
304-
SIMPLEFS,
305-
FS;
305+
NIOFS("niofs"),
306+
MMAPFS("mmapfs"),
307+
SIMPLEFS("simplefs"),
308+
FS("fs");
309+
310+
private final String settingsKey;
311+
312+
Type(final String settingsKey) {
313+
this.settingsKey = settingsKey;
314+
}
315+
316+
private static final Map<String, Type> TYPES;
317+
318+
static {
319+
final Map<String, Type> types = new HashMap<>(4);
320+
for (final Type type : values()) {
321+
types.put(type.settingsKey, type);
322+
}
323+
TYPES = Collections.unmodifiableMap(types);
324+
}
306325

307326
public String getSettingsKey() {
308-
return this.name().toLowerCase(Locale.ROOT);
327+
return this.settingsKey;
328+
}
329+
330+
public static Type fromSettingsKey(final String key) {
331+
final Type type = TYPES.get(key);
332+
if (type == null) {
333+
throw new IllegalArgumentException("no matching type for [" + key + "]");
334+
}
335+
return type;
309336
}
337+
310338
/**
311339
* Returns true iff this settings matches the type.
312340
*/
313341
public boolean match(String setting) {
314342
return getSettingsKey().equals(setting);
315343
}
344+
316345
}
317346

318347
/**
@@ -325,6 +354,16 @@ public interface IndexSearcherWrapperFactory {
325354
IndexSearcherWrapper newWrapper(IndexService indexService);
326355
}
327356

357+
public static Type defaultStoreType(final boolean allowMmapfs) {
358+
if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
359+
return Type.MMAPFS;
360+
} else if (Constants.WINDOWS) {
361+
return Type.SIMPLEFS;
362+
} else {
363+
return Type.NIOFS;
364+
}
365+
}
366+
328367
public IndexService newIndexService(
329368
NodeEnvironment environment,
330369
NamedXContentRegistry xContentRegistry,
@@ -343,20 +382,7 @@ public IndexService newIndexService(
343382
IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null
344383
? (shard) -> null : indexSearcherWrapper.get();
345384
eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
346-
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
347-
final IndexStore store;
348-
if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
349-
store = new IndexStore(indexSettings);
350-
} else {
351-
Function<IndexSettings, IndexStore> factory = indexStoreFactories.get(storeType);
352-
if (factory == null) {
353-
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
354-
}
355-
store = factory.apply(indexSettings);
356-
if (store == null) {
357-
throw new IllegalStateException("store must not be null");
358-
}
359-
}
385+
final IndexStore store = getIndexStore(indexSettings, indexStoreFactories);
360386
final QueryCache queryCache;
361387
if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
362388
BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
@@ -375,6 +401,39 @@ public IndexService newIndexService(
375401
indicesFieldDataCache, searchOperationListeners, indexOperationListeners, namedWriteableRegistry);
376402
}
377403

404+
private static IndexStore getIndexStore(
405+
final IndexSettings indexSettings, final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories) {
406+
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
407+
final Type type;
408+
final Boolean allowMmapfs = NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings());
409+
if (storeType.isEmpty() || Type.FS.getSettingsKey().equals(storeType)) {
410+
type = defaultStoreType(allowMmapfs);
411+
} else {
412+
if (isBuiltinType(storeType)) {
413+
type = Type.fromSettingsKey(storeType);
414+
} else {
415+
type = null;
416+
}
417+
}
418+
if (type != null && type == Type.MMAPFS && allowMmapfs == false) {
419+
throw new IllegalArgumentException("store type [mmapfs] is not allowed");
420+
}
421+
final IndexStore store;
422+
if (storeType.isEmpty() || isBuiltinType(storeType)) {
423+
store = new IndexStore(indexSettings);
424+
} else {
425+
Function<IndexSettings, IndexStore> factory = indexStoreFactories.get(storeType);
426+
if (factory == null) {
427+
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
428+
}
429+
store = factory.apply(indexSettings);
430+
if (store == null) {
431+
throw new IllegalStateException("store must not be null");
432+
}
433+
}
434+
return store;
435+
}
436+
378437
/**
379438
* creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing.
380439
* doing so will result in an exception.

‎server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package org.elasticsearch.index.store;
2121

2222
import org.apache.lucene.store.Directory;
23-
import org.apache.lucene.store.FSDirectory;
2423
import org.apache.lucene.store.FileSwitchDirectory;
2524
import org.apache.lucene.store.LockFactory;
2625
import org.apache.lucene.store.MMapDirectory;
@@ -77,10 +76,21 @@ public Directory newDirectory() throws IOException {
7776
}
7877

7978
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
80-
final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
81-
IndexModule.Type.FS.getSettingsKey());
79+
final String storeType =
80+
indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
8281
if (IndexModule.Type.FS.match(storeType)) {
83-
return FSDirectory.open(location, lockFactory); // use lucene defaults
82+
final IndexModule.Type type =
83+
IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
84+
switch (type) {
85+
case MMAPFS:
86+
return new MMapDirectory(location, lockFactory);
87+
case SIMPLEFS:
88+
return new SimpleFSDirectory(location, lockFactory);
89+
case NIOFS:
90+
return new NIOFSDirectory(location, lockFactory);
91+
default:
92+
throw new AssertionError("unexpected built-in store type [" + type + "]");
93+
}
8494
} else if (IndexModule.Type.SIMPLEFS.match(storeType)) {
8595
return new SimpleFSDirectory(location, lockFactory);
8696
} else if (IndexModule.Type.NIOFS.match(storeType)) {

‎server/src/main/java/org/elasticsearch/indices/IndicesService.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,14 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
228228
this.cacheCleaner = new CacheCleaner(indicesFieldDataCache, indicesRequestCache, logger, threadPool, this.cleanInterval);
229229
this.metaStateService = metaStateService;
230230
this.engineFactoryProviders = engineFactoryProviders;
231+
232+
// do not allow any plugin-provided index store type to conflict with a built-in type
233+
for (final String indexStoreType : indexStoreFactories.keySet()) {
234+
if (IndexModule.isBuiltinType(indexStoreType)) {
235+
throw new IllegalStateException("registered index store type [" + indexStoreType + "] conflicts with a built-in type");
236+
}
237+
}
238+
231239
this.indexStoreFactories = indexStoreFactories;
232240
}
233241

‎server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java‎

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252

5353
public class BootstrapChecksTests extends ESTestCase {
5454

55-
private static final BootstrapContext defaultContext = new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA);
55+
static final BootstrapContext defaultContext = new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA);
5656

5757
public void testNonProductionMode() throws NodeValidationException {
5858
// nothing should happen since we are in non-production mode
@@ -356,31 +356,6 @@ long getRlimInfinity() {
356356
BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
357357
}
358358

359-
public void testMaxMapCountCheck() throws NodeValidationException {
360-
final int limit = 1 << 18;
361-
final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, limit - 1));
362-
final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck() {
363-
@Override
364-
long getMaxMapCount() {
365-
return maxMapCount.get();
366-
}
367-
};
368-
369-
final NodeValidationException e = expectThrows(
370-
NodeValidationException.class,
371-
() -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
372-
assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
373-
374-
maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
375-
376-
BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
377-
378-
// nothing should happen if current vm.max_map_count is not
379-
// available
380-
maxMapCount.set(-1);
381-
BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
382-
}
383-
384359
public void testClientJvmCheck() throws NodeValidationException {
385360
final AtomicReference<String> vmName = new AtomicReference<>("Java HotSpot(TM) 32-Bit Client VM");
386361
final BootstrapCheck check = new BootstrapChecks.ClientJvmCheck() {

0 commit comments

Comments
 (0)