Skip to content

Commit 64b31f4

Browse files
ywelschywelsch
committed
No mapper service and index caches for replicated closed indices (#40423)
Replicated closed indices can't be indexed into or searched, and therefore don't need a shard with full indexing and search capabilities allocated. We can save on a lot of heap memory for those indices by not allocating a mapper service and caching infrastructure (which preallocates a constant amount per instance). Before this change, a 1GB ES instance could host 250 replicated closed metricbeat indices (each index with one shard). After this change, the same instance can host 7300 replicated closed metricbeat instances (not that this would be a recommended configuration). Most of the remaining memory is in the cluster state and the IndexSettings object.
1 parent 8f7c573 commit 64b31f4

File tree

7 files changed

+58
-30
lines changed

7 files changed

+58
-30
lines changed

‎server/src/main/java/org/elasticsearch/index/IndexModule.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ public static Type defaultStoreType(final boolean allowMmap) {
366366
}
367367

368368
public IndexService newIndexService(
369+
IndexService.IndexCreationContext indexCreationContext,
369370
NodeEnvironment environment,
370371
NamedXContentRegistry xContentRegistry,
371372
IndexService.ShardStoreDeleter shardStoreDeleter,
@@ -395,7 +396,7 @@ public IndexService newIndexService(
395396
} else {
396397
queryCache = new DisabledQueryCache(indexSettings);
397398
}
398-
return new IndexService(indexSettings, environment, xContentRegistry,
399+
return new IndexService(indexSettings, indexCreationContext, environment, xContentRegistry,
399400
new SimilarityService(indexSettings, scriptService, similarities),
400401
shardStoreDeleter, analysisRegistry, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService,
401402
client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry,

‎server/src/main/java/org/elasticsearch/index/IndexService.java‎

Lines changed: 38 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
136136

137137
public IndexService(
138138
IndexSettings indexSettings,
139+
IndexCreationContext indexCreationContext,
139140
NodeEnvironment nodeEnv,
140141
NamedXContentRegistry xContentRegistry,
141142
SimilarityService similarityService,
@@ -162,21 +163,36 @@ public IndexService(
162163
this.similarityService = similarityService;
163164
this.namedWriteableRegistry = namedWriteableRegistry;
164165
this.circuitBreakerService = circuitBreakerService;
165-
this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
166-
mapperRegistry,
167-
// we parse all percolator queries as they would be parsed on shard 0
168-
() -> newQueryShardContext(0, null, System::currentTimeMillis, null));
169-
this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService);
170-
if (indexSettings.getIndexSortConfig().hasIndexSort()) {
171-
// we delay the actual creation of the sort order for this index because the mapping has not been merged yet.
172-
// The sort order is validated right after the merge of the mapping later in the process.
173-
this.indexSortSupplier = () -> indexSettings.getIndexSortConfig().buildIndexSort(
174-
mapperService::fullName,
175-
indexFieldData::getForField
176-
);
177-
} else {
166+
if (indexSettings.getIndexMetaData().getState() == IndexMetaData.State.CLOSE &&
167+
indexCreationContext == IndexCreationContext.CREATE_INDEX) { // metadata verification needs a mapper service
168+
this.mapperService = null;
169+
this.indexFieldData = null;
178170
this.indexSortSupplier = () -> null;
171+
this.bitsetFilterCache = null;
172+
this.warmer = null;
173+
this.indexCache = null;
174+
} else {
175+
this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
176+
mapperRegistry,
177+
// we parse all percolator queries as they would be parsed on shard 0
178+
() -> newQueryShardContext(0, null, System::currentTimeMillis, null));
179+
this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService);
180+
if (indexSettings.getIndexSortConfig().hasIndexSort()) {
181+
// we delay the actual creation of the sort order for this index because the mapping has not been merged yet.
182+
// The sort order is validated right after the merge of the mapping later in the process.
183+
this.indexSortSupplier = () -> indexSettings.getIndexSortConfig().buildIndexSort(
184+
mapperService::fullName,
185+
indexFieldData::getForField
186+
);
187+
} else {
188+
this.indexSortSupplier = () -> null;
189+
}
190+
indexFieldData.setListener(new FieldDataCacheListener(this));
191+
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
192+
this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
193+
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
179194
}
195+
180196
this.shardStoreDeleter = shardStoreDeleter;
181197
this.bigArrays = bigArrays;
182198
this.threadPool = threadPool;
@@ -185,10 +201,6 @@ public IndexService(
185201
this.eventListener = eventListener;
186202
this.nodeEnv = nodeEnv;
187203
this.indexStore = indexStore;
188-
indexFieldData.setListener(new FieldDataCacheListener(this));
189-
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
190-
this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
191-
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
192204
this.engineFactory = Objects.requireNonNull(engineFactory);
193205
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE
194206
this.searcherWrapper = wrapperFactory.newWrapper(this);
@@ -202,6 +214,11 @@ public IndexService(
202214
updateFsyncTaskIfNecessary();
203215
}
204216

217+
public enum IndexCreationContext {
218+
CREATE_INDEX,
219+
META_DATA_VERIFICATION
220+
}
221+
205222
public int numberOfShards() {
206223
return shards.size();
207224
}
@@ -548,7 +565,10 @@ List<SearchOperationListener> getSearchOperationListener() { // pkg private for
548565

549566
@Override
550567
public boolean updateMapping(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) throws IOException {
551-
return mapperService().updateMapping(currentIndexMetaData, newIndexMetaData);
568+
if (mapperService == null) {
569+
return false;
570+
}
571+
return mapperService.updateMapping(currentIndexMetaData, newIndexMetaData);
552572
}
553573

554574
private class StoreCloseListener implements Store.OnClose {

‎server/src/main/java/org/elasticsearch/index/shard/IndexShard.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2493,8 +2493,9 @@ private EngineConfig newEngineConfig() {
24932493
Sort indexSort = indexSortSupplier.get();
24942494
return new EngineConfig(shardId, shardRouting.allocationId().getId(),
24952495
threadPool, indexSettings, warmer, store, indexSettings.getMergePolicy(),
2496-
mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener,
2497-
indexCache.query(), cachingPolicy, translogConfig,
2496+
mapperService != null ? mapperService.indexAnalyzer() : null,
2497+
similarityService.similarity(mapperService), codecService, shardEventListener,
2498+
indexCache != null ? indexCache.query() : null, cachingPolicy, translogConfig,
24982499
IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()),
24992500
Collections.singletonList(refreshListeners),
25002501
Collections.singletonList(new RefreshMetricUpdater(refreshMetric)),
@@ -3077,7 +3078,9 @@ public void afterRefresh(boolean didRefresh) throws IOException {
30773078

30783079
private EngineConfig.TombstoneDocSupplier tombstoneDocSupplier() {
30793080
final RootObjectMapper.Builder noopRootMapper = new RootObjectMapper.Builder("__noop");
3080-
final DocumentMapper noopDocumentMapper = new DocumentMapper.Builder(noopRootMapper, mapperService).build(mapperService);
3081+
final DocumentMapper noopDocumentMapper = mapperService != null ?
3082+
new DocumentMapper.Builder(noopRootMapper, mapperService).build(mapperService) :
3083+
null;
30813084
return new EngineConfig.TombstoneDocSupplier() {
30823085
@Override
30833086
public ParsedDocument newDeleteTombstoneDoc(String type, String id) {

‎server/src/main/java/org/elasticsearch/indices/IndicesService.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,8 @@
157157
import static java.util.Collections.unmodifiableMap;
158158
import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
159159
import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
160+
import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX;
161+
import static org.elasticsearch.index.IndexService.IndexCreationContext.META_DATA_VERIFICATION;
160162
import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
161163

162164
public class IndicesService extends AbstractLifecycleComponent
@@ -492,7 +494,7 @@ public void onStoreClosed(ShardId shardId) {
492494
finalListeners.add(oldShardsStats);
493495
final IndexService indexService =
494496
createIndexService(
495-
"create index",
497+
CREATE_INDEX,
496498
indexMetaData,
497499
indicesQueryCache,
498500
indicesFieldDataCache,
@@ -514,7 +516,7 @@ public void onStoreClosed(ShardId shardId) {
514516
/**
515517
* This creates a new IndexService without registering it
516518
*/
517-
private synchronized IndexService createIndexService(final String reason,
519+
private synchronized IndexService createIndexService(IndexService.IndexCreationContext indexCreationContext,
518520
IndexMetaData indexMetaData,
519521
IndicesQueryCache indicesQueryCache,
520522
IndicesFieldDataCache indicesFieldDataCache,
@@ -532,7 +534,7 @@ private synchronized IndexService createIndexService(final String reason,
532534
indexMetaData.getIndex(),
533535
idxSettings.getNumberOfShards(),
534536
idxSettings.getNumberOfReplicas(),
535-
reason);
537+
indexCreationContext);
536538

537539
final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings), indexStoreFactories);
538540
for (IndexingOperationListener operationListener : indexingOperationListeners) {
@@ -543,6 +545,7 @@ private synchronized IndexService createIndexService(final String reason,
543545
indexModule.addIndexEventListener(listener);
544546
}
545547
return indexModule.newIndexService(
548+
indexCreationContext,
546549
nodeEnv,
547550
xContentRegistry,
548551
this,
@@ -621,7 +624,7 @@ public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaDa
621624
closeables.add(indicesQueryCache);
622625
// this will also fail if some plugin fails etc. which is nice since we can verify that early
623626
final IndexService service =
624-
createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList());
627+
createIndexService(META_DATA_VERIFICATION, metaData, indicesQueryCache, indicesFieldDataCache, emptyList());
625628
closeables.add(() -> service.close("metadata verification", false));
626629
service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY);
627630
if (metaData.equals(metaDataUpdate) == false) {

‎server/src/test/java/org/elasticsearch/index/IndexModuleTests.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
import java.util.function.Function;
9090

9191
import static java.util.Collections.emptyMap;
92+
import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX;
9293
import static org.hamcrest.Matchers.containsString;
9394
import static org.hamcrest.Matchers.hasToString;
9495
import static org.hamcrest.Matchers.instanceOf;
@@ -148,8 +149,8 @@ public void tearDown() throws Exception {
148149
}
149150

150151
private IndexService newIndexService(IndexModule module) throws IOException {
151-
return module.newIndexService(nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool,
152-
scriptService, null, indicesQueryCache, mapperRegistry,
152+
return module.newIndexService(CREATE_INDEX, nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays,
153+
threadPool, scriptService, null, indicesQueryCache, mapperRegistry,
153154
new IndicesFieldDataCache(settings, listener), writableRegistry());
154155
}
155156

‎test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1052,7 +1052,7 @@ public static List<Translog.Operation> readAllOperationsInLucene(Engine engine,
10521052
* Asserts the provided engine has a consistent document history between translog and Lucene index.
10531053
*/
10541054
public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine engine, MapperService mapper) throws IOException {
1055-
if (mapper.documentMapper() == null || engine.config().getIndexSettings().isSoftDeleteEnabled() == false
1055+
if (mapper == null || mapper.documentMapper() == null || engine.config().getIndexSettings().isSoftDeleteEnabled() == false
10561056
|| (engine instanceof InternalEngine) == false) {
10571057
return;
10581058
}

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,7 @@ public void onIndexModule(IndexModule module) {
687687
throw new IllegalArgumentException("permission filters are not allowed to use the current timestamp");
688688

689689
}, null),
690-
indexService.cache().bitsetFilterCache(),
690+
indexService.cache() != null ? indexService.cache().bitsetFilterCache() : null,
691691
indexService.getThreadPool().getThreadContext(), getLicenseState(),
692692
indexService.getScriptService()));
693693
/* We need to forcefully overwrite the query cache implementation to use security's opt out query cache implementation.

0 commit comments

Comments
 (0)