Skip to content

Commit 6899ce6

Browse files
williamrandolphwilliamrandolph
authored
System index auto-creation should not be disabled by user settings (#62984) (#63147)
* Add System Indices check to AutoCreateIndex By default, Elasticsearch auto-creates indices when a document is submitted to a non-existent index. There is a setting that allows users to disable this behavior. However, this setting should not apply to system indices, so that Elasticsearch modules and plugins are able to use auto-create behavior whether or not it is exposed to users. This commit constructs the AutoCreateIndex object with a reference to the SystemIndices object so that we bypass the check for the user-facing autocreate setting when it's a system index that is being autocreated. We also modify the logic in TransportBulkAction to make sure that if a system index is included in a bulk request, we don't skip the autocreation step.
1 parent fc13b72 commit 6899ce6

File tree

11 files changed

+105
-31
lines changed

11 files changed

+105
-31
lines changed

‎modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@
3535
import org.elasticsearch.common.bytes.BytesReference;
3636
import org.elasticsearch.common.settings.ClusterSettings;
3737
import org.elasticsearch.common.settings.Settings;
38+
import org.elasticsearch.indices.SystemIndices;
3839
import org.elasticsearch.test.ESTestCase;
3940

41+
import java.util.HashMap;
42+
4043
import static java.util.Collections.emptyMap;
4144
import static org.hamcrest.Matchers.containsString;
4245

@@ -60,7 +63,8 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
6063
.put(index("source2", "source_multi"), true)).build();
6164
private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver();
6265
private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY,
63-
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER);
66+
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER,
67+
new SystemIndices(new HashMap<>()));
6468

6569
private final BytesReference query = new BytesArray("{ \"foo\" : \"bar\" }");
6670

‎server/src/main/java/org/elasticsearch/action/ActionModule.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@
250250
import org.elasticsearch.common.settings.Settings;
251251
import org.elasticsearch.common.settings.SettingsFilter;
252252
import org.elasticsearch.index.seqno.RetentionLeaseActions;
253+
import org.elasticsearch.indices.SystemIndices;
253254
import org.elasticsearch.indices.breaker.CircuitBreakerService;
254255
import org.elasticsearch.persistent.CompletionPersistentTaskAction;
255256
import org.elasticsearch.persistent.RemovePersistentTaskAction;
@@ -427,7 +428,7 @@ public class ActionModule extends AbstractModule {
427428
public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
428429
IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
429430
ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
430-
CircuitBreakerService circuitBreakerService, UsageService usageService) {
431+
CircuitBreakerService circuitBreakerService, UsageService usageService, SystemIndices systemIndices) {
431432
this.transportClient = transportClient;
432433
this.settings = settings;
433434
this.indexNameExpressionResolver = indexNameExpressionResolver;
@@ -438,7 +439,9 @@ public ActionModule(boolean transportClient, Settings settings, IndexNameExpress
438439
this.threadPool = threadPool;
439440
actions = setupActions(actionPlugins);
440441
actionFilters = setupActionFilters(actionPlugins);
441-
autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
442+
autoCreateIndex = transportClient
443+
? null
444+
: new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, systemIndices);
442445
destructiveOperations = new DestructiveOperations(settings, clusterSettings);
443446
Set<RestHeaderDefinition> headers = Stream.concat(
444447
actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()),

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,9 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, String exec
229229
return;
230230
}
231231

232-
if (needToCheck()) {
232+
final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
233+
234+
if (includesSystem || needToCheck()) {
233235
// Attempt to create all the indices that we're going to need during the bulk before we start.
234236
// Step 1: collect all the indices in the request
235237
final Map<String, Boolean> indices = bulkRequest.requests.stream()
@@ -352,15 +354,20 @@ static void prohibitCustomRoutingOnDataStream(DocWriteRequest<?> writeRequest, M
352354
}
353355

354356
boolean isOnlySystem(BulkRequest request, SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices) {
355-
final boolean onlySystem = request.getIndices().stream().allMatch(indexName -> {
356-
final IndexAbstraction abstraction = indicesLookup.get(indexName);
357-
if (abstraction != null) {
358-
return abstraction.isSystem();
359-
} else {
360-
return systemIndices.isSystemIndex(indexName);
361-
}
362-
});
363-
return onlySystem;
357+
return request.getIndices().stream().allMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName));
358+
}
359+
360+
boolean includesSystem(BulkRequest request, SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices) {
361+
return request.getIndices().stream().anyMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName));
362+
}
363+
364+
private boolean isSystemIndex(SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices, String indexName) {
365+
final IndexAbstraction abstraction = indicesLookup.get(indexName);
366+
if (abstraction != null) {
367+
return abstraction.isSystem();
368+
} else {
369+
return systemIndices.isSystemIndex(indexName);
370+
}
364371
}
365372

366373
boolean needToCheck() {

‎server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.common.settings.Settings;
3232
import org.elasticsearch.index.IndexNotFoundException;
3333
import org.elasticsearch.index.mapper.MapperService;
34+
import org.elasticsearch.indices.SystemIndices;
3435

3536
import java.util.ArrayList;
3637
import java.util.List;
@@ -46,11 +47,16 @@ public final class AutoCreateIndex {
4647

4748
private final boolean dynamicMappingDisabled;
4849
private final IndexNameExpressionResolver resolver;
50+
private final SystemIndices systemIndices;
4951
private volatile AutoCreate autoCreate;
5052

51-
public AutoCreateIndex(Settings settings, ClusterSettings clusterSettings, IndexNameExpressionResolver resolver) {
53+
public AutoCreateIndex(Settings settings,
54+
ClusterSettings clusterSettings,
55+
IndexNameExpressionResolver resolver,
56+
SystemIndices systemIndices) {
5257
this.resolver = resolver;
5358
dynamicMappingDisabled = !MapperService.INDEX_MAPPER_DYNAMIC_SETTING.get(settings);
59+
this.systemIndices = systemIndices;
5460
this.autoCreate = AUTO_CREATE_INDEX_SETTING.get(settings);
5561
clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate);
5662
}
@@ -70,6 +76,12 @@ public boolean shouldAutoCreate(String index, ClusterState state) {
7076
if (resolver.hasIndexAbstraction(index, state)) {
7177
return false;
7278
}
79+
80+
// Always auto-create system indexes
81+
if (systemIndices.isSystemIndex(index)) {
82+
return true;
83+
}
84+
7385
// One volatile read, so that all checks are done against the same instance:
7486
final AutoCreate autoCreate = this.autoCreate;
7587
if (autoCreate.autoCreateIndex == false) {

‎server/src/main/java/org/elasticsearch/client/transport/TransportClient.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4949
import org.elasticsearch.core.internal.io.IOUtils;
5050
import org.elasticsearch.indices.IndicesModule;
51+
import org.elasticsearch.indices.SystemIndices;
5152
import org.elasticsearch.indices.breaker.CircuitBreakerService;
5253
import org.elasticsearch.node.InternalSettingsPreparer;
5354
import org.elasticsearch.node.Node;
@@ -67,14 +68,17 @@
6768
import java.util.ArrayList;
6869
import java.util.Arrays;
6970
import java.util.Collection;
70-
import java.util.Collections;
7171
import java.util.List;
7272
import java.util.Set;
7373
import java.util.concurrent.TimeUnit;
7474
import java.util.function.Function;
7575
import java.util.stream.Collectors;
7676
import java.util.stream.Stream;
7777

78+
import static java.util.Collections.emptyList;
79+
import static java.util.Collections.emptyMap;
80+
import static java.util.Collections.emptySet;
81+
import static java.util.Collections.unmodifiableList;
7882
import static java.util.stream.Collectors.toList;
7983
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
8084

@@ -150,18 +154,18 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
150154
final List<Closeable> resourcesToClose = new ArrayList<>();
151155
final ThreadPool threadPool = new ThreadPool(settings);
152156
resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
153-
final NetworkService networkService = new NetworkService(Collections.emptyList());
157+
final NetworkService networkService = new NetworkService(emptyList());
154158
try {
155159
final List<Setting<?>> additionalSettings = new ArrayList<>(pluginsService.getPluginSettings());
156160
final List<String> additionalSettingsFilter = new ArrayList<>(pluginsService.getPluginSettingsFilter());
157161
for (final ExecutorBuilder<?> builder : threadPool.builders()) {
158162
additionalSettings.addAll(builder.getRegisteredSettings());
159163
}
160164
SettingsModule settingsModule =
161-
new SettingsModule(settings, additionalSettings, additionalSettingsFilter, Collections.emptySet());
165+
new SettingsModule(settings, additionalSettings, additionalSettingsFilter, emptySet());
162166

163167
SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
164-
IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
168+
IndicesModule indicesModule = new IndicesModule(emptyList());
165169
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
166170
entries.addAll(NetworkModule.getNamedWriteables());
167171
entries.addAll(searchModule.getNamedWriteables());
@@ -185,11 +189,11 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
185189
modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
186190
ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getIndexScopedSettings(),
187191
settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(), threadPool,
188-
pluginsService.filterPlugins(ActionPlugin.class), null, null, null);
192+
pluginsService.filterPlugins(ActionPlugin.class), null, null, null, new SystemIndices(emptyMap()));
189193
modules.add(actionModule);
190194

191195
CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
192-
Collections.emptyList(),
196+
emptyList(),
193197
settingsModule.getClusterSettings());
194198
resourcesToClose.add(circuitBreakerService);
195199
PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings);
@@ -202,7 +206,7 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings
202206
final TransportService transportService = new TransportService(settings, transport, threadPool,
203207
networkModule.getTransportInterceptor(),
204208
boundTransportAddress -> DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 0),
205-
UUIDs.randomBase64UUID()), null, Collections.emptySet());
209+
UUIDs.randomBase64UUID()), null, emptySet());
206210
modules.add((b -> {
207211
b.bind(BigArrays.class).toInstance(bigArrays);
208212
b.bind(PageCacheRecycler.class).toInstance(pageCacheRecycler);
@@ -300,7 +304,7 @@ protected TransportClient(Settings settings, Settings defaultSettings, Collectio
300304
private TransportClient(ClientTemplate template) {
301305
super(template.getSettings(), template.getThreadPool());
302306
this.injector = template.injector;
303-
this.pluginLifecycleComponents = Collections.unmodifiableList(template.pluginLifecycleComponents);
307+
this.pluginLifecycleComponents = unmodifiableList(template.pluginLifecycleComponents);
304308
this.nodesService = template.nodesService;
305309
this.proxy = template.proxy;
306310
this.namedWriteableRegistry = template.namedWriteableRegistry;

‎server/src/main/java/org/elasticsearch/node/Node.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -539,7 +539,7 @@ protected Node(final Environment initialEnvironment,
539539

540540
ActionModule actionModule = new ActionModule(false, settings, clusterModule.getIndexNameExpressionResolver(),
541541
settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
542-
threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService);
542+
threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService, systemIndices);
543543
modules.add(actionModule);
544544

545545
final RestController restController = actionModule.getRestController();

‎server/src/test/java/org/elasticsearch/action/ActionModuleTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() {
109109
UsageService usageService = new UsageService();
110110
ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
111111
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
112-
null, usageService);
112+
null, usageService, null);
113113
actionModule.initRestHandlers(null);
114114
// At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
115115
Exception e = expectThrows(IllegalArgumentException.class, () ->
@@ -148,7 +148,7 @@ public String getName() {
148148
UsageService usageService = new UsageService();
149149
ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
150150
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
151-
singletonList(dupsMainAction), null, null, usageService);
151+
singletonList(dupsMainAction), null, null, usageService, null);
152152
Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
153153
assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/] for method: GET"));
154154
} finally {
@@ -182,7 +182,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
182182
UsageService usageService = new UsageService();
183183
ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
184184
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
185-
singletonList(registersFakeHandler), null, null, usageService);
185+
singletonList(registersFakeHandler), null, null, usageService, null);
186186
actionModule.initRestHandlers(null);
187187
// At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
188188
Exception e = expectThrows(IllegalArgumentException.class, () ->

‎server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ class TestTransportBulkAction extends TransportBulkAction {
147147
null, null, new ActionFilters(Collections.emptySet()), null,
148148
new AutoCreateIndex(
149149
SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
150-
new IndexNameExpressionResolver()
150+
new IndexNameExpressionResolver(),
151+
new SystemIndices(emptyMap())
151152
), new IndexingPressure(SETTINGS), new SystemIndices(emptyMap())
152153
);
153154
}

‎server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ class TestTransportBulkAction extends TransportBulkAction {
8888
TestTransportBulkAction() {
8989
super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null,
9090
null, new ActionFilters(Collections.emptySet()), new Resolver(),
91-
new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()),
91+
new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(emptyMap())),
9292
new IndexingPressure(Settings.EMPTY), new SystemIndices(emptyMap()));
9393
}
9494

@@ -271,6 +271,28 @@ public void testOnlySystem() {
271271
assertFalse(bulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
272272
}
273273

274+
public void testIncludesSystem() {
275+
SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
276+
Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build();
277+
indicesLookup.put(".foo",
278+
new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
279+
indicesLookup.put(".bar",
280+
new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
281+
SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("plugin",
282+
org.elasticsearch.common.collect.List.of(new SystemIndexDescriptor(".test", ""))));
283+
List<String> onlySystem = org.elasticsearch.common.collect.List.of(".foo", ".bar");
284+
assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
285+
286+
onlySystem = org.elasticsearch.common.collect.List.of(".foo", ".bar", ".test");
287+
assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
288+
289+
List<String> nonSystem = org.elasticsearch.common.collect.List.of("foo", "bar");
290+
assertFalse(bulkAction.includesSystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices));
291+
292+
List<String> mixed = org.elasticsearch.common.collect.List.of(".foo", ".test", "other");
293+
assertTrue(bulkAction.includesSystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
294+
}
295+
274296
private BulkRequest buildBulkRequest(List<String> indices) {
275297
BulkRequest request = new BulkRequest();
276298
for (String index : indices) {

0 commit comments

Comments
 (0)