Skip to content

Commit 7e9b8cd

Browse files
martijnvgmartijnvg
authored
Reuse previous indices lookup when possible (#79804)
Backporting #79004 to 7.16 branch. In cases when indices, aliases and data streams aren't modified then the indices lookup can be reused. For example in: * The IndexMetadataUpdater#applyChanges(...) method builds a new metadata instance, but only primary term or insync allocations may be updated. No new indices, aliases or data streams are added, so re-building indices lookup is not necessary. * MasterService#patchVersions Additionally the logic that checks when indices lookup can be reused, this logic also checks the hidden and system flags of indices/datastreams. In clusters with many indices the cost of building indices lookup is non-neglectable and should be avoided in this case. Closes #78980 Partially addresses #77888
1 parent 5b7211f commit 7e9b8cd

File tree

11 files changed

+225
-13
lines changed

11 files changed

+225
-13
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,23 @@ public boolean isSystem() {
182182
public List<String> getAliases() {
183183
return aliases;
184184
}
185+
186+
@Override
187+
public boolean equals(Object o) {
188+
if (this == o) return true;
189+
if (o == null || getClass() != o.getClass()) return false;
190+
ConcreteIndex that = (ConcreteIndex) o;
191+
return isHidden == that.isHidden &&
192+
isSystem == that.isSystem &&
193+
concreteIndexName.equals(that.concreteIndexName) &&
194+
Objects.equals(aliases, that.aliases) &&
195+
Objects.equals(dataStream, that.dataStream);
196+
}
197+
198+
@Override
199+
public int hashCode() {
200+
return Objects.hash(concreteIndexName, isHidden, isSystem, aliases, dataStream);
201+
}
185202
}
186203

187204
/**
@@ -302,6 +319,24 @@ private void validateAliasProperties(List<IndexMetadata> referenceIndexMetadatas
302319
private boolean isNonEmpty(List<IndexMetadata> idxMetas) {
303320
return (Objects.isNull(idxMetas) || idxMetas.isEmpty()) == false;
304321
}
322+
323+
@Override
324+
public boolean equals(Object o) {
325+
if (this == o) return true;
326+
if (o == null || getClass() != o.getClass()) return false;
327+
Alias alias = (Alias) o;
328+
return isHidden == alias.isHidden &&
329+
isSystem == alias.isSystem &&
330+
dataStreamAlias == alias.dataStreamAlias &&
331+
aliasName.equals(alias.aliasName) &&
332+
referenceIndexMetadatas.equals(alias.referenceIndexMetadatas) &&
333+
Objects.equals(writeIndex, alias.writeIndex);
334+
}
335+
336+
@Override
337+
public int hashCode() {
338+
return Objects.hash(aliasName, referenceIndexMetadatas, writeIndex, isHidden, isSystem, dataStreamAlias);
339+
}
305340
}
306341

307342
class DataStream implements IndexAbstraction {
@@ -363,6 +398,20 @@ public List<String> getAliases() {
363398
public org.elasticsearch.cluster.metadata.DataStream getDataStream() {
364399
return dataStream;
365400
}
401+
402+
@Override
403+
public boolean equals(Object o) {
404+
if (this == o) return true;
405+
if (o == null || getClass() != o.getClass()) return false;
406+
DataStream that = (DataStream) o;
407+
return dataStream.equals(that.dataStream) &&
408+
Objects.equals(referencedByDataStreamAliases, that.referencedByDataStreamAliases);
409+
}
410+
411+
@Override
412+
public int hashCode() {
413+
return Objects.hash(dataStream, referencedByDataStreamAliases);
414+
}
366415
}
367416

368417
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java‎

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1160,15 +1160,18 @@ public static class Builder {
11601160
private final ImmutableOpenMap.Builder<String, IndexTemplateMetadata> templates;
11611161
private final ImmutableOpenMap.Builder<String, Custom> customs;
11621162

1163+
private SortedMap<String, IndexAbstraction> previousIndicesLookup;
1164+
11631165
public Builder() {
11641166
clusterUUID = UNKNOWN_CLUSTER_UUID;
11651167
indices = ImmutableOpenMap.builder();
11661168
templates = ImmutableOpenMap.builder();
11671169
customs = ImmutableOpenMap.builder();
11681170
indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize
1171+
previousIndicesLookup = null;
11691172
}
11701173

1171-
public Builder(Metadata metadata) {
1174+
Builder(Metadata metadata) {
11721175
this.clusterUUID = metadata.clusterUUID;
11731176
this.clusterUUIDCommitted = metadata.clusterUUIDCommitted;
11741177
this.coordinationMetadata = metadata.coordinationMetadata;
@@ -1179,13 +1182,17 @@ public Builder(Metadata metadata) {
11791182
this.indices = ImmutableOpenMap.builder(metadata.indices);
11801183
this.templates = ImmutableOpenMap.builder(metadata.templates);
11811184
this.customs = ImmutableOpenMap.builder(metadata.customs);
1185+
previousIndicesLookup = metadata.getIndicesLookup();
11821186
}
11831187

11841188
public Builder put(IndexMetadata.Builder indexMetadataBuilder) {
11851189
// we know its a new one, increment the version and store
11861190
indexMetadataBuilder.version(indexMetadataBuilder.version() + 1);
11871191
IndexMetadata indexMetadata = indexMetadataBuilder.build();
1188-
indices.put(indexMetadata.getIndex().getName(), indexMetadata);
1192+
IndexMetadata previous = indices.put(indexMetadata.getIndex().getName(), indexMetadata);
1193+
if (unsetPreviousIndicesLookup(previous, indexMetadata)) {
1194+
previousIndicesLookup = null;
1195+
}
11891196
return this;
11901197
}
11911198

@@ -1197,10 +1204,37 @@ public Builder put(IndexMetadata indexMetadata, boolean incrementVersion) {
11971204
if (incrementVersion) {
11981205
indexMetadata = IndexMetadata.builder(indexMetadata).version(indexMetadata.getVersion() + 1).build();
11991206
}
1200-
indices.put(indexMetadata.getIndex().getName(), indexMetadata);
1207+
IndexMetadata previous = indices.put(indexMetadata.getIndex().getName(), indexMetadata);
1208+
if (unsetPreviousIndicesLookup(previous, indexMetadata)) {
1209+
previousIndicesLookup = null;
1210+
}
12011211
return this;
12021212
}
12031213

1214+
boolean unsetPreviousIndicesLookup(IndexMetadata previous, IndexMetadata current) {
1215+
if (previous == null) {
1216+
return true;
1217+
}
1218+
1219+
if (previous.getAliases().equals(current.getAliases()) == false) {
1220+
return true;
1221+
}
1222+
1223+
if (previous.isHidden() != current.isHidden()) {
1224+
return true;
1225+
}
1226+
1227+
if (previous.isSystem() != current.isSystem()) {
1228+
return true;
1229+
}
1230+
1231+
if (previous.getState() != current.getState()) {
1232+
return true;
1233+
}
1234+
1235+
return false;
1236+
}
1237+
12041238
public IndexMetadata get(String index) {
12051239
return indices.get(index);
12061240
}
@@ -1219,16 +1253,22 @@ public IndexMetadata getSafe(Index index) {
12191253
}
12201254

12211255
public Builder remove(String index) {
1256+
previousIndicesLookup = null;
1257+
12221258
indices.remove(index);
12231259
return this;
12241260
}
12251261

12261262
public Builder removeAllIndices() {
1263+
previousIndicesLookup = null;
1264+
12271265
indices.clear();
12281266
return this;
12291267
}
12301268

12311269
public Builder indices(ImmutableOpenMap<String, IndexMetadata> indices) {
1270+
previousIndicesLookup = null;
1271+
12321272
this.indices.putAll(indices);
12331273
return this;
12341274
}
@@ -1309,6 +1349,8 @@ public Builder removeIndexTemplate(String name) {
13091349
}
13101350

13111351
public DataStream dataStream(String dataStreamName) {
1352+
previousIndicesLookup = null;
1353+
13121354
DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) customs.get(DataStreamMetadata.TYPE);
13131355
if (dataStreamMetadata != null) {
13141356
return dataStreamMetadata.dataStreams().get(dataStreamName);
@@ -1318,11 +1360,15 @@ public DataStream dataStream(String dataStreamName) {
13181360
}
13191361

13201362
public Builder dataStreams(Map<String, DataStream> dataStreams, Map<String, DataStreamAlias> dataStreamAliases) {
1363+
previousIndicesLookup = null;
1364+
13211365
this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams, dataStreamAliases));
13221366
return this;
13231367
}
13241368

13251369
public Builder put(DataStream dataStream) {
1370+
previousIndicesLookup = null;
1371+
13261372
Objects.requireNonNull(dataStream, "it is invalid to add a null data stream");
13271373
Map<String, DataStream> existingDataStreams =
13281374
Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
@@ -1339,6 +1385,8 @@ public Builder put(DataStream dataStream) {
13391385
}
13401386

13411387
public boolean put(String aliasName, String dataStream, Boolean isWriteDataStream, String filter) {
1388+
previousIndicesLookup = null;
1389+
13421390
Map<String, DataStream> existingDataStream =
13431391
Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
13441392
.map(dsmd -> new HashMap<>(dsmd.dataStreams()))
@@ -1377,6 +1425,8 @@ public boolean put(String aliasName, String dataStream, Boolean isWriteDataStrea
13771425
}
13781426

13791427
public Builder removeDataStream(String name) {
1428+
previousIndicesLookup = null;
1429+
13801430
Map<String, DataStream> existingDataStreams =
13811431
Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
13821432
.map(dsmd -> new HashMap<>(dsmd.dataStreams()))
@@ -1413,6 +1463,8 @@ public Builder removeDataStream(String name) {
14131463
}
14141464

14151465
public boolean removeDataStreamAlias(String aliasName, String dataStreamName, boolean mustExist) {
1466+
previousIndicesLookup = null;
1467+
14161468
Map<String, DataStreamAlias> dataStreamAliases =
14171469
Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
14181470
.map(dsmd -> new HashMap<>(dsmd.getDataStreamAliases()))
@@ -1665,10 +1717,15 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
16651717
ImmutableOpenMap<String, IndexMetadata> indices = this.indices.build();
16661718

16671719
SortedMap<String, IndexAbstraction> indicesLookup;
1668-
if (builtIndicesLookupEagerly) {
1669-
indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices));
1720+
if (previousIndicesLookup != null) {
1721+
assert previousIndicesLookup.equals(buildIndicesLookup(dataStreamMetadata, indices));
1722+
indicesLookup = previousIndicesLookup;
16701723
} else {
1671-
indicesLookup = null;
1724+
if (builtIndicesLookupEagerly) {
1725+
indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices));
1726+
} else {
1727+
indicesLookup = null;
1728+
}
16721729
}
16731730

16741731

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,9 @@ public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable)
121121
}
122122

123123
if (metadataBuilder != null) {
124-
return metadataBuilder.build();
124+
Metadata newMetadata = metadataBuilder.build();
125+
assert oldMetadata.getIndicesLookup() == newMetadata.getIndicesLookup();
126+
return newMetadata;
125127
} else {
126128
return oldMetadata;
127129
}

‎server/src/main/java/org/elasticsearch/cluster/service/MasterService.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.cluster.ClusterStateTaskListener;
2424
import org.elasticsearch.cluster.coordination.ClusterStatePublisher;
2525
import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
26+
import org.elasticsearch.cluster.metadata.IndexAbstraction;
2627
import org.elasticsearch.cluster.metadata.Metadata;
2728
import org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException;
2829
import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -53,6 +54,7 @@
5354
import java.util.Locale;
5455
import java.util.Map;
5556
import java.util.Objects;
57+
import java.util.SortedMap;
5658
import java.util.concurrent.TimeUnit;
5759
import java.util.function.LongSupplier;
5860
import java.util.function.Supplier;
@@ -374,6 +376,7 @@ private ClusterState patchVersions(ClusterState previousClusterState, ClusterTas
374376

375377
if (previousClusterState != newClusterState) {
376378
// only the master controls the version numbers
379+
final SortedMap<String, IndexAbstraction> previousIndicesLookup = newClusterState.metadata().getIndicesLookup();
377380
Builder builder = incrementVersion(newClusterState);
378381
if (previousClusterState.routingTable() != newClusterState.routingTable()) {
379382
builder.routingTable(RoutingTable.builder(newClusterState.routingTable())
@@ -384,6 +387,7 @@ private ClusterState patchVersions(ClusterState previousClusterState, ClusterTas
384387
}
385388

386389
newClusterState = builder.build();
390+
assert previousIndicesLookup == newClusterState.metadata().getIndicesLookup();
387391
}
388392

389393
return newClusterState;

‎server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ private static ClusterState createNonInitializedState(final int numNodes, final
354354
private static ClusterState nextState(final ClusterState previousState, List<TestCustomMetadata> customMetadataList) {
355355
final ClusterState.Builder builder = ClusterState.builder(previousState);
356356
builder.stateUUID(UUIDs.randomBase64UUID());
357-
Metadata.Builder metadataBuilder = new Metadata.Builder(previousState.metadata());
357+
Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata());
358358
for (ObjectObjectCursor<String, Metadata.Custom> customMetadata : previousState.metadata().customs()) {
359359
if (customMetadata.value instanceof TestCustomMetadata) {
360360
metadataBuilder.removeCustom(customMetadata.key);

0 commit comments

Comments
 (0)