Skip to content

Commit 97b9f72

Browse files
mayya-sharipovamayya-sharipova
committed
Revert "Add default leaf sorter for data streams"
This reverts commit 5360987. As we want to dynamically specify in an index belongs to data stream.
1 parent 5360987 commit 97b9f72

File tree

21 files changed

+37
-191
lines changed

21 files changed

+37
-191
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ EngineConfig engineConfigWithLargerIndexingMemory(EngineConfig config) {
5757
config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(),
5858
config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(),
5959
config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(),
60-
config.getPrimaryTermSupplier(), config.getSnapshotCommitSupplier(), null);
60+
config.getPrimaryTermSupplier(), config.getSnapshotCommitSupplier());
6161
}
6262

6363
@Override

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,6 @@
77
*/
88
package org.elasticsearch.cluster.metadata;
99

10-
import org.apache.lucene.document.LongPoint;
11-
import org.apache.lucene.index.LeafReader;
12-
import org.apache.lucene.index.PointValues;
1310
import org.elasticsearch.cluster.AbstractDiffable;
1411
import org.elasticsearch.cluster.Diff;
1512
import org.elasticsearch.core.Nullable;
@@ -28,7 +25,6 @@
2825
import java.util.ArrayList;
2926
import java.util.Collection;
3027
import java.util.Collections;
31-
import java.util.Comparator;
3228
import java.util.HashMap;
3329
import java.util.List;
3430
import java.util.Locale;
@@ -40,21 +36,6 @@ public final class DataStream extends AbstractDiffable<DataStream> implements To
4036

4137
public static final String BACKING_INDEX_PREFIX = ".ds-";
4238
public static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd");
43-
// Datastreams' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations
44-
public static Comparator<LeafReader> DATASTREAM_LEAF_READERS_SORTER =
45-
Comparator.comparingLong(
46-
(LeafReader r) -> {
47-
try {
48-
PointValues points = r.getPointValues(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD);
49-
if (points != null) {
50-
byte[] sortValue = points.getMaxPackedValue();
51-
return LongPoint.decodeDimension(sortValue, 0);
52-
}
53-
} catch (IOException e) {
54-
}
55-
// this should not happen, as all data stream segments must contain timestamp field
56-
return Long.MAX_VALUE; })
57-
.reversed();
5839

5940
private final LongSupplier timeProvider;
6041
private final String name;

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java‎

Lines changed: 4 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -339,13 +339,11 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
339339
static final String KEY_ROLLOVER_INFOS = "rollover_info";
340340
static final String KEY_SYSTEM = "system";
341341
static final String KEY_TIMESTAMP_RANGE = "timestamp_range";
342-
static final String KEY_DATASTREAM_INDEX = "datastream_index";
343342
public static final String KEY_PRIMARY_TERMS = "primary_terms";
344343

345344
public static final String INDEX_STATE_FILE_PREFIX = "state-";
346345

347346
static final Version SYSTEM_INDEX_FLAG_ADDED = Version.V_7_10_0;
348-
static final Version DATASTREAM_INDEX_FLAG_ADDED = Version.CURRENT; //TODO: correct to 7.15
349347

350348
private final int routingNumShards;
351349
private final int routingFactor;
@@ -389,7 +387,6 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
389387
private final ActiveShardCount waitForActiveShards;
390388
private final ImmutableOpenMap<String, RolloverInfo> rolloverInfos;
391389
private final boolean isSystem;
392-
private final boolean isDataStreamIndex;
393390

394391
private final IndexLongFieldRange timestampRange;
395392

@@ -418,8 +415,7 @@ private IndexMetadata(
418415
final ActiveShardCount waitForActiveShards,
419416
final ImmutableOpenMap<String, RolloverInfo> rolloverInfos,
420417
final boolean isSystem,
421-
final IndexLongFieldRange timestampRange,
422-
final boolean isDataStreamIndex) {
418+
final IndexLongFieldRange timestampRange) {
423419

424420
this.index = index;
425421
this.version = version;
@@ -452,7 +448,6 @@ private IndexMetadata(
452448
this.rolloverInfos = rolloverInfos;
453449
this.isSystem = isSystem;
454450
this.timestampRange = timestampRange;
455-
this.isDataStreamIndex = isDataStreamIndex;
456451
assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards;
457452
}
458453

@@ -679,9 +674,6 @@ public boolean equals(Object o) {
679674
if (isSystem != that.isSystem) {
680675
return false;
681676
}
682-
if (isDataStreamIndex != that.isDataStreamIndex) {
683-
return false;
684-
}
685677
return true;
686678
}
687679

@@ -700,7 +692,6 @@ public int hashCode() {
700692
result = 31 * result + inSyncAllocationIds.hashCode();
701693
result = 31 * result + rolloverInfos.hashCode();
702694
result = 31 * result + Boolean.hashCode(isSystem);
703-
result = 31 * result + Boolean.hashCode(isDataStreamIndex);
704695
return result;
705696
}
706697

@@ -742,7 +733,6 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
742733
private final Diff<ImmutableOpenMap<String, RolloverInfo>> rolloverInfos;
743734
private final boolean isSystem;
744735
private final IndexLongFieldRange timestampRange;
745-
private final boolean isDataStreamIndex;
746736

747737
IndexMetadataDiff(IndexMetadata before, IndexMetadata after) {
748738
index = after.index.getName();
@@ -761,7 +751,6 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
761751
DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance());
762752
rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer());
763753
isSystem = after.isSystem;
764-
isDataStreamIndex = after.isDataStreamIndex;
765754
timestampRange = after.timestampRange;
766755
}
767756

@@ -801,11 +790,6 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
801790
isSystem = false;
802791
}
803792
timestampRange = IndexLongFieldRange.readFrom(in);
804-
if (in.getVersion().onOrAfter(DATASTREAM_INDEX_FLAG_ADDED)) {
805-
isDataStreamIndex = in.readBoolean();
806-
} else {
807-
isDataStreamIndex = false;
808-
}
809793
}
810794

811795
@Override
@@ -830,9 +814,6 @@ public void writeTo(StreamOutput out) throws IOException {
830814
out.writeBoolean(isSystem);
831815
}
832816
timestampRange.writeTo(out);
833-
if (out.getVersion().onOrAfter(DATASTREAM_INDEX_FLAG_ADDED)) {
834-
out.writeBoolean(isDataStreamIndex);
835-
}
836817
}
837818

838819
@Override
@@ -852,7 +833,6 @@ public IndexMetadata apply(IndexMetadata part) {
852833
builder.inSyncAllocationIds.putAll(inSyncAllocationIds.apply(part.inSyncAllocationIds));
853834
builder.rolloverInfos.putAll(rolloverInfos.apply(part.rolloverInfos));
854835
builder.system(isSystem);
855-
builder.dataStreamIndex(isDataStreamIndex);
856836
builder.timestampRange(timestampRange);
857837
return builder.build();
858838
}
@@ -900,9 +880,6 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException {
900880
builder.system(in.readBoolean());
901881
}
902882
builder.timestampRange(IndexLongFieldRange.readFrom(in));
903-
if (in.getVersion().onOrAfter(DATASTREAM_INDEX_FLAG_ADDED)) {
904-
builder.dataStreamIndex(in.readBoolean());
905-
}
906883
return builder.build();
907884
}
908885

@@ -945,19 +922,12 @@ public void writeTo(StreamOutput out) throws IOException {
945922
out.writeBoolean(isSystem);
946923
}
947924
timestampRange.writeTo(out);
948-
if (out.getVersion().onOrAfter(DATASTREAM_INDEX_FLAG_ADDED)) {
949-
out.writeBoolean(isDataStreamIndex);
950-
}
951925
}
952926

953927
public boolean isSystem() {
954928
return isSystem;
955929
}
956930

957-
public boolean isDataStreamIndex() {
958-
return isDataStreamIndex;
959-
}
960-
961931
public static Builder builder(String index) {
962932
return new Builder(index);
963933
}
@@ -984,7 +954,6 @@ public static class Builder {
984954
private Integer routingNumShards;
985955
private boolean isSystem;
986956
private IndexLongFieldRange timestampRange = IndexLongFieldRange.NO_SHARDS;
987-
private boolean isDataStreamIndex;
988957

989958
public Builder(String index) {
990959
this.index = index;
@@ -994,7 +963,6 @@ public Builder(String index) {
994963
this.inSyncAllocationIds = ImmutableOpenIntMap.builder();
995964
this.rolloverInfos = ImmutableOpenMap.builder();
996965
this.isSystem = false;
997-
this.isDataStreamIndex = false;
998966
}
999967

1000968
public Builder(IndexMetadata indexMetadata) {
@@ -1014,7 +982,6 @@ public Builder(IndexMetadata indexMetadata) {
1014982
this.rolloverInfos = ImmutableOpenMap.builder(indexMetadata.rolloverInfos);
1015983
this.isSystem = indexMetadata.isSystem;
1016984
this.timestampRange = indexMetadata.timestampRange;
1017-
this.isDataStreamIndex = indexMetadata.isDataStreamIndex;
1018985
}
1019986

1020987
public Builder index(String index) {
@@ -1236,16 +1203,6 @@ public IndexLongFieldRange getTimestampRange() {
12361203
return timestampRange;
12371204
}
12381205

1239-
// Sets if this index is a part of a datastream
1240-
public Builder dataStreamIndex(boolean isDataStreamIndex) {
1241-
this.isDataStreamIndex = isDataStreamIndex;
1242-
return this;
1243-
}
1244-
1245-
public boolean isDataStreamIndex() {
1246-
return isDataStreamIndex;
1247-
}
1248-
12491206
public IndexMetadata build() {
12501207
ImmutableOpenMap.Builder<String, AliasMetadata> tmpAliases = aliases;
12511208
Settings tmpSettings = settings;
@@ -1350,8 +1307,7 @@ public IndexMetadata build() {
13501307
waitForActiveShards,
13511308
rolloverInfos.build(),
13521309
isSystem,
1353-
timestampRange,
1354-
isDataStreamIndex);
1310+
timestampRange);
13551311
}
13561312

13571313
public static void toXContent(IndexMetadata indexMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException {
@@ -1452,10 +1408,11 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build
14521408
}
14531409
builder.endObject();
14541410
builder.field(KEY_SYSTEM, indexMetadata.isSystem);
1411+
14551412
builder.startObject(KEY_TIMESTAMP_RANGE);
14561413
indexMetadata.timestampRange.toXContent(builder, params);
14571414
builder.endObject();
1458-
builder.field(KEY_DATASTREAM_INDEX, indexMetadata.isDataStreamIndex);
1415+
14591416
builder.endObject();
14601417
}
14611418

@@ -1579,8 +1536,6 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti
15791536
builder.setRoutingNumShards(parser.intValue());
15801537
} else if (KEY_SYSTEM.equals(currentFieldName)) {
15811538
builder.system(parser.booleanValue());
1582-
} else if (KEY_DATASTREAM_INDEX.equals(currentFieldName)) {
1583-
builder.dataStreamIndex(parser.booleanValue());
15841539
} else {
15851540
throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
15861541
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ private ClusterState applyCreateIndexWithTemporaryService(final ClusterState cur
422422
try {
423423
indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper,
424424
temporaryIndexMeta.getSettings(), temporaryIndexMeta.getRoutingNumShards(), sourceMetadata,
425-
temporaryIndexMeta.isSystem(), temporaryIndexMeta.isDataStreamIndex());
425+
temporaryIndexMeta.isSystem());
426426
} catch (Exception e) {
427427
logger.info("failed to build index metadata [{}]", request.index());
428428
throw e;
@@ -457,7 +457,6 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata(final Settings aggr
457457
tmpImdBuilder.setRoutingNumShards(routingNumShards);
458458
tmpImdBuilder.settings(indexSettings);
459459
tmpImdBuilder.system(isSystem);
460-
tmpImdBuilder.dataStreamIndex(request.dataStreamName() != null);
461460

462461
// Set up everything, now locally create the index to see that things are ok, and apply
463462
IndexMetadata tempMetadata = tmpImdBuilder.build();
@@ -915,10 +914,9 @@ static ClusterState clusterStateCreateIndex(ClusterState currentState, Set<Clust
915914

916915
static IndexMetadata buildIndexMetadata(String indexName, List<AliasMetadata> aliases,
917916
Supplier<DocumentMapper> documentMapperSupplier, Settings indexSettings, int routingNumShards,
918-
@Nullable IndexMetadata sourceMetadata, boolean isSystem, boolean isDataStreamIndex) {
917+
@Nullable IndexMetadata sourceMetadata, boolean isSystem) {
919918
IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
920919
indexMetadataBuilder.system(isSystem);
921-
indexMetadataBuilder.dataStreamIndex(isDataStreamIndex);
922920
// now, update the mappings with the actual source
923921
Map<String, MappingMetadata> mappingsMetadata = new HashMap<>();
924922
DocumentMapper mapper = documentMapperSupplier.get();

‎server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,8 @@ public ClusterState execute(ClusterState currentState) throws Exception {
8282
final List<IndexMetadata> updatedMetadata = new ArrayList<>();
8383
for (ObjectObjectCursor<String, IndexMetadata> cursor : indexMetadataMap) {
8484
if (cursor.value != lastIndexMetadataMap.get(cursor.key)) {
85-
final boolean isSystemDataStream = systemIndices.isSystemIndexBackingDataStream(cursor.value.getIndex().getName());
86-
final boolean isSystem = systemIndices.isSystemIndex(cursor.value.getIndex()) || isSystemDataStream;
85+
final boolean isSystem = systemIndices.isSystemIndex(cursor.value.getIndex()) ||
86+
systemIndices.isSystemIndexBackingDataStream(cursor.value.getIndex().getName());
8787
IndexMetadata.Builder builder = IndexMetadata.builder(cursor.value);
8888
boolean updated = false;
8989
if (isSystem != cursor.value.isSystem()) {
@@ -96,10 +96,6 @@ public ClusterState execute(ClusterState currentState) throws Exception {
9696
.put(IndexMetadata.SETTING_INDEX_HIDDEN, false));
9797
updated = true;
9898
}
99-
if (isSystemDataStream != cursor.value.isDataStreamIndex()) {
100-
builder.dataStreamIndex(isSystemDataStream);
101-
updated = true;
102-
}
10399
if (updated) {
104100
updatedMetadata.add(builder.build());
105101
}

‎server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99

1010
import org.apache.lucene.analysis.Analyzer;
1111
import org.apache.lucene.codecs.Codec;
12-
import org.apache.lucene.index.LeafReader;
1312
import org.apache.lucene.index.MergePolicy;
1413
import org.apache.lucene.search.QueryCache;
1514
import org.apache.lucene.search.QueryCachingPolicy;
@@ -33,7 +32,6 @@
3332
import org.elasticsearch.plugins.IndexStorePlugin;
3433
import org.elasticsearch.threadpool.ThreadPool;
3534

36-
import java.util.Comparator;
3735
import java.util.List;
3836
import java.util.Objects;
3937
import java.util.function.LongSupplier;
@@ -72,7 +70,6 @@ public final class EngineConfig {
7270
private final CircuitBreakerService circuitBreakerService;
7371
private final LongSupplier globalCheckpointSupplier;
7472
private final Supplier<RetentionLeases> retentionLeasesSupplier;
75-
private final Comparator<LeafReader> leafSorter;
7673

7774
/**
7875
* A supplier of the outstanding retention leases. This is used during merged operations to determine which operations that have been
@@ -134,8 +131,7 @@ public EngineConfig(
134131
LongSupplier globalCheckpointSupplier,
135132
Supplier<RetentionLeases> retentionLeasesSupplier,
136133
LongSupplier primaryTermSupplier,
137-
IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier,
138-
Comparator<LeafReader> leafSorter) {
134+
IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier) {
139135
this.shardId = shardId;
140136
this.indexSettings = indexSettings;
141137
this.threadPool = threadPool;
@@ -173,7 +169,6 @@ public EngineConfig(
173169
this.retentionLeasesSupplier = Objects.requireNonNull(retentionLeasesSupplier);
174170
this.primaryTermSupplier = primaryTermSupplier;
175171
this.snapshotCommitSupplier = snapshotCommitSupplier;
176-
this.leafSorter = leafSorter;
177172
}
178173

179174
/**
@@ -358,12 +353,4 @@ public LongSupplier getPrimaryTermSupplier() {
358353
public IndexStorePlugin.SnapshotCommitSupplier getSnapshotCommitSupplier() {
359354
return snapshotCommitSupplier;
360355
}
361-
362-
/**
363-
* Returns how segments should be sorted for reading or @null if no sorting should be applied.
364-
*/
365-
@Nullable
366-
public Comparator<LeafReader> getLeafSorter() {
367-
return leafSorter;
368-
}
369356
}

‎server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2182,11 +2182,6 @@ private IndexWriterConfig getIndexWriterConfig() {
21822182
if (config().getIndexSort() != null) {
21832183
iwc.setIndexSort(config().getIndexSort());
21842184
}
2185-
// Provide a custom leaf sorter, so that index readers opened from this writer
2186-
// will have its leaves sorted according the given leaf sorter.
2187-
if (engineConfig.getLeafSorter() != null) {
2188-
iwc.setLeafSorter(engineConfig.getLeafSorter());
2189-
}
21902185
return iwc;
21912186
}
21922187

0 commit comments

Comments
 (0)