Skip to content

Commit 12e2f55

Browse files
original-brownbearoriginal-brownbear
authored
Optimize SLM Policy Queries (#79341)
Same as #79321 but for SLM policies. Enhances RepositoryData accordingly to enable the optimization.
1 parent 3d35144 commit 12e2f55

File tree

8 files changed

+180
-74
lines changed

8 files changed

+180
-74
lines changed

‎plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ public void testEnforcedCooldownPeriod() throws IOException {
156156
SnapshotState.SUCCESS,
157157
SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion(),
158158
0L, // -1 would refresh RepositoryData and find the real version
159-
0L // -1 would refresh RepositoryData and find the real version
159+
0L, // -1 would refresh RepositoryData and find the real version,
160+
"" // null would refresh RepositoryData and find the real version
160161
)));
161162
final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(),
162163
SnapshotsService.OLD_SNAPSHOT_FORMAT));

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
260260
.collect(
261261
Collectors.toMap(
262262
SnapshotId::getUUID,
263-
s -> new RepositoryData.SnapshotDetails(repositoryData.getSnapshotState(s), null, -1, -1)
263+
s -> new RepositoryData.SnapshotDetails(repositoryData.getSnapshotState(s), null, -1, -1, null)
264264
)
265265
),
266266
Collections.emptyMap(),

‎server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java‎

Lines changed: 72 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -666,7 +666,7 @@ private static Predicate<SnapshotInfo> buildAfterPredicate(
666666
}
667667
}
668668

669-
private static Predicate<SnapshotInfo> filterBySLMPolicies(String[] slmPolicies) {
669+
private static SnapshotPredicate filterBySLMPolicies(String[] slmPolicies) {
670670
final List<String> includePatterns = new ArrayList<>();
671671
final List<String> excludePatterns = new ArrayList<>();
672672
boolean seenWildcard = false;
@@ -686,25 +686,47 @@ private static Predicate<SnapshotInfo> filterBySLMPolicies(String[] slmPolicies)
686686
final String[] includes = includePatterns.toArray(Strings.EMPTY_ARRAY);
687687
final String[] excludes = excludePatterns.toArray(Strings.EMPTY_ARRAY);
688688
final boolean matchWithoutPolicy = matchNoPolicy;
689-
return snapshotInfo -> {
690-
final Map<String, Object> metadata = snapshotInfo.userMetadata();
691-
final String policy;
692-
if (metadata == null) {
693-
policy = null;
694-
} else {
695-
final Object policyFound = metadata.get(SnapshotsService.POLICY_ID_METADATA_FIELD);
696-
policy = policyFound instanceof String ? (String) policyFound : null;
697-
}
698-
if (policy == null) {
699-
return matchWithoutPolicy;
689+
return new SnapshotPredicate() {
690+
@Override
691+
public boolean matchesPreflight(SnapshotId snapshotId, RepositoryData repositoryData) {
692+
final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId);
693+
final String policy;
694+
if (details == null || (details.getSlmPolicy() == null)) {
695+
// no SLM policy recorded
696+
return true;
697+
} else {
698+
final String policyFound = details.getSlmPolicy();
699+
// empty string means that snapshot was not created by an SLM policy
700+
policy = policyFound.isEmpty() ? null : policyFound;
701+
}
702+
return matchPolicy(includes, excludes, matchWithoutPolicy, policy);
700703
}
701-
if (Regex.simpleMatch(includes, policy) == false) {
702-
return false;
704+
705+
@Override
706+
public boolean matches(SnapshotInfo snapshotInfo) {
707+
final Map<String, Object> metadata = snapshotInfo.userMetadata();
708+
final String policy;
709+
if (metadata == null) {
710+
policy = null;
711+
} else {
712+
final Object policyFound = metadata.get(SnapshotsService.POLICY_ID_METADATA_FIELD);
713+
policy = policyFound instanceof String ? (String) policyFound : null;
714+
}
715+
return matchPolicy(includes, excludes, matchWithoutPolicy, policy);
703716
}
704-
return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false;
705717
};
706718
}
707719

720+
private static boolean matchPolicy(String[] includes, String[] excludes, boolean matchWithoutPolicy, @Nullable String policy) {
721+
if (policy == null) {
722+
return matchWithoutPolicy;
723+
}
724+
if (Regex.simpleMatch(includes, policy) == false) {
725+
return false;
726+
}
727+
return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false;
728+
}
729+
708730
private static Predicate<SnapshotInfo> filterByLongOffset(ToLongFunction<SnapshotInfo> extractor, long after, SortOrder order) {
709731
return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info);
710732
}
@@ -759,19 +781,23 @@ private static final class SnapshotPredicates {
759781
Predicate<SnapshotInfo> snapshotPredicate = null;
760782
final String[] slmPolicies = request.policies();
761783
final String fromSortValue = request.fromSortValue();
784+
BiPredicate<SnapshotId, RepositoryData> preflightPredicate = null;
762785
if (slmPolicies.length > 0) {
763-
snapshotPredicate = filterBySLMPolicies(slmPolicies);
786+
final SnapshotPredicate predicate = filterBySLMPolicies(slmPolicies);
787+
snapshotPredicate = predicate::matches;
788+
preflightPredicate = predicate::matchesPreflight;
764789
}
765790
final GetSnapshotsRequest.SortBy sortBy = request.sort();
766791
final SortOrder order = request.order();
767792
if (fromSortValue == null) {
768-
preflightPredicate = null;
793+
this.preflightPredicate = preflightPredicate;
769794
} else {
770795
final Predicate<SnapshotInfo> fromSortValuePredicate;
796+
final BiPredicate<SnapshotId, RepositoryData> preflightPred;
771797
switch (sortBy) {
772798
case START_TIME:
773799
final long after = Long.parseLong(fromSortValue);
774-
preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
800+
preflightPred = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
775801
final long startTime = getStartTime(snapshotId, repositoryData);
776802
return startTime == -1 || after <= startTime;
777803
} : (snapshotId, repositoryData) -> {
@@ -781,14 +807,14 @@ private static final class SnapshotPredicates {
781807
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::startTime, after, order);
782808
break;
783809
case NAME:
784-
preflightPredicate = order == SortOrder.ASC
810+
preflightPred = order == SortOrder.ASC
785811
? (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) <= 0
786812
: (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) >= 0;
787813
fromSortValuePredicate = null;
788814
break;
789815
case DURATION:
790816
final long afterDuration = Long.parseLong(fromSortValue);
791-
preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
817+
preflightPred = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
792818
final long duration = getDuration(snapshotId, repositoryData);
793819
return duration == -1 || afterDuration <= duration;
794820
} : (snapshotId, repositoryData) -> {
@@ -799,22 +825,22 @@ private static final class SnapshotPredicates {
799825
break;
800826
case INDICES:
801827
final int afterIndexCount = Integer.parseInt(fromSortValue);
802-
preflightPredicate = order == SortOrder.ASC
828+
preflightPred = order == SortOrder.ASC
803829
? (snapshotId, repositoryData) -> afterIndexCount <= indexCount(snapshotId, repositoryData)
804830
: (snapshotId, repositoryData) -> afterIndexCount >= indexCount(snapshotId, repositoryData);
805831
fromSortValuePredicate = null;
806832
break;
807833
case REPOSITORY:
808834
// already handled in #maybeFilterRepositories
809-
preflightPredicate = null;
835+
preflightPred = null;
810836
fromSortValuePredicate = null;
811837
break;
812838
case SHARDS:
813-
preflightPredicate = null;
839+
preflightPred = null;
814840
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(fromSortValue), order);
815841
break;
816842
case FAILED_SHARDS:
817-
preflightPredicate = null;
843+
preflightPred = null;
818844
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(fromSortValue), order);
819845
break;
820846
default:
@@ -826,6 +852,15 @@ private static final class SnapshotPredicates {
826852
} else if (fromSortValuePredicate != null) {
827853
snapshotPredicate = fromSortValuePredicate.and(snapshotPredicate);
828854
}
855+
if (preflightPredicate == null) {
856+
this.preflightPredicate = preflightPred;
857+
} else {
858+
if (preflightPred != null) {
859+
this.preflightPredicate = preflightPredicate.and(preflightPred);
860+
} else {
861+
this.preflightPredicate = preflightPredicate;
862+
}
863+
}
829864
}
830865
this.snapshotPredicate = snapshotPredicate;
831866
}
@@ -842,6 +877,19 @@ public BiPredicate<SnapshotId, RepositoryData> preflightPredicate() {
842877

843878
}
844879

880+
private interface SnapshotPredicate {
881+
882+
/**
883+
* Checks if a snapshot matches the predicate by testing its {@link SnapshotId} for a given {@link RepositoryData}.
884+
*/
885+
boolean matchesPreflight(SnapshotId snapshotId, RepositoryData repositoryData);
886+
887+
/**
888+
* Checks if a snapshot matches the predicate by testing its {@link SnapshotInfo}.
889+
*/
890+
boolean matches(SnapshotInfo snapshotInfo);
891+
}
892+
845893
private static final class SnapshotsInRepo {
846894

847895
private final List<SnapshotInfo> snapshotInfos;

‎server/src/main/java/org/elasticsearch/repositories/RepositoryData.java‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,8 @@ public boolean hasMissingDetails(SnapshotId snapshotId) {
283283
return snapshotDetails == null
284284
|| snapshotDetails.getVersion() == null
285285
|| snapshotDetails.getStartTimeMillis() == -1
286-
|| snapshotDetails.getEndTimeMillis() == -1;
286+
|| snapshotDetails.getEndTimeMillis() == -1
287+
|| snapshotDetails.getSlmPolicy() == null;
287288
}
288289

289290
/**
@@ -639,6 +640,7 @@ public Map<String, IndexId> resolveNewIndices(List<String> indicesToResolve, Map
639640
private static final String MIN_VERSION = "min_version";
640641
private static final String START_TIME_MILLIS = "start_time_millis";
641642
private static final String END_TIME_MILLIS = "end_time_millis";
643+
private static final String SLM_POLICY = "slm_policy";
642644

643645
/**
644646
* Writes the snapshots metadata and the related indices metadata to x-content.
@@ -725,6 +727,9 @@ public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final
725727
if (snapshotDetails.getEndTimeMillis() != -1) {
726728
builder.field(END_TIME_MILLIS, snapshotDetails.getEndTimeMillis());
727729
}
730+
if (snapshotDetails.getSlmPolicy() != null) {
731+
builder.field(SLM_POLICY, snapshotDetails.getSlmPolicy());
732+
}
728733

729734
builder.endObject();
730735
}
@@ -891,6 +896,7 @@ private static void parseSnapshots(
891896
Version version = null;
892897
long startTimeMillis = -1;
893898
long endTimeMillis = -1;
899+
String slmPolicy = null;
894900
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
895901
String currentFieldName = parser.currentName();
896902
parser.nextToken();
@@ -918,12 +924,14 @@ private static void parseSnapshots(
918924
assert endTimeMillis == -1;
919925
endTimeMillis = parser.longValue();
920926
break;
927+
case SLM_POLICY:
928+
slmPolicy = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
921929
}
922930
}
923931
assert (startTimeMillis == -1) == (endTimeMillis == -1) : "unexpected: " + startTimeMillis + ", " + endTimeMillis + ", ";
924932
final SnapshotId snapshotId = new SnapshotId(name, uuid);
925933
if (state != null || version != null) {
926-
snapshotsDetails.put(uuid, new SnapshotDetails(state, version, startTimeMillis, endTimeMillis));
934+
snapshotsDetails.put(uuid, new SnapshotDetails(state, version, startTimeMillis, endTimeMillis, slmPolicy));
927935
}
928936
snapshots.put(uuid, snapshotId);
929937
if (metaGenerations != null && metaGenerations.isEmpty() == false) {
@@ -1037,7 +1045,7 @@ private static String parseLegacySnapshotUUID(XContentParser parser) throws IOEx
10371045
*/
10381046
public static class SnapshotDetails {
10391047

1040-
public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1);
1048+
public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1, null);
10411049

10421050
@Nullable // TODO forbid nulls here, this only applies to very old repositories
10431051
private final SnapshotState snapshotState;
@@ -1051,11 +1059,23 @@ public static class SnapshotDetails {
10511059
// May be -1 if unknown, which happens if the snapshot was taken before 7.14 and hasn't been updated yet
10521060
private final long endTimeMillis;
10531061

1054-
public SnapshotDetails(@Nullable SnapshotState snapshotState, @Nullable Version version, long startTimeMillis, long endTimeMillis) {
1062+
// May be null if unknown, which happens if the snapshot was taken before 7.16 and hasn't been updated yet. Empty string indicates
1063+
// that this snapshot was not created by an SLM policy.
1064+
@Nullable
1065+
private final String slmPolicy;
1066+
1067+
public SnapshotDetails(
1068+
@Nullable SnapshotState snapshotState,
1069+
@Nullable Version version,
1070+
long startTimeMillis,
1071+
long endTimeMillis,
1072+
@Nullable String slmPolicy
1073+
) {
10551074
this.snapshotState = snapshotState;
10561075
this.version = version;
10571076
this.startTimeMillis = startTimeMillis;
10581077
this.endTimeMillis = endTimeMillis;
1078+
this.slmPolicy = slmPolicy;
10591079
}
10601080

10611081
@Nullable
@@ -1082,6 +1102,15 @@ public long getEndTimeMillis() {
10821102
return endTimeMillis;
10831103
}
10841104

1105+
/**
1106+
* @return the SLM policy that the snapshot was created by or an empty string if it was not created by an SLM policy or
1107+
* {@code null} if unknown.
1108+
*/
1109+
@Nullable
1110+
public String getSlmPolicy() {
1111+
return slmPolicy;
1112+
}
1113+
10851114
@Override
10861115
public boolean equals(Object o) {
10871116
if (this == o) return true;
@@ -1090,12 +1119,13 @@ public boolean equals(Object o) {
10901119
return startTimeMillis == that.startTimeMillis
10911120
&& endTimeMillis == that.endTimeMillis
10921121
&& snapshotState == that.snapshotState
1093-
&& Objects.equals(version, that.version);
1122+
&& Objects.equals(version, that.version)
1123+
&& Objects.equals(slmPolicy, that.slmPolicy);
10941124
}
10951125

10961126
@Override
10971127
public int hashCode() {
1098-
return Objects.hash(snapshotState, version, startTimeMillis, endTimeMillis);
1128+
return Objects.hash(snapshotState, version, startTimeMillis, endTimeMillis, slmPolicy);
10991129
}
11001130

11011131
}

0 commit comments

Comments
 (0)