From 8aee1c6d9d36636e457b89b5447e0e523addefbb Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 28 Jan 2022 15:20:23 -0500 Subject: [PATCH 01/25] Update YAML Rest tests to check for product header on all responses. --- .../rest/yaml/ClientYamlTestResponse.java | 15 ++++++++---- .../test/rest/yaml/section/DoSection.java | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java index bdd8ba9dab1df..86121fa0d7da0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java @@ -87,13 +87,20 @@ public String getReasonPhrase() { * Get a list of all of the values of all warning headers returned in the response. */ public List getWarningHeaders() { - List warningHeaders = new ArrayList<>(); + return getHeaders("Warning"); + } + + /** + * Get a list of all the values of a given header returned in the response. + */ + public List getHeaders(String name) { + List headers = new ArrayList<>(); for (Header header : response.getHeaders()) { - if (header.getName().equals("Warning")) { - warningHeaders.add(header.getValue()); + if (header.getName().equalsIgnoreCase(name)) { + headers.add(header.getValue()); } } - return warningHeaders; + return headers; } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 23a7146561da9..5f3091dca73fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -367,6 +367,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx final String testPath = executionContext.getClientYamlTestCandidate() != null ? executionContext.getClientYamlTestCandidate().getTestPath() : null; + checkElasticProductHeader(response.getHeaders("X-elastic-product")); checkWarningHeaders(response.getWarningHeaders(), testPath); } catch (ClientYamlTestResponseException e) { ClientYamlTestResponse restTestResponse = e.getRestTestResponse(); @@ -392,6 +393,28 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx } } + void checkElasticProductHeader(final List productHeaders) { + if (productHeaders.isEmpty()) { + fail("Response is missing required X-Elastic-Product response header"); + } + boolean headerPresent = false; + final List unexpected = new ArrayList<>(); + for (String header : productHeaders) { + if (header.equals("Elasticsearch")) { + headerPresent = true; + break; + } else { + unexpected.add(header); + } + } + if (headerPresent == false) { + StringBuilder failureMessage = new StringBuilder(); + appendBadHeaders(failureMessage, unexpected, "did not get expected product header [Elasticsearch], found header" + + (unexpected.size() > 1 ? "s" : "")); + fail(failureMessage.toString()); + } + } + void checkWarningHeaders(final List warningHeaders) { checkWarningHeaders(warningHeaders, null); } From f05df00d1aa98c7ec0b6826ccf12581320c40cfa Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 28 Jan 2022 16:35:47 -0500 Subject: [PATCH 02/25] Preserve thread context when waiting for snapshot completion --- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 77885fb624e4c..b22967eb7b626 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -2957,7 +2958,8 @@ static Map filterDataStreamAliases( * @param listener listener */ private void addListener(Snapshot snapshot, ActionListener> listener) { - snapshotCompletionListeners.computeIfAbsent(snapshot, k -> new CopyOnWriteArrayList<>()).add(listener); + snapshotCompletionListeners.computeIfAbsent(snapshot, k -> new CopyOnWriteArrayList<>()) + .add(ContextPreservingActionListener.wrapPreservingContext(listener, threadPool.getThreadContext())); } @Override From 8cab8c00725e473dd3fe1d9a01126c287c061888 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 28 Jan 2022 16:36:29 -0500 Subject: [PATCH 03/25] Update docs/changelog/83290.yaml --- docs/changelog/83290.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/83290.yaml diff --git a/docs/changelog/83290.yaml b/docs/changelog/83290.yaml new file mode 100644 index 0000000000000..9b3bb8ef056e5 --- /dev/null +++ b/docs/changelog/83290.yaml @@ -0,0 +1,5 @@ +pr: 83290 +summary: Update YAML Rest tests to check for product header on all responses +area: Infra/REST API +type: enhancement +issues: [] From fc4a8750698ece31fc8b845e7601274bf812579a Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 31 Jan 2022 13:56:59 -0500 Subject: [PATCH 04/25] Preserve context on restore snapshot calls --- .../snapshots/restore/TransportRestoreSnapshotAction.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index 7b247f1b14a42..6d097cd720235 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -72,7 +73,11 @@ protected void masterOperation( ) { restoreService.restoreSnapshot(request, listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> { if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) { - RestoreClusterStateListener.createAndRegisterListener(clusterService, restoreCompletionResponse, delegatedListener); + RestoreClusterStateListener.createAndRegisterListener( + clusterService, + restoreCompletionResponse, + new ContextPreservingActionListener<>(threadPool.getThreadContext().newRestorableContext(true), delegatedListener) + ); } else { delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo())); } From c96b8f4e0d8616c46115b251263ad8696a4fd5dd Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 31 Jan 2022 15:10:05 -0500 Subject: [PATCH 05/25] Preserve thread context when completing remote reindex operations --- .../reindex/AbstractAsyncBulkByScrollAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java index bab93e56b653f..beac9ab88c78c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java @@ -593,7 +593,7 @@ protected void finishHim(Exception failure) { */ protected void finishHim(Exception failure, List indexingFailures, List searchFailures, boolean timedOut) { logger.debug("[{}]: finishing without any catastrophic failures", task.getId()); - scrollSource.close(() -> { + scrollSource.close(threadPool.getThreadContext().preserveContext(() -> { if (failure == null) { BulkByScrollResponse response = buildResponse( timeValueNanos(System.nanoTime() - startTime.get()), @@ -605,7 +605,7 @@ protected void finishHim(Exception failure, List indexingFailures, List } else { listener.onFailure(failure); } - }); + })); } /** From 3ae69479182ab7188a8dc1e0dcf33acc3f4ab248 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 31 Jan 2022 15:59:47 -0500 Subject: [PATCH 06/25] Preserve thread context when putting followers --- .../xpack/ccr/action/TransportPutFollowAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index 1661585b5062f..bfa46f54843ed 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -250,7 +251,7 @@ public void onFailure(Exception e) { } }; } else { - listener = originalListener; + listener = new ContextPreservingActionListener<>(threadPool.getThreadContext().newRestorableContext(true), originalListener); } RestoreClusterStateListener.createAndRegisterListener( From 6b2b152952a6f7c825ecd4562cd332a3c06f660a Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 31 Jan 2022 17:10:37 -0500 Subject: [PATCH 07/25] Preserve thread context when iterating over xpack usage results --- .../xpack/core/action/TransportXPackUsageAction.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java index 959ffc448f548..6248a011932db 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterState; @@ -83,8 +84,12 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat l.onResponse(Collections.emptyList()); })); }; + final ActionListener> finalListener = new ContextPreservingActionListener<>( + threadPool.getThreadContext().newRestorableContext(true), + usageActionListener + ); IteratingActionListener, XPackUsageFeatureAction> iteratingActionListener = - new IteratingActionListener<>(usageActionListener, consumer, usageActions, threadPool.getThreadContext(), (ignore) -> { + new IteratingActionListener<>(finalListener, consumer, usageActions, threadPool.getThreadContext(), (ignore) -> { final List usageList = new ArrayList<>(featureSetUsages.length()); for (int i = 0; i < featureSetUsages.length(); i++) { usageList.add(featureSetUsages.get(i)); From 5a33c69167d8e1dd93bf107c89158a4cd10d46f9 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 31 Jan 2022 17:33:41 -0500 Subject: [PATCH 08/25] Fix code formatting --- .../elasticsearch/test/rest/yaml/section/DoSection.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 5f3091dca73fb..efc53b08fad27 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -409,8 +409,11 @@ void checkElasticProductHeader(final List productHeaders) { } if (headerPresent == false) { StringBuilder failureMessage = new StringBuilder(); - appendBadHeaders(failureMessage, unexpected, "did not get expected product header [Elasticsearch], found header" + - (unexpected.size() > 1 ? "s" : "")); + appendBadHeaders( + failureMessage, + unexpected, + "did not get expected product header [Elasticsearch], found header" + (unexpected.size() > 1 ? "s" : "") + ); fail(failureMessage.toString()); } } From 3cac42c8908c3010860a2bec210d212a47f63cd9 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 31 Jan 2022 17:51:40 -0500 Subject: [PATCH 09/25] Restore thread context on unfollow action listener completion --- .../xpack/ccr/action/TransportUnfollowAction.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java index 1f775b97ee4d0..e76154ee5f470 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; @@ -178,10 +179,16 @@ private void removeRetentionLeaseForShard( ) { logger.trace("{} removing retention lease [{}] while unfollowing leader index", followerShardId, retentionLeaseId); final ThreadContext threadContext = threadPool.getThreadContext(); + // We're about to stash the thread context for this retention lease removal. The listener will be completed while the + // context is stashed. The context needs to be restored in the listener when it is completing or else it is simply wiped. + final ActionListener preservedListener = new ContextPreservingActionListener<>( + threadContext.newRestorableContext(true), + listener + ); try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { // we have to execute under the system context so that if security is enabled the removal is authorized threadContext.markAsSystemContext(); - CcrRetentionLeases.asyncRemoveRetentionLease(leaderShardId, retentionLeaseId, remoteClient, listener); + CcrRetentionLeases.asyncRemoveRetentionLease(leaderShardId, retentionLeaseId, remoteClient, preservedListener); } } From 45271b2f66e05490abc683842314b899430734eb Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 1 Feb 2022 11:49:13 -0500 Subject: [PATCH 10/25] Fix DoSection test --- .../org/elasticsearch/test/rest/yaml/section/DoSectionTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index fdd3451012d5c..b7238588ffe36 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -605,6 +605,7 @@ public void testNodeSelectorByVersion() throws IOException { doSection.getApiCallSection().getNodeSelector() ) ).thenReturn(mockResponse); + when(mockResponse.getHeaders("X-elastic-product")).thenReturn(List.of("Elasticsearch")); doSection.execute(context); verify(context).callApi( "indices.get_field_mapping", From 2680ddfdea26a2bc9f1733ee691263d622b53fac Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 2 Feb 2022 11:38:12 -0500 Subject: [PATCH 11/25] disable bwc tests until fixes are backported --- .../rest-api-spec/test/snapshot.restore/10_basic.yml | 2 ++ .../resources/rest-api-spec/test/snapshot.status/10_basic.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml index 1ea5b542625e8..b0b9520dddbef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml @@ -70,6 +70,8 @@ setup: - skip: features: ["allowed_warnings"] + version: "7.0.0 - 7.17.1" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml index c35f2419bdc91..01dc7b6ec52e5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -11,6 +11,10 @@ setup: --- "Get snapshot status": + - skip: + version: "7.0.0 - 7.17.1" + reason: "Pause BWC tests until #83290 is backported" + - do: indices.create: index: test_index From 0a6b8642c696a399cbca5a89dd02f603db4232b7 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 2 Feb 2022 16:27:02 -0500 Subject: [PATCH 12/25] Disable some more BWC tests --- .../test/cat.snapshots/10_basic.yml | 4 ++++ .../test/snapshot.clone/10_basic.yml | 3 +++ .../test/snapshot.create/10_basic.yml | 4 ++-- .../rest-api-spec/test/snapshot.get/10_basic.yml | 16 +++++++++++----- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml index f7d60671c7e88..bf2ecc51225be 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml @@ -23,6 +23,10 @@ $/ --- "Test cat snapshots output": + - skip: + version: "7.0.0 - 7.17.1" + reason: "Pause BWC tests until #83290 is backported" + - do: snapshot.create_repository: repository: test_cat_snapshots_1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml index fb289355e08fb..1a2bd39f388b4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml @@ -1,5 +1,8 @@ --- setup: + - skip: + version: "- 7.17.1" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml index f7c522b712244..342211f79ed7b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml @@ -41,8 +41,8 @@ setup: --- "Create a snapshot and clean up repository": - skip: - version: " - 7.3.99" - reason: cleanup introduced in 7.4 + version: " - 7.17.1" + reason: "cleanup introduced in 7.4, Pause BWC tests until #83290 is backported" - do: snapshot.cleanup_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml index b50ece87e9f88..532027a4d441d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -11,6 +11,9 @@ setup: --- "Get snapshot info": + - skip: + version: "7.0.0 - 7.17.1" + reason: "Pause BWC tests until #83290 is backported" - do: indices.create: @@ -61,6 +64,10 @@ setup: --- "Get snapshot info when verbose is false": + - skip: + version: "7.0.0 - 7.17.1" + reason: "Pause BWC tests until #83290 is backported" + - do: indices.create: index: test_index @@ -97,8 +104,8 @@ setup: --- "Get snapshot info contains include_global_state": - skip: - version: " - 6.1.99" - reason: "include_global_state field has been added in the response in 6.2.0" + version: " - 7.17.1" + reason: "include_global_state field has been added in the response in 6.2.0, Pause BWC tests until #83290 is backported" - do: indices.create: @@ -196,9 +203,8 @@ setup: --- "Get snapshot info with index details": - skip: - version: " - 7.12.99" - reason: "Introduced in 7.13.0" - + version: " - 7.17.1" + reason: "Introduced in 7.13.0, Pause BWC tests until #83290 is backported" - do: indices.create: index: test_index From 65e88fb53e1b7b9b99b56291564f9e8974391e7b Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 4 Feb 2022 14:49:35 -0500 Subject: [PATCH 13/25] Rework the transport xpack usage action listener logic to not need context preservation --- .../action/TransportXPackUsageAction.java | 54 +++++++------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java index 6248a011932db..be3dc718f99ab 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java @@ -7,8 +7,8 @@ package org.elasticsearch.xpack.core.action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterState; @@ -21,15 +21,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; -import org.elasticsearch.xpack.core.XPackFeatureSet.Usage; -import org.elasticsearch.xpack.core.common.IteratingActionListener; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.function.BiConsumer; public class TransportXPackUsageAction extends TransportMasterNodeAction { @@ -67,36 +61,24 @@ protected List usageActions() { @Override protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, ActionListener listener) { - final ActionListener> usageActionListener = listener.delegateFailure( - (l, usages) -> l.onResponse(new XPackUsageResponse(usages)) - ); - final AtomicReferenceArray featureSetUsages = new AtomicReferenceArray<>(usageActions.size()); - final AtomicInteger position = new AtomicInteger(0); - final BiConsumer>> consumer = (featureUsageAction, iteratingListener) -> { - // Since we're executing the actions locally we should create a new request - // to avoid mutating the original request and setting the wrong parent task, - // since it is possible that the parent task gets cancelled and new child tasks are banned. - final XPackUsageRequest childRequest = new XPackUsageRequest(); - childRequest.setParentTask(request.getParentTask()); - client.executeLocally(featureUsageAction, childRequest, iteratingListener.delegateFailure((l, usageResponse) -> { - featureSetUsages.set(position.getAndIncrement(), usageResponse.getUsage()); - // the value sent back doesn't matter since our predicate keeps iterating - l.onResponse(Collections.emptyList()); - })); - }; - final ActionListener> finalListener = new ContextPreservingActionListener<>( - threadPool.getThreadContext().newRestorableContext(true), - usageActionListener - ); - IteratingActionListener, XPackUsageFeatureAction> iteratingActionListener = - new IteratingActionListener<>(finalListener, consumer, usageActions, threadPool.getThreadContext(), (ignore) -> { - final List usageList = new ArrayList<>(featureSetUsages.length()); - for (int i = 0; i < featureSetUsages.length(); i++) { - usageList.add(featureSetUsages.get(i)); + new ActionRunnable<>(listener) { + final List responses = new ArrayList<>(usageActions.size()); + + @Override + protected void doRun() { + if (responses.size() < usageActions().size()) { + final var childRequest = new XPackUsageRequest(); + childRequest.setParentTask(request.getParentTask()); + client.executeLocally(usageActions.get(responses.size()), childRequest, listener.delegateFailure((delegate, response) -> { + responses.add(response.getUsage()); + run(); // XPackUsageFeatureTransportAction always forks to MANAGEMENT so no risk of stack overflow here + })); + } else { + assert responses.size() == usageActions.size() : responses.size() + " vs " + usageActions.size(); + listener.onResponse(new XPackUsageResponse(responses)); } - return usageList; - }, (ignore) -> true); - iteratingActionListener.run(); + } + }.run(); } @Override From 4dd834ef0f44dfd988c6f5dd372b2468672dde6b Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 4 Feb 2022 15:36:11 -0500 Subject: [PATCH 14/25] Move the context preservation for cluster applier listeners down to the service level --- .../TransportRestoreSnapshotAction.java | 7 +------ .../service/ClusterApplierService.java | 19 ++++++++++++++++++- .../ccr/action/TransportPutFollowAction.java | 3 +-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index 6d097cd720235..7b247f1b14a42 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -73,11 +72,7 @@ protected void masterOperation( ) { restoreService.restoreSnapshot(request, listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> { if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) { - RestoreClusterStateListener.createAndRegisterListener( - clusterService, - restoreCompletionResponse, - new ContextPreservingActionListener<>(threadPool.getThreadContext().newRestorableContext(true), delegatedListener) - ); + RestoreClusterStateListener.createAndRegisterListener(clusterService, restoreCompletionResponse, delegatedListener); } else { delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo())); } diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index 66d5428d5d135..c948e0136cef8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -216,7 +216,24 @@ public void removeApplier(ClusterStateApplier applier) { * Add a listener for updated cluster states */ public void addListener(ClusterStateListener listener) { - clusterStateListeners.add(listener); + clusterStateListeners.add(new SafeClusterStateListener(listener, threadPool.getThreadContext().newRestorableContext(true))); + } + + private static class SafeClusterStateListener implements ClusterStateListener { + private final ClusterStateListener delegate; + private final Supplier supplier; + + public SafeClusterStateListener(ClusterStateListener delegate, Supplier supplier) { + this.delegate = delegate; + this.supplier = supplier; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + try (ThreadContext.StoredContext stored = supplier.get()) { + delegate.clusterChanged(event); + } + } } /** diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index bfa46f54843ed..1661585b5062f 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; -import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -251,7 +250,7 @@ public void onFailure(Exception e) { } }; } else { - listener = new ContextPreservingActionListener<>(threadPool.getThreadContext().newRestorableContext(true), originalListener); + listener = originalListener; } RestoreClusterStateListener.createAndRegisterListener( From e1634a90fd527c2441e1fe0bcd078fb007a65e74 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 4 Feb 2022 15:38:17 -0500 Subject: [PATCH 15/25] more bwc tests --- .../resources/rest-api-spec/test/snapshot.create/10_basic.yml | 3 +++ .../test/snapshot.get_repository/20_repository_uuid.yml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml index 342211f79ed7b..c00b4e4ce145e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml @@ -19,6 +19,9 @@ setup: --- "Create a snapshot": + - - skip: + version: "7.0.0 - 7.17.1" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml index 0532d208d0cba..60a851bc4803c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml @@ -1,8 +1,8 @@ --- "Get repository returns UUID": - skip: - version: " - 7.12.99" - reason: repository UUIDs introduced in 7.12.0 + version: " - 7.17.1" + reason: "repository UUIDs introduced in 7.12.0, Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: From 38c7253e81bb3895d9e3859423ea1cfbbe383972 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 4 Feb 2022 17:07:25 -0500 Subject: [PATCH 16/25] appease style checker --- .../elasticsearch/cluster/service/ClusterApplierService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index c948e0136cef8..3c77b5b2322bb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -223,7 +223,7 @@ private static class SafeClusterStateListener implements ClusterStateListener { private final ClusterStateListener delegate; private final Supplier supplier; - public SafeClusterStateListener(ClusterStateListener delegate, Supplier supplier) { + SafeClusterStateListener(ClusterStateListener delegate, Supplier supplier) { this.delegate = delegate; this.supplier = supplier; } From 8f275d06e9a6e766274fb61e57fb2246b06bc888 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 7 Feb 2022 14:09:57 -0500 Subject: [PATCH 17/25] Update listener name, fix removeListener method --- .../cluster/service/ClusterApplierService.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index 3c77b5b2322bb..b962381d7c10d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -216,14 +216,14 @@ public void removeApplier(ClusterStateApplier applier) { * Add a listener for updated cluster states */ public void addListener(ClusterStateListener listener) { - clusterStateListeners.add(new SafeClusterStateListener(listener, threadPool.getThreadContext().newRestorableContext(true))); + clusterStateListeners.add(new ContextPreservingClusterStateListener(listener, threadPool.getThreadContext().newRestorableContext(true))); } - private static class SafeClusterStateListener implements ClusterStateListener { + private static class ContextPreservingClusterStateListener implements ClusterStateListener { private final ClusterStateListener delegate; private final Supplier supplier; - SafeClusterStateListener(ClusterStateListener delegate, Supplier supplier) { + ContextPreservingClusterStateListener(ClusterStateListener delegate, Supplier supplier) { this.delegate = delegate; this.supplier = supplier; } @@ -239,8 +239,14 @@ public void clusterChanged(ClusterChangedEvent event) { /** * Removes a listener for updated cluster states. */ - public void removeListener(ClusterStateListener listener) { - clusterStateListeners.remove(listener); + public void removeListener(final ClusterStateListener listener) { + clusterStateListeners.removeIf(storedListener -> { + if (storedListener instanceof ContextPreservingClusterStateListener) { + return ((ContextPreservingClusterStateListener) storedListener).delegate.equals(listener); + } else { + return storedListener.equals(listener); + } + }); } /** From 67f75918fc4f99a2319293f1b721058cc6ae1ed4 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 7 Feb 2022 14:24:24 -0500 Subject: [PATCH 18/25] checkstyle --- .../xpack/core/action/TransportXPackUsageAction.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java index be3dc718f99ab..4f09c349723de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java @@ -69,10 +69,13 @@ protected void doRun() { if (responses.size() < usageActions().size()) { final var childRequest = new XPackUsageRequest(); childRequest.setParentTask(request.getParentTask()); - client.executeLocally(usageActions.get(responses.size()), childRequest, listener.delegateFailure((delegate, response) -> { - responses.add(response.getUsage()); - run(); // XPackUsageFeatureTransportAction always forks to MANAGEMENT so no risk of stack overflow here - })); + client.executeLocally( + usageActions.get(responses.size()), + childRequest, + listener.delegateFailure((delegate, response) -> { + responses.add(response.getUsage()); + run(); // XPackUsageFeatureTransportAction always forks to MANAGEMENT so no risk of stack overflow here + })); } else { assert responses.size() == usageActions.size() : responses.size() + " vs " + usageActions.size(); listener.onResponse(new XPackUsageResponse(responses)); From 74908bd4fd159ff9951dece22e76dd17b2cfc194 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 7 Feb 2022 16:42:50 -0500 Subject: [PATCH 19/25] assert unique listeners, fix precommit again --- .../service/ClusterApplierService.java | 49 +++++++++---------- .../action/TransportXPackUsageAction.java | 3 +- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index b962381d7c10d..ae90ceea1cde2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -79,7 +79,24 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements private final Collection normalPriorityStateAppliers = new CopyOnWriteArrayList<>(); private final Collection lowPriorityStateAppliers = new CopyOnWriteArrayList<>(); - private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); + private static class ContextPreservingClusterStateListener implements ClusterStateListener { + private final ClusterStateListener delegate; + private final Supplier supplier; + + ContextPreservingClusterStateListener(ClusterStateListener delegate, Supplier supplier) { + this.delegate = delegate; + this.supplier = supplier; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + try (ThreadContext.StoredContext stored = supplier.get()) { + delegate.clusterChanged(event); + } + } + } + + private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); private final Map timeoutClusterStateListeners = new ConcurrentHashMap<>(); private final AtomicReference state; // last applied state @@ -216,37 +233,17 @@ public void removeApplier(ClusterStateApplier applier) { * Add a listener for updated cluster states */ public void addListener(ClusterStateListener listener) { - clusterStateListeners.add(new ContextPreservingClusterStateListener(listener, threadPool.getThreadContext().newRestorableContext(true))); - } - - private static class ContextPreservingClusterStateListener implements ClusterStateListener { - private final ClusterStateListener delegate; - private final Supplier supplier; - - ContextPreservingClusterStateListener(ClusterStateListener delegate, Supplier supplier) { - this.delegate = delegate; - this.supplier = supplier; - } - - @Override - public void clusterChanged(ClusterChangedEvent event) { - try (ThreadContext.StoredContext stored = supplier.get()) { - delegate.clusterChanged(event); - } - } + assert clusterStateListeners.stream().anyMatch(stored -> stored.delegate.equals(listener)) == false : "listener must be unique"; + clusterStateListeners.add( + new ContextPreservingClusterStateListener(listener, threadPool.getThreadContext().newRestorableContext(true)) + ); } /** * Removes a listener for updated cluster states. */ public void removeListener(final ClusterStateListener listener) { - clusterStateListeners.removeIf(storedListener -> { - if (storedListener instanceof ContextPreservingClusterStateListener) { - return ((ContextPreservingClusterStateListener) storedListener).delegate.equals(listener); - } else { - return storedListener.equals(listener); - } - }); + clusterStateListeners.removeIf(storedListener -> storedListener.delegate.equals(listener)); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java index 4f09c349723de..6a9d00e62e975 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java @@ -75,7 +75,8 @@ protected void doRun() { listener.delegateFailure((delegate, response) -> { responses.add(response.getUsage()); run(); // XPackUsageFeatureTransportAction always forks to MANAGEMENT so no risk of stack overflow here - })); + }) + ); } else { assert responses.size() == usageActions.size() : responses.size() + " vs " + usageActions.size(); listener.onResponse(new XPackUsageResponse(responses)); From c5e47293ce3780cf9eac4e8205a9ba1e4b7d461e Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 8 Feb 2022 11:47:23 -0500 Subject: [PATCH 20/25] Revert cluster state listener context change. Add javadoc note --- .../restore/RestoreClusterStateListener.java | 65 +++++++++++-------- .../TransportRestoreSnapshotAction.java | 7 +- .../service/ClusterApplierService.java | 28 ++------ .../ccr/action/TransportPutFollowAction.java | 3 +- 4 files changed, 50 insertions(+), 53 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java index 2f3e92d2f55a9..c2931714e72a7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java @@ -16,10 +16,13 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreService; +import java.util.function.Supplier; + import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; public class RestoreClusterStateListener implements ClusterStateListener { @@ -29,43 +32,48 @@ public class RestoreClusterStateListener implements ClusterStateListener { private final ClusterService clusterService; private final String uuid; private final ActionListener listener; + private final Supplier contextSupplier; private RestoreClusterStateListener( ClusterService clusterService, RestoreService.RestoreCompletionResponse response, - ActionListener listener + ActionListener listener, + Supplier contextSupplier ) { this.clusterService = clusterService; this.uuid = response.getUuid(); this.listener = listener; + this.contextSupplier = contextSupplier; } @Override public void clusterChanged(ClusterChangedEvent changedEvent) { - final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); - final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); - if (prevEntry == null) { - // When there is a master failure after a restore has been started, this listener might not be registered - // on the current master and as such it might miss some intermediary cluster states due to batching. - // Clean up listener in that case and acknowledge completion of restore operation to client. - clusterService.removeListener(this); - listener.onResponse(new RestoreSnapshotResponse((RestoreInfo) null)); - } else if (newEntry == null) { - clusterService.removeListener(this); - ImmutableOpenMap shards = prevEntry.shards(); - assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state(); - assert RestoreService.completed(shards) : "expected all restore entries to be completed"; - RestoreInfo ri = new RestoreInfo( - prevEntry.snapshot().getSnapshotId().getName(), - prevEntry.indices(), - shards.size(), - shards.size() - RestoreService.failedShards(shards) - ); - RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri); - logger.debug("restore of [{}] completed", prevEntry.snapshot().getSnapshotId()); - listener.onResponse(response); - } else { - // restore not completed yet, wait for next cluster state update + try (ThreadContext.StoredContext stored = contextSupplier.get()) { + final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); + final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); + if (prevEntry == null) { + // When there is a master failure after a restore has been started, this listener might not be registered + // on the current master and as such it might miss some intermediary cluster states due to batching. + // Clean up listener in that case and acknowledge completion of restore operation to client. + clusterService.removeListener(this); + listener.onResponse(new RestoreSnapshotResponse((RestoreInfo) null)); + } else if (newEntry == null) { + clusterService.removeListener(this); + ImmutableOpenMap shards = prevEntry.shards(); + assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state(); + assert RestoreService.completed(shards) : "expected all restore entries to be completed"; + RestoreInfo ri = new RestoreInfo( + prevEntry.snapshot().getSnapshotId().getName(), + prevEntry.indices(), + shards.size(), + shards.size() - RestoreService.failedShards(shards) + ); + RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri); + logger.debug("restore of [{}] completed", prevEntry.snapshot().getSnapshotId()); + listener.onResponse(response); + } else { + // restore not completed yet, wait for next cluster state update + } } } @@ -76,8 +84,11 @@ public void clusterChanged(ClusterChangedEvent changedEvent) { public static void createAndRegisterListener( ClusterService clusterService, RestoreService.RestoreCompletionResponse response, - ActionListener listener + ActionListener listener, + ThreadContext threadContext ) { - clusterService.addListener(new RestoreClusterStateListener(clusterService, response, listener)); + clusterService.addListener( + new RestoreClusterStateListener(clusterService, response, listener, threadContext.newRestorableContext(true)) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index 7b247f1b14a42..73b66fa5d1bb5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -72,7 +72,12 @@ protected void masterOperation( ) { restoreService.restoreSnapshot(request, listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> { if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) { - RestoreClusterStateListener.createAndRegisterListener(clusterService, restoreCompletionResponse, delegatedListener); + RestoreClusterStateListener.createAndRegisterListener( + clusterService, + restoreCompletionResponse, + delegatedListener, + threadPool.getThreadContext() + ); } else { delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo())); } diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index ae90ceea1cde2..122659c64422e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -79,24 +79,7 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements private final Collection normalPriorityStateAppliers = new CopyOnWriteArrayList<>(); private final Collection lowPriorityStateAppliers = new CopyOnWriteArrayList<>(); - private static class ContextPreservingClusterStateListener implements ClusterStateListener { - private final ClusterStateListener delegate; - private final Supplier supplier; - - ContextPreservingClusterStateListener(ClusterStateListener delegate, Supplier supplier) { - this.delegate = delegate; - this.supplier = supplier; - } - - @Override - public void clusterChanged(ClusterChangedEvent event) { - try (ThreadContext.StoredContext stored = supplier.get()) { - delegate.clusterChanged(event); - } - } - } - - private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); + private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); private final Map timeoutClusterStateListeners = new ConcurrentHashMap<>(); private final AtomicReference state; // last applied state @@ -230,20 +213,17 @@ public void removeApplier(ClusterStateApplier applier) { } /** - * Add a listener for updated cluster states + * Add a listener for updated cluster states. Listeners are executed in the system thread context. */ public void addListener(ClusterStateListener listener) { - assert clusterStateListeners.stream().anyMatch(stored -> stored.delegate.equals(listener)) == false : "listener must be unique"; - clusterStateListeners.add( - new ContextPreservingClusterStateListener(listener, threadPool.getThreadContext().newRestorableContext(true)) - ); + clusterStateListeners.add(listener); } /** * Removes a listener for updated cluster states. */ public void removeListener(final ClusterStateListener listener) { - clusterStateListeners.removeIf(storedListener -> storedListener.delegate.equals(listener)); + clusterStateListeners.remove(listener); } /** diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index 1661585b5062f..e6053ce1ff818 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -268,7 +268,8 @@ public void onFailure(Exception e) { assert restoreInfo.failedShards() > 0 : "Should have failed shards"; delegatedListener.onResponse(new PutFollowAction.Response(true, false, false)); } - }) + }), + threadPool.getThreadContext() ); } From 3282b878965910b7d1ad7c25a2a2caf9bfa00416 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 8 Feb 2022 12:41:31 -0500 Subject: [PATCH 21/25] Fix typo --- .../resources/rest-api-spec/test/snapshot.create/10_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml index c00b4e4ce145e..7a51c1076a192 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml @@ -19,7 +19,7 @@ setup: --- "Create a snapshot": - - - skip: + - skip: version: "7.0.0 - 7.17.1" reason: "Pause BWC tests until #83290 is backported" From 071944db7ab23698567d2c0bdf51838681cea330 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 8 Feb 2022 14:49:20 -0500 Subject: [PATCH 22/25] Fix the watcher usage action stashing context --- .../xpack/watcher/WatcherUsageTransportAction.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java index cf4a178ba85fa..97f47e13abb7d 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -75,6 +76,10 @@ protected void masterOperation( ActionListener listener ) { if (enabled) { + ActionListener preservingListener = ContextPreservingActionListener.wrapPreservingContext( + listener, + client.threadPool().getThreadContext() + ); try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(WATCHER_ORIGIN)) { WatcherStatsRequest statsRequest = new WatcherStatsRequest(); statsRequest.includeStats(true); @@ -91,8 +96,8 @@ protected void masterOperation( true, mergedCounters.toNestedMap() ); - listener.onResponse(new XPackUsageFeatureResponse(usage)); - }, listener::onFailure)); + preservingListener.onResponse(new XPackUsageFeatureResponse(usage)); + }, preservingListener::onFailure)); } } else { WatcherFeatureSetUsage usage = new WatcherFeatureSetUsage( From 763bcd4cbc4cc103277d440604c5c83912b5e0ad Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 8 Feb 2022 17:21:51 -0500 Subject: [PATCH 23/25] Simplify bwc mute --- .../test/cat.snapshots/10_basic.yml | 2 +- .../test/snapshot.clone/10_basic.yml | 4 ++-- .../test/snapshot.create/10_basic.yml | 10 +++++----- .../test/snapshot.get/10_basic.yml | 17 +++++++---------- .../20_repository_uuid.yml | 9 +++++++-- .../test/snapshot.restore/10_basic.yml | 5 +++-- .../test/snapshot.status/10_basic.yml | 7 +++---- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml index bf2ecc51225be..460a0e5397bf2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml @@ -24,7 +24,7 @@ --- "Test cat snapshots output": - skip: - version: "7.0.0 - 7.17.1" + version: " - 8.1.0" reason: "Pause BWC tests until #83290 is backported" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml index 1a2bd39f388b4..36433a1ee4962 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "- 7.17.1" - reason: "Pause BWC tests until #83290 is backported" + version: " - 8.1.0" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml index 7a51c1076a192..f47c6ce0ba4ca 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml @@ -1,5 +1,8 @@ --- setup: + - skip: + version: " - 8.1.0" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: @@ -19,9 +22,6 @@ setup: --- "Create a snapshot": - - skip: - version: "7.0.0 - 7.17.1" - reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create: @@ -44,8 +44,8 @@ setup: --- "Create a snapshot and clean up repository": - skip: - version: " - 7.17.1" - reason: "cleanup introduced in 7.4, Pause BWC tests until #83290 is backported" + version: " - 7.3.99" + reason: cleanup introduced in 7.4 - do: snapshot.cleanup_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml index 532027a4d441d..d4c4119fc101e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -1,5 +1,8 @@ --- setup: + - skip: + version: " - 8.1.0" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: @@ -11,9 +14,6 @@ setup: --- "Get snapshot info": - - skip: - version: "7.0.0 - 7.17.1" - reason: "Pause BWC tests until #83290 is backported" - do: indices.create: @@ -64,9 +64,6 @@ setup: --- "Get snapshot info when verbose is false": - - skip: - version: "7.0.0 - 7.17.1" - reason: "Pause BWC tests until #83290 is backported" - do: indices.create: @@ -104,8 +101,8 @@ setup: --- "Get snapshot info contains include_global_state": - skip: - version: " - 7.17.1" - reason: "include_global_state field has been added in the response in 6.2.0, Pause BWC tests until #83290 is backported" + version: " - 6.1.99" + reason: "include_global_state field has been added in the response in 6.2.0" - do: indices.create: @@ -203,8 +200,8 @@ setup: --- "Get snapshot info with index details": - skip: - version: " - 7.17.1" - reason: "Introduced in 7.13.0, Pause BWC tests until #83290 is backported" + version: " - 7.12.99" + reason: "Introduced in 7.13.0" - do: indices.create: index: test_index diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml index 60a851bc4803c..b05dfbaafc4fe 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml @@ -1,8 +1,13 @@ --- +setup: + - skip: + version: " - 8.1.0" + reason: "Pause BWC tests until #83290 is backported" +--- "Get repository returns UUID": - skip: - version: " - 7.17.1" - reason: "repository UUIDs introduced in 7.12.0, Pause BWC tests until #83290 is backported" + version: " - 7.12.99" + reason: repository UUIDs introduced in 7.12.0 - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml index b0b9520dddbef..33f9a6778e8f8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml @@ -1,5 +1,8 @@ --- setup: + - skip: + version: " - 8.1.0" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: @@ -70,8 +73,6 @@ setup: - skip: features: ["allowed_warnings"] - version: "7.0.0 - 7.17.1" - reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 01dc7b6ec52e5..6db4be21e3a16 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -1,5 +1,8 @@ --- setup: + - skip: + version: " - 8.1.0" + reason: "Pause BWC tests until #83290 is backported" - do: snapshot.create_repository: @@ -11,10 +14,6 @@ setup: --- "Get snapshot status": - - skip: - version: "7.0.0 - 7.17.1" - reason: "Pause BWC tests until #83290 is backported" - - do: indices.create: index: test_index From 9593279b9bfc63ee5cb0a5302ae6427d1aa58ddc Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 8 Feb 2022 19:23:30 -0500 Subject: [PATCH 24/25] Correctly range the bwc versions --- .../resources/rest-api-spec/test/cat.snapshots/10_basic.yml | 2 +- .../resources/rest-api-spec/test/snapshot.clone/10_basic.yml | 2 +- .../resources/rest-api-spec/test/snapshot.create/10_basic.yml | 2 +- .../resources/rest-api-spec/test/snapshot.get/10_basic.yml | 2 +- .../test/snapshot.get_repository/20_repository_uuid.yml | 2 +- .../resources/rest-api-spec/test/snapshot.restore/10_basic.yml | 2 +- .../resources/rest-api-spec/test/snapshot.status/10_basic.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml index 460a0e5397bf2..23860cb412722 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.snapshots/10_basic.yml @@ -24,7 +24,7 @@ --- "Test cat snapshots output": - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml index 36433a1ee4962..80e7139cd8df3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.clone/10_basic.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml index f47c6ce0ba4ca..e060e7dff5bda 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.create/10_basic.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml index d4c4119fc101e..08753e4e732bf 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml index b05dfbaafc4fe..503c6cc7133de 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" --- "Get repository returns UUID": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml index 33f9a6778e8f8..e91f38e985e43 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 6db4be21e3a16..2c4573ccd58b8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 8.1.0" + version: " - 8.1.99" reason: "Pause BWC tests until #83290 is backported" - do: From 073dbe66761cd9d0750236de0f913175e7f88271 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 10 Feb 2022 13:18:10 -0500 Subject: [PATCH 25/25] Mute tsdb test until backport --- .../resources/rest-api-spec/test/tsdb/30_snapshot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 104b383ae811f..39c6dd4345bdf 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -1,5 +1,9 @@ --- setup: + - skip: + version: " - 8.1.99" + reason: "Pause BWC tests until #83290 is backported" + - do: snapshot.create_repository: repository: test_repo