From 6efc4223ba20f9c99caf785620e07ede97c8308f Mon Sep 17 00:00:00 2001 From: Henning Andersen Date: Tue, 22 Oct 2019 12:50:48 +0200 Subject: [PATCH] Reindex resilience against red shard test Now that #45739 has fixed that RED shards are no longer rarely silently ignored when using `allow_partial_search_results=false`, we can reenable the parts of the resilience testing that verifies that reindex survives RED source shards. Relates #42612 --- .../reindex/ReindexResilientSearchIT.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexResilientSearchIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexResilientSearchIT.java index 189de797c6448..e8cdaf6cc8946 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexResilientSearchIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexResilientSearchIT.java @@ -90,8 +90,6 @@ public void testDataNodeRestart() throws Exception { .mapToObj(i -> client().prepareIndex("test", "doc", String.valueOf(i)).setSource("data", i)).collect(Collectors.toList())); ensureGreen("test"); - String reindexNode = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); - NodeClient reindexNodeClient = internalCluster().getInstance(NodeClient.class, reindexNode); ReindexRequest reindexRequest = new ReindexRequest(); reindexRequest.setSourceIndices("test").setSourceBatchSize(1); @@ -106,12 +104,12 @@ public void testDataNodeRestart() throws Exception { TaskId taskId = new TaskId(response.getTaskId()); - Set names = Arrays.stream(internalCluster().getNodeNames()) + Set reindexNodeNames = Arrays.stream(internalCluster().getNodeNames()) .map(name -> Tuple.tuple(internalCluster().getInstance(NodeClient.class, name).getLocalNodeId(), name)) .filter(idAndName -> taskId.getNodeId().equals(idAndName.v1())).map(Tuple::v2).collect(Collectors.toSet()); - assertEquals(1, names.size()); - String notToRestart = names.iterator().next(); + assertEquals(1, reindexNodeNames.size()); + String notToRestart = reindexNodeNames.iterator().next(); internalCluster().getInstances(NodeClient.class); assertBusy(() -> { @@ -123,19 +121,21 @@ public void testDataNodeRestart() throws Exception { assertThat(client().admin().cluster().prepareListTasks().setActions(SearchAction.NAME).get().getTasks(), Matchers.empty()); }, 30, TimeUnit.SECONDS); + Set restartableNodes = internalCluster().nodesInclude("test") + .stream().filter(id -> id.equals(notToRestart) == false).collect(Collectors.toSet()); for (int i = 0; i < randomIntBetween(1,5); ++i) { - // todo: replace following two lines with below once search fails on RED every time. - String nodeToRestart = randomFrom(internalCluster().nodesInclude("test") - .stream().filter(id -> id.equals(notToRestart) == false).collect(Collectors.toSet())); - internalCluster().restartNode(nodeToRestart, new InternalTestCluster.RestartCallback()); - ensureGreen(); -// internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() { -// @Override -// public Settings onNodeStopped(String nodeName) throws Exception { -// internalCluster().restartRandomDataNode(); -// return super.onNodeStopped(nodeName); -// } -// }); + String node1ToRestart = randomFrom(restartableNodes); + internalCluster().restartNode(node1ToRestart, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) throws Exception { + if (restartableNodes.size() > 1) { + String node2ToRestart = randomValueOtherThan(node1ToRestart, () -> randomFrom(restartableNodes)); + logger.info("--> restarting second node: " + node2ToRestart); + internalCluster().restartNode(node2ToRestart, new InternalTestCluster.RestartCallback()); + } + return super.onNodeStopped(nodeName); + } + }); } rethrottle().setTaskId(taskId)