From 698d5573cd010d880645cfd13010f984d7b51c5e Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 19 Feb 2019 12:38:53 +0100 Subject: [PATCH 1/2] Fix Deadlock in HotThreadsIT The way this dead-locks is: * The `size` method on the internal cluster needs a monitor on the internal cluster object in the select thread on node_1 * `client()` call on the test thread connects to node_1 at the same time, holding the monitor on the internal test cluster during the connect call * That connect can not finish because the select loop that would dispatch it is blocked waiting for the `size()` call to complete => Deadlock --- .../org/elasticsearch/action/admin/HotThreadsIT.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java index 7c00705b2a28e..2bcc899b1c219 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -43,12 +43,13 @@ import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class HotThreadsIT extends ESIntegTestCase { - public void testHotThreadsDontFail() throws ExecutionException, InterruptedException { - /** - * This test just checks if nothing crashes or gets stuck etc. - */ + /** + * This test just checks if nothing crashes or gets stuck etc. + */ + public void testHotThreadsDontFail() throws InterruptedException { createIndex("test"); final int iters = scaledRandomIntBetween(2, 20); final AtomicBoolean hasErrors = new AtomicBoolean(false); @@ -81,6 +82,7 @@ public void testHotThreadsDontFail() throws ExecutionException, InterruptedExcep type = null; } final CountDownLatch latch = new CountDownLatch(1); + final int clusterNodes = cluster().size(); nodesHotThreadsRequestBuilder.execute(new ActionListener() { @Override public void onResponse(NodesHotThreadsResponse nodeHotThreads) { @@ -88,7 +90,7 @@ public void onResponse(NodesHotThreadsResponse nodeHotThreads) { try { assertThat(nodeHotThreads, notNullValue()); Map nodesMap = nodeHotThreads.getNodesMap(); - assertThat(nodesMap.size(), equalTo(cluster().size())); + assertThat(nodesMap.size(), equalTo(clusterNodes)); for (NodeHotThreads ht : nodeHotThreads.getNodes()) { assertNotNull(ht.getHotThreads()); //logger.info(ht.getHotThreads()); From 41141147931f6767e2688a054cc684db120c5bf4 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 19 Feb 2019 15:36:53 +0100 Subject: [PATCH 2/2] remove unnecessary change --- .../test/java/org/elasticsearch/action/admin/HotThreadsIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java index 2bcc899b1c219..10069ad28ba7f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -43,7 +43,6 @@ import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class HotThreadsIT extends ESIntegTestCase { /**