[CI] TimeSeriesLifecycleActionsIT.testDeleteActionDoesntDeleteSearchableSnapshot() test failure #67712
Description
Build scan: https://gradle-enterprise.elastic.co/s/c3mez4rfm56tu
Repro line: ./gradlew ':x-pack:plugin:ilm:qa:multi-node:javaRestTest' --tests "org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.testDeleteActionDoesntDeleteSearchableSnapshot" -Dtests.seed=B0524C5D620D9C0D -Dtests.security.manager=true -Dtests.locale=de-LU -Dtests.timezone=America/Boa_Vista -Druntime.java=8
Reproduces locally?: No
Applicable branches: master, 7.x, 7.11, 7.10
Failure history: First time this year, but did fail a few time last year.
Failure excerpt:
java.lang.AssertionError |
-- | --
| at __randomizedtesting.SeedInfo.seed([B0524C5D620D9C0D:D878CF02E53A7B7C]:0) |
| at org.junit.Assert.fail(Assert.java:86) |
| at org.junit.Assert.assertTrue(Assert.java:41) |
| at org.junit.Assert.assertFalse(Assert.java:64) |
| at org.junit.Assert.assertFalse(Assert.java:74) |
| at org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.lambda$testDeleteActionDoesntDeleteSearchableSnapshot$113(TimeSeriesLifecycleActionsIT.java:1449) |
| at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:1016) |
| at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:989) |
| at org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.testDeleteActionDoesntDeleteSearchableSnapshot(TimeSeriesLifecycleActionsIT.java:1449)
I found a few eye catchy error that are worth highlighting and may cause the actual failure:
java.lang.IllegalStateException: expected index [.ds-logs-rxsvsrmggk-2021.01.19-000001] with policy [policy-XDOPt] to have current step consistent with provided step key ({"phase":"cold","action":"searchable_snapshot","name":"branch-check-prerequisites"}) but it was {"phase":"cold","action":"searchable_snapshot","name":"ERROR"} |
-- | --
| » at org.elasticsearch.xpack.ilm.IndexLifecycleRunner.maybeRunAsyncAction(IndexLifecycleRunner.java:285) ~[?:?] |
| » at org.elasticsearch.xpack.ilm.ExecuteStepsUpdateTask.clusterStateProcessed(ExecuteStepsUpdateTask.java:194) ~[?:?] |
| » at org.elasticsearch.cluster.service.MasterService$SafeClusterStateTaskListener.clusterStateProcessed(MasterService.java:534) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.MasterService$TaskOutputs.lambda$processedDifferentClusterState$1(MasterService.java:421) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at java.util.ArrayList.forEach(ArrayList.java:1259) [?:1.8.0_271] |
| » at org.elasticsearch.cluster.service.MasterService$TaskOutputs.processedDifferentClusterState(MasterService.java:421) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.MasterService.onPublicationSuccess(MasterService.java:281) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:273) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_271] |
| » at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_271] |
| » at java.lang.Thread.run(Thread.java:748) [?:1.8.0_271]
ERROR][o.e.x.i.h.ILMHistoryStore] [javaRestTest-2] failed to index 14 items into ILM history index |
-- | --
| » java.lang.IllegalStateException: Duplicate key [ilm-history-5] IndexNotFoundException[no such index [ilm-history-5]] |
| » at java.util.stream.Collectors.lambda$throwingMerger$0(Collectors.java:133) ~[?:1.8.0_271] |
| » at java.util.HashMap.merge(HashMap.java:1254) ~[?:1.8.0_271] |
| » at java.util.stream.Collectors.lambda$toMap$58(Collectors.java:1320) ~[?:1.8.0_271] |
| » at java.util.stream.ReduceOps$3ReducingSink.accept(ReduceOps.java:169) ~[?:1.8.0_271] |
| » at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[?:1.8.0_271] |
| » at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_271] |
| » at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482) ~[?:1.8.0_271] |
| » at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472) ~[?:1.8.0_271] |
| » at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708) ~[?:1.8.0_271] |
| » at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_271] |
| » at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499) ~[?:1.8.0_271] |
| » at org.elasticsearch.xpack.ilm.history.ILMHistoryStore$1.afterBulk(ILMHistoryStore.java:93) ~[?:?] |
| » at org.elasticsearch.action.bulk.BulkRequestHandler$1.onResponse(BulkRequestHandler.java:65) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.BulkRequestHandler$1.onResponse(BulkRequestHandler.java:62) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:264) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:171) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:115) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:77) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:94) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:88) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.ActionListener$5.onResponse(ActionListener.java:297) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation.doRun(TransportBulkAction.java:509) ~[elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.TransportBulkAction.executeBulk(TransportBulkAction.java:655) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.action.bulk.TransportBulkAction$1$1.doRun(TransportBulkAction.java:264) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:743) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.12.0-SNAPSHOT.jar:7.12.0-SNAPSHOT] |
| » at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_271] |
| » at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_271] |
| » at java.lang.Thread.run(Thread.java:748) [?:1.8.0_271]