Zen1IT testMultipleNodeMigrationFromZen1ToZen2WithThreeNodes failes on DeprecationLogging #45443
Description
very likely this is a race condition. Can only happen in test env as in prod we only have one thread context
https://gradle-enterprise.elastic.co/s/rf4gigie3ky22
It fails on:
aused by: java.lang.IllegalStateException: threadcontext is already closed
at __randomizedtesting.SeedInfo.seed([D61C383C0E0E7A14]:0)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextThreadLocal.ensureOpen(ThreadContext.java:673)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextThreadLocal.get(ThreadContext.java:666)
at org.elasticsearch.common.util.concurrent.ThreadContext.getHeader(ThreadContext.java:293)
at org.elasticsearch.common.logging.DeprecationLogger.lambda$getXOpaqueId$1(DeprecationLogger.java:266)
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
at java.util.Spliterators$ArraySpliterator.tryAdvance(Spliterators.java:958)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
at org.elasticsearch.common.logging.DeprecationLogger.getXOpaqueId(DeprecationLogger.java:267)
at org.elasticsearch.common.logging.DeprecationLogger.deprecatedAndMaybeLog(DeprecationLogger.java:137)
at org.elasticsearch.common.settings.Setting.checkDeprecation(Setting.java:504)
at org.elasticsearch.common.settings.Setting.getRaw(Setting.java:478)
at org.elasticsearch.common.settings.Setting.get(Setting.java:430)
at org.elasticsearch.common.settings.Setting.get(Setting.java:426)
at org.elasticsearch.cluster.coordination.DiscoveryUpgradeService.activate(DiscoveryUpgradeService.java:131)
at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:549)
at org.elasticsearch.cluster.coordination.Coordinator.onLeaderFailure(Coordinator.java:220)
at org.elasticsearch.cluster.coordination.LeaderChecker$CheckScheduler$2.run(LeaderChecker.java:299)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703)
when this should be protected by
DeprecationLogger.getXOpaqueId
public String getXOpaqueId(Set<ThreadContext> threadContexts) {
return threadContexts.stream()
.filter(t -> t.isClosed() == false)
.filter(t -> t.getHeader(Task.X_OPAQUE_ID) != null)
.findFirst()
.map(t -> t.getHeader(Task.X_OPAQUE_ID))
.orElse("");
}