[CI] TransportMonitoringMigrateAlertsActionTests classMethod failing

[dimitris-athanasiou]

Build scan:
https://gradle-enterprise.elastic.co/s/ys5hkvmqqgse4/tests/:x-pack:plugin:monitoring:test/org.elasticsearch.xpack.monitoring.action.TransportMonitoringMigrateAlertsActionTests/classMethod

Reproduction line:
null

Applicable branches:
main

Reproduces locally?:
Didn't try

Failure history:
https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.xpack.monitoring.action.TransportMonitoringMigrateAlertsActionTests&tests.test=classMethod

Failure excerpt:

com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=95, name=elasticsearch[node_s2][scheduler][T#1], state=RUNNABLE, group=TGRP-TransportMonitoringMigrateAlertsActionTests]


  Caused by: java.lang.AssertionError: java.lang.AssertionError: listener.onFailure failed

    at __randomizedtesting.SeedInfo.seed([E1953ECFFA89B81C]:0)
    at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:97)
    at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:425)
    at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)
    at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1368)
    at org.elasticsearch.transport.TransportService$5.doRun(TransportService.java:910)
    at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
    at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:223)
    at org.elasticsearch.transport.TransportService.handleInternalSendException(TransportService.java:896)
    at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:861)
    at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:758)
    at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:704)
    at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$5(CoordinationDiagnosticsService.java:547)
    at org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:162)
    at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
    at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:100)
    at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:131)
    at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:139)
    at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:52)
    at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
    at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:432)
    at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$9(CoordinationDiagnosticsService.java:581)
    at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:710)
    at org.elasticsearch.threadpool.ThreadPool$1.run(ThreadPool.java:436)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
    at java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    at java.lang.Thread.run(Thread.java:833)

    Caused by: java.lang.AssertionError: listener.onFailure failed

      at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:97)
      at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:425)
      at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)
      at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1368)
      at org.elasticsearch.transport.TransportService$5.doRun(TransportService.java:910)
      at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
      at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:223)
      at org.elasticsearch.transport.TransportService.handleInternalSendException(TransportService.java:896)
      at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:861)
      at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:758)
      at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:704)
      at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$5(CoordinationDiagnosticsService.java:547)
      at org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:162)
      at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
      at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:100)
      at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:131)
      at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:139)
      at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:52)
      at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
      at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:432)
      at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$9(CoordinationDiagnosticsService.java:581)
      at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:710)
      at org.elasticsearch.threadpool.ThreadPool$1.run(ThreadPool.java:436)
      at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
      at java.util.concurrent.FutureTask.run(FutureTask.java:264)
      at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
      at java.lang.Thread.run(Thread.java:833)

      Caused by: org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@2e1e9dcc[Not completed, task = java.util.concurrent.Executors$RunnableAdapter@4a7830e1[Wrapped task = org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService$$Lambda$7314/0x0000000801adfd88@f21c7cf]] on org.elasticsearch.threadpool.Scheduler$SafeScheduledThreadPoolExecutor@4542c159[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 224] (shutdown)

        at org.elasticsearch.common.util.concurrent.EsRejectedExecutionHandler.newRejectedException(EsRejectedExecutionHandler.java:40)
        at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:34)
        at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)
        at java.util.concurrent.ScheduledThreadPoolExecutor.delayedExecute(ScheduledThreadPoolExecutor.java:340)
        at java.util.concurrent.ScheduledThreadPoolExecutor.schedule(ScheduledThreadPoolExecutor.java:562)
        at org.elasticsearch.threadpool.ThreadPool.schedule(ThreadPool.java:458)
        at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.fetchClusterFormationInfo(CoordinationDiagnosticsService.java:571)
        at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$rescheduleFetchConsumer$3(CoordinationDiagnosticsService.java:512)
        at java.util.function.Consumer.lambda$andThen$0(Consumer.java:65)
        at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$8(CoordinationDiagnosticsService.java:568)
        at org.elasticsearch.action.ActionListener$2.onFailure(ActionListener.java:170)
        at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:115)
        at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:100)
        at org.elasticsearch.common.util.concurrent.BaseFuture.setException(BaseFuture.java:149)
        at org.elasticsearch.common.util.concurrent.ListenableFuture.onFailure(ListenableFuture.java:147)
        at org.elasticsearch.action.StepListener.innerOnFailure(StepListener.java:57)
        at org.elasticsearch.action.NotifyOnceListener.onFailure(NotifyOnceListener.java:36)
        at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:92)
        at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:425)
        at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)
        at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1368)
        at org.elasticsearch.transport.TransportService$5.doRun(TransportService.java:910)
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
        at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:223)
        at org.elasticsearch.transport.TransportService.handleInternalSendException(TransportService.java:896)
        at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:861)
        at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:758)
        at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:704)
        at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$5(CoordinationDiagnosticsService.java:547)
        at org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:162)
        at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
        at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:100)
        at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:131)
        at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:139)
        at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:52)
        at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29)
        at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:432)
        at org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService.lambda$fetchClusterFormationInfo$9(CoordinationDiagnosticsService.java:581)
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:710)
        at org.elasticsearch.threadpool.ThreadPool$1.run(ThreadPool.java:436)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.lang.Thread.run(Thread.java:833)

[elasticsearchmachine]

Pinging @elastic/es-data-management (Team:Data Management)

[masseyke]

This one looks like it is caused by #88397.