Fix testClusterRecoversAfterExceptionDuringSerialization (#95038)

In #94325 we introduced another forking step when submitting a
publication, so we must extend the timeout in this test (and
`DEFAULT_CLUSTER_STATE_UPDATE_DELAY`) by `DEFAULT_DELAY_VARIABILITY`.

Closes #94905

Author: DaveCTurner

server/src/test/java/org/elasticsearch/cluster/coordination/AtomicRegisterCoordinatorTests.java

@@ -165,10 +165,6 @@ public void testJoiningNodeReceivesFullState() {
         }
     }
 
-    @Override
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/94905")
-    public void testClusterRecoversAfterExceptionDuringSerialization() {}
-
     @Override
     protected CoordinatorStrategy getCoordinatorStrategy() {
         var atomicRegister = new AtomicRegister();

server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java

@@ -1568,7 +1568,6 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
 
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/94905")
     public void testClusterRecoversAfterExceptionDuringSerialization() {
         try (Cluster cluster = new Cluster(randomIntBetween(2, 5))) {
             cluster.runRandomly();
@@ -1584,7 +1583,11 @@ public void testClusterRecoversAfterExceptionDuringSerialization() {
                 assertThat(e.getCause().getMessage(), equalTo(BrokenCustom.EXCEPTION_MESSAGE));
                 failed.set(true);
             });
-            cluster.runFor(2 * DEFAULT_DELAY_VARIABILITY + 1, "processing broken task");
+            // allow for forking 3 times:
+            // - once onto the master-service thread
+            // - once to fork the publication in FakeThreadPoolMasterService
+            // - once to fork the publication listener back onto the master-service thread
+            cluster.runFor(3 * DEFAULT_DELAY_VARIABILITY + 1, "processing broken task");
             assertTrue(failed.get());
 
             cluster.stabilise();

test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java

@@ -212,7 +212,7 @@ protected static int defaultInt(Setting<Integer> setting) {
         return setting.get(Settings.EMPTY);
     }
 
-    // Updating the cluster state involves up to the following number of delays:
+    // Updating the cluster state involves the following delays:
     // 1. submit the task to the master service
     // 2. state publisher task on master
     // 3. master sends out PublishRequests to nodes
@@ -221,7 +221,8 @@ protected static int defaultInt(Setting<Integer> setting) {
     // 6. nodes apply committed cluster state
     // 7. master receives ApplyCommitResponses
     // 8. apply committed state on master (last one to apply cluster state)
-    public static final int CLUSTER_STATE_UPDATE_NUMBER_OF_DELAYS = 8;
+    // 9. complete the publication listener back on the master service thread
+    public static final int CLUSTER_STATE_UPDATE_NUMBER_OF_DELAYS = 9;
     public static final long DEFAULT_CLUSTER_STATE_UPDATE_DELAY = CLUSTER_STATE_UPDATE_NUMBER_OF_DELAYS * DEFAULT_DELAY_VARIABILITY;
 
     private static final int ELECTION_RETRIES = 10;