diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java
index 18b1071280307..2f3ea6c83a536 100644
--- a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java
+++ b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java
@@ -239,6 +239,9 @@ public void testOverflowToDisk() throws Exception {
             throw e;
         }
 
+        // flushing the job forces an index refresh, see https://github.com/elastic/elasticsearch/issues/31173
+        flushJob(job.getId(), false);
+
         List<ForecastRequestStats> forecastStats = getForecastStats();
         assertThat(forecastStats.size(), equalTo(1));
         ForecastRequestStats forecastRequestStats = forecastStats.get(0);
@@ -261,6 +264,16 @@ public void testOverflowToDisk() throws Exception {
         }
 
         closeJob(job.getId());
+
+        forecastStats = getForecastStats();
+        assertThat(forecastStats.size(), equalTo(2));
+        for (ForecastRequestStats stats : forecastStats) {
+            forecasts = getForecasts(job.getId(), stats);
+
+            assertThat(forecastRequestStats.getRecordCount(), equalTo(8000L));
+            assertThat(forecasts.size(), equalTo(8000));
+        }
+
     }
 
     private void createDataWithLotsOfClientIps(TimeValue bucketSpan, Job.Builder job) throws IOException {