[ML] Fix bug in inference stats persister for when feature reset is called (#77252)

The test failure originates from an unhandled exception being thrown in the ml_utility threadpool.

This exception was thrown by the results persister service, I found a place where the exception thrown was not ultimately handled.

closes #77182

Co-authored-by: Elastic Machine <[email protected]>

Author: benwtrent

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java

@@ -9,6 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -208,10 +209,15 @@ void updateStats() {
         if (shouldStop()) {
             return;
         }
-        resultsPersisterService.bulkIndexWithRetry(bulkRequest,
-            stats.stream().map(InferenceStats::getModelId).collect(Collectors.joining(",")),
-            () -> shouldStop() == false,
-            (msg) -> {});
+        String jobPattern = stats.stream().map(InferenceStats::getModelId).collect(Collectors.joining(","));
+        try {
+            resultsPersisterService.bulkIndexWithRetry(bulkRequest,
+                jobPattern,
+                () -> shouldStop() == false,
+                (msg) -> {});
+        } catch (ElasticsearchException ex) {
+            logger.warn(() -> new ParameterizedMessage("failed to store stats for [{}]", jobPattern), ex);
+        }
     }
 
     static boolean verifyIndicesExistAndPrimaryShardsAreActive(ClusterState clusterState, IndexNameExpressionResolver expressionResolver) {