Fix another potential race in the file settings watcher (#90302)

Fix another potential race in the file settings watcher for issue #89500

Backports #90302 to 8.5

Author: thecoop

docs/changelog/90302.yaml

@@ -0,0 +1,5 @@
+pr: 90302
+summary: Fix another potential race in the file settings watcher
+area: Infra/Core
+type: bug
+issues: [89500]

server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java

@@ -341,6 +341,7 @@ synchronized void stopWatcher() {
         logger.debug("stopping watcher ...");
         if (watching()) {
             try {
+                // make sure the watcher thread hits the processing latch correctly
                 cleanupWatchKeys();
                 fileUpdateState = null;
                 watchService.close();
@@ -350,6 +351,8 @@ synchronized void stopWatcher() {
                 if (watcherThreadLatch != null) {
                     watcherThreadLatch.await();
                 }
+                // the watcher thread might have snuck in behind us and re-created the settings watch again
+                cleanupWatchKeys();
             } catch (IOException e) {
                 logger.warn("encountered exception while closing watch service", e);
             } catch (InterruptedException interruptedException) {