Validates updated settings on closed indices (#24487)

We allow non-dynamic settings to be updated on closed indices but we don't
check if the updated settings can be used to open/create the index.
This can lead to unrecoverable state where the settings are updated but the index
cannot be reopened since the settings are not valid. Trying to update the invalid settings
is also not possible since the update will fail to validate the current settings.
This change adds the validation of the updated settings for closed indices and make sure that the new settings
do not prevent the reopen of the index.

Fixes #23787

Author: jimczi

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -276,7 +276,11 @@ public ClusterState execute(ClusterState currentState) {
                     for (Index index : closeIndices) {
                         final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                         final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
+                        // Verifies that the current index settings can be updated with the updated dynamic settings.
                         indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData);
+                        // Now check that we can create the index with the updated settings (dynamic and non-dynamic).
+                        // This step is mandatory since we allow to update non-dynamic settings on closed indices.
+                        indicesService.verifyIndexMetadata(updatedMetaData, updatedMetaData);
                     }
                 } catch (IOException ex) {
                     throw ExceptionsHelper.convertToElastic(ex);

core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java

@@ -49,6 +49,18 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateSettingsIT extends ESIntegTestCase {
+    public void testInvalidUpdateOnClosedIndex() {
+        createIndex("test");
+        assertAcked(client().admin().indices().prepareClose("test").get());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.analysis.char_filter.invalid_char.type", "invalid"))
+                .get());
+        assertEquals(exception.getMessage(), "Unknown char_filter type [invalid] for [invalid_char]");
+    }
 
     public void testInvalidDynamicUpdate() {
         createIndex("test");