Improve upgrade experience of node level index settings

In 5.0 we don't allow index settings to be specified on the node level ie.
in yaml files or via commandline argument. This can cause problems during
upgrade if this was used extensively. For instance if analyzers where
specified on a node level this might cause the index to be closed when
imported (see #17187). In such a case all indices relying on this
must be updated via `PUT /${index}/_settings`. Yet, this API has slightly
different semantics since it overrides existing settings. To make this less
painful this change adds a `preserve_existing` parameter on that API to ensure
we have the same semantics as if the setting was applied on the node level.

This change also adds a better error message and a change to the migration guide
to ensure upgrades are smooth if index settings are specified on the node level.

If a index setting is detected this change fails the node startup and prints a message
like this:
```
*************************************************************************************
Found index level settings on node level configuration.

Since elasticsearch 5.x index level settings can NOT be set on the nodes
configuration like the elasticsearch.yaml, in system properties or command line
arguments.In order to upgrade all indices the settings must be updated via the
/${index}/_settings API. Unless all settings are dynamic all indices must be closed
in order to apply the upgradeIndices created in the future should use index templates
to set default values.

Please ensure all required values are updated on all indices by executing:

curl -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '{
  "index.number_of_shards" : "1",
  "index.query.default_field" : "main_field",
  "index.translog.durability" : "async",
  "index.ttl.disable_purge" : "true"
}'
*************************************************************************************
```

Author: s1monw

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

@@ -80,6 +80,7 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste
         UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
                 .indices(concreteIndices)
                 .settings(request.settings())
+                .setPreserveExisting(request.isPreserveExisting())
                 .ackTimeout(request.timeout())
                 .masterNodeTimeout(request.masterNodeTimeout());
 

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java

@@ -29,8 +29,23 @@ public class UpdateSettingsClusterStateUpdateRequest extends IndicesClusterState
 
     private Settings settings;
 
-    public UpdateSettingsClusterStateUpdateRequest() {
+    private boolean preserveExisting = false;
 
+    /**
+     * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
+     * it should not be overwritten by this update. The default is <code>false</code>
+     */
+    public boolean isPreserveExisting() {
+        return preserveExisting;
+    }
+
+    /**
+     * Iff set to <code>true</code> this settings update will only add settings not already set on an index. Existing settings remain
+     * unchanged.
+     */
+    public UpdateSettingsClusterStateUpdateRequest setPreserveExisting(boolean preserveExisting) {
+        this.preserveExisting = preserveExisting;
+        return this;
     }
 
     /**

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java

@@ -47,6 +47,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
     private String[] indices;
     private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true);
     private Settings settings = EMPTY_SETTINGS;
+    private boolean preserveExisting = false;
 
     public UpdateSettingsRequest() {
     }
@@ -127,6 +128,23 @@ public UpdateSettingsRequest settings(String source) {
         return this;
     }
 
+    /**
+     * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
+     * it should not be overwritten by this update. The default is <code>false</code>
+     */
+    public boolean isPreserveExisting() {
+        return preserveExisting;
+    }
+
+    /**
+     * Iff set to <code>true</code> this settings update will only add settings not already set on an index. Existing settings remain
+     * unchanged.
+     */
+    public UpdateSettingsRequest setPreserveExisting(boolean preserveExisting) {
+        this.preserveExisting = preserveExisting;
+        return this;
+    }
+
     /**
      * Sets the settings to be updated (either json/yaml/properties format)
      */
@@ -149,6 +167,7 @@ public void readFrom(StreamInput in) throws IOException {
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         settings = readSettingsFromStream(in);
         readTimeout(in);
+        preserveExisting = in.readBoolean();
     }
 
     @Override
@@ -158,5 +177,6 @@ public void writeTo(StreamOutput out) throws IOException {
         indicesOptions.writeIndicesOptions(out);
         writeSettingsToStream(settings, out);
         writeTimeout(out);
+        out.writeBoolean(preserveExisting);
     }
 }

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java

@@ -84,4 +84,9 @@ public UpdateSettingsRequestBuilder setSettings(Map<String, Object> source) {
         request.settings(source);
         return this;
     }
+
+    public UpdateSettingsRequestBuilder setPreserveExisting(boolean preserveExisting) {
+        request.setPreserveExisting(preserveExisting);
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -176,6 +176,7 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
         final Settings skippedSettigns = skipppedSettings.build();
         final Settings closedSettings = settingsForClosedIndices.build();
         final Settings openSettings = settingsForOpenIndices.build();
+        final boolean preserveExisting = request.isPreserveExisting();
 
         clusterService.submitStateUpdateTask("update-settings",
                 new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
@@ -221,7 +222,7 @@ public ClusterState execute(ClusterState currentState) {
                 }
 
                 int updatedNumberOfReplicas = openSettings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
-                if (updatedNumberOfReplicas != -1) {
+                if (updatedNumberOfReplicas != -1 && preserveExisting == false) {
                     routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices);
@@ -239,6 +240,9 @@ public ClusterState execute(ClusterState currentState) {
                         Settings.Builder updates = Settings.builder();
                         Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
                         if (indexScopedSettings.updateDynamicSettings(openSettings, indexSettings, updates, index.getName())) {
+                            if (preserveExisting) {
+                                indexSettings.put(indexMetaData.getSettings());
+                            }
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                         }
                     }
@@ -250,6 +254,9 @@ public ClusterState execute(ClusterState currentState) {
                         Settings.Builder updates = Settings.builder();
                         Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
                         if (indexScopedSettings.updateSettings(closedSettings, indexSettings, updates, index.getName())) {
+                            if (preserveExisting) {
+                                indexSettings.put(indexMetaData.getSettings());
+                            }
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                         }
                     }

core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java

@@ -30,6 +30,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
@@ -221,9 +224,17 @@ public final void validate(Settings.Builder settingsBuilder) {
      * * Validates that all given settings are registered and valid
      */
     public final void validate(Settings settings) {
-        for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
-            validate(entry.getKey(), settings);
+        List<RuntimeException> exceptions = new ArrayList<>();
+        // we want them sorted for deterministic error messages
+        SortedMap<String, String> sortedSettings = new TreeMap<>(settings.getAsMap());
+        for (Map.Entry<String, String> entry : sortedSettings.entrySet()) {
+            try {
+                validate(entry.getKey(), settings);
+            } catch (RuntimeException ex) {
+                exceptions.add(ex);
+            }
         }
+        ExceptionsHelper.rethrowAndSuppress(exceptions);
     }
 
 

core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java

@@ -20,13 +20,22 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tribe.TribeService;
 
+import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * A module that binds the provided settings to the {@link Settings} interface.
@@ -37,9 +46,12 @@ public class SettingsModule extends AbstractModule {
     private final Set<String> settingsFilterPattern = new HashSet<>();
     private final Map<String, Setting<?>> nodeSettings = new HashMap<>();
     private final Map<String, Setting<?>> indexSettings = new HashMap<>();
-    private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.") && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
+    private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.")
+        && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
+    private final ESLogger logger;
 
     public SettingsModule(Settings settings) {
+        logger = Loggers.getLogger(getClass(), settings);
         this.settings = settings;
         for (Setting<?> setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) {
             registerSetting(setting);
@@ -53,6 +65,56 @@ public SettingsModule(Settings settings) {
     protected void configure() {
         final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
         final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
+        Settings indexSettings = settings.filter((s) -> s.startsWith("index.") && clusterSettings.get(s) == null);
+        if (indexSettings.isEmpty() == false) {
+            try {
+                String separator = IntStream.range(0, 85).mapToObj(s -> "*").collect(Collectors.joining("")).trim();
+                StringBuilder builder = new StringBuilder();
+                builder.append(System.lineSeparator());
+                builder.append(separator);
+                builder.append(System.lineSeparator());
+                builder.append("Found index level settings on node level configuration.");
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                int count = 0;
+                for (String word : ("Since elasticsearch 5.x index level settings can NOT be set on the nodes configuration like " +
+                    "the elasticsearch.yaml, in system properties or command line arguments." +
+                    "In order to upgrade all indices the settings must be updated via the /${index}/_settings API. " +
+                    "Unless all settings are dynamic all indices must be closed in order to apply the upgrade" +
+                    "Indices created in the future should use index templates to set default values."
+                    ).split(" ")) {
+                    if (count + word.length() > 85) {
+                        builder.append(System.lineSeparator());
+                        count = 0;
+                    }
+                    count += word.length() + 1;
+                    builder.append(word).append(" ");
+                }
+
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                builder.append("Please ensure all required values are updated on all indices by executing: ");
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                builder.append("curl -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '");
+                try (XContentBuilder xContentBuilder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                    xContentBuilder.prettyPrint();
+                    xContentBuilder.startObject();
+                    indexSettings.toXContent(xContentBuilder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+                    xContentBuilder.endObject();
+                    builder.append(xContentBuilder.string());
+                }
+                builder.append("'");
+                builder.append(System.lineSeparator());
+                builder.append(separator);
+                builder.append(System.lineSeparator());
+
+                logger.warn(builder.toString());
+                throw new IllegalArgumentException("node settings must not contain any index level settings");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
         // by now we are fully configured, lets check node level settings for unregistered index settings
         final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate();
         clusterSettings.validate(settings.filter(acceptOnlyClusterSettings));

core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java

@@ -47,6 +47,7 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
             "timeout",
             "master_timeout",
             "index",
+            "preserve_existing",
             "expand_wildcards",
             "ignore_unavailable",
             "allow_no_indices"));
@@ -62,6 +63,7 @@ public RestUpdateSettingsAction(Settings settings, RestController controller, Cl
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
         UpdateSettingsRequest updateSettingsRequest = updateSettingsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         updateSettingsRequest.timeout(request.paramAsTime("timeout", updateSettingsRequest.timeout()));
+        updateSettingsRequest.setPreserveExisting(request.paramAsBoolean("preserve_existing", updateSettingsRequest.isPreserveExisting()));
         updateSettingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", updateSettingsRequest.masterNodeTimeout()));
         updateSettingsRequest.indicesOptions(IndicesOptions.fromRequest(request, updateSettingsRequest.indicesOptions()));
 

core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java

@@ -36,12 +36,35 @@ public void testValidate() {
         {
             Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]").build();
             SettingsModule module = new SettingsModule(settings);
-            try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
-                fail();
-            } catch (IllegalArgumentException ex) {
-                assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
-            }
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]")
+                .put("some.foo.bar", 1).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
+            assertEquals(1, ex.getSuppressed().length);
+            assertEquals("unknown setting [some.foo.bar]", ex.getSuppressed()[0].getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("index.codec", "default")
+                .put("index.foo.bar", 1).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("node settings must not contain any index level settings", ex.getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("index.codec", "default").build();
+            SettingsModule module = new SettingsModule(settings);
+            assertInstanceBinding(module, Settings.class, (s) -> s == settings);
         }
     }
 

docs/reference/migration/migrate_5_0/settings.asciidoc

@@ -12,6 +12,15 @@ that plugins that define custom settings must register all of their settings
 during plugin loading using the `SettingsModule#registerSettings(Setting)`
 method.
 
+==== Index Level Settings
+
+In previous versions Elasticsearch allowed to specify index level setting
+as _defaults_ on the node level, inside the `elasticsearch.yaml` file or even via
+command-line parameters. From Elasticsearch 5.0 on only selected settings like
+for instance `index.codec` can be set on the node level. All other settings must be
+set on each individual index. To set default values on every index, index templates
+should be used instead.
+
 ==== Node settings
 
 The `name` setting has been removed and is replaced by `node.name`. Usage of