Merge branch 'master' into index-lifecycle

Author: colings86

client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java

@@ -129,7 +129,6 @@ public void testClusterHealthYellowClusterLevel() throws IOException {
         createIndex("index2", Settings.EMPTY);
         ClusterHealthRequest request = new ClusterHealthRequest();
         request.timeout("5s");
-        request.level(ClusterHealthRequest.Level.CLUSTER);
         ClusterHealthResponse response = execute(request, highLevelClient().cluster()::health, highLevelClient().cluster()::healthAsync);
 
         assertYellowShards(response);
@@ -170,6 +169,7 @@ public void testClusterHealthYellowSpecificIndex() throws IOException {
         createIndex("index", Settings.EMPTY);
         createIndex("index2", Settings.EMPTY);
         ClusterHealthRequest request = new ClusterHealthRequest("index");
+        request.level(ClusterHealthRequest.Level.SHARDS);
         request.timeout("5s");
         ClusterHealthResponse response = execute(request, highLevelClient().cluster()::health, highLevelClient().cluster()::healthAsync);
 

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -1568,7 +1568,7 @@ public void testClusterHealth() {
             healthRequest.level(level);
             expectedParams.put("level", level.name().toLowerCase(Locale.ROOT));
         } else {
-            expectedParams.put("level", "shards");
+            expectedParams.put("level", "cluster");
         }
         if (randomBoolean()) {
             Priority priority = randomFrom(Priority.values());

docs/java-rest/high-level/cluster/health.asciidoc

@@ -67,6 +67,7 @@ include-tagged::{doc-tests}/ClusterClientDocumentationIT.java[health-request-wai
 include-tagged::{doc-tests}/ClusterClientDocumentationIT.java[health-request-level]
 --------------------------------------------------
 <1> The level of detail of the returned health information. Accepts a `ClusterHealthRequest.Level` value.
+Default value is `cluster`.
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------

docs/reference/migration/migrate_7_0/restclient.asciidoc

@@ -10,4 +10,11 @@ header, e.g. `client.index(indexRequest)` becomes
 `client.index(indexRequest, RequestOptions.DEFAULT)`.
 In case you are specifying headers
 e.g. `client.index(indexRequest, new Header("name" "value"))` becomes
-`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());`
+`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());`
+
+==== Cluster Health API default to `cluster` level
+
+The Cluster Health API used to default to `shards` level to ease migration
+from transport client that doesn't support the `level` parameter and always
+returns information including indices and shards details. The level default
+value has been aligned with the Elasticsearch default level: `cluster`.

server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java

@@ -48,9 +48,9 @@ public class ClusterHealthRequest extends MasterNodeReadRequest<ClusterHealthReq
     private Priority waitForEvents = null;
     /**
      * Only used by the high-level REST Client. Controls the details level of the health information returned.
-     * The default value is 'shards' so it is backward compatible with the transport client behaviour.
+     * The default value is 'cluster'.
      */
-    private Level level = Level.SHARDS;
+    private Level level = Level.CLUSTER;
 
     public ClusterHealthRequest() {
     }
@@ -250,8 +250,7 @@ public Priority waitForEvents() {
 
     /**
      * Set the level of detail for the health information to be returned.
-     * Only used by the high-level REST Client
-     * The default value is 'shards' so it is backward compatible with the transport client behaviour.
+     * Only used by the high-level REST Client.
      */
     public void level(Level level) {
         this.level = Objects.requireNonNull(level, "level must not be null");
@@ -260,7 +259,6 @@ public void level(Level level) {
     /**
      * Get the level of detail for the health information to be returned.
      * Only used by the high-level REST Client.
-     * The default value is 'shards' so it is backward compatible with the transport client behaviour.
      */
     public Level level() {
         return level;

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -82,8 +82,10 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
         Settings.Builder settingsForOpenIndices = Settings.builder();
         final Set<String> skippedSettings = new HashSet<>();
 
-        indexScopedSettings.validate(normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false  /* don't validate wildcards */),
-            false); //don't validate dependencies here we check it below never allow to change the number of shards
+        indexScopedSettings.validate(
+                normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false), // don't validate wildcards
+                false, // don't validate dependencies here we check it below never allow to change the number of shards
+                true); // validate internal index settings
         for (String key : normalizedSettings.keySet()) {
             Setting setting = indexScopedSettings.get(key);
             boolean isWildcard = setting == null && Regex.isSimpleMatchPattern(key);

server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java

@@ -282,6 +282,18 @@ public final void validate(final Settings settings, final boolean validateDepend
         validate(settings, validateDependencies, false, false);
     }
 
+    /**
+     * Validates that all settings are registered and valid.
+     *
+     * @param settings              the settings to validate
+     * @param validateDependencies  true if dependent settings should be validated
+     * @param validateInternalIndex true if internal index settings should be validated
+     * @see Setting#getSettingsDependencies(String)
+     */
+    public final void validate(final Settings settings, final boolean validateDependencies, final boolean validateInternalIndex) {
+        validate(settings, validateDependencies, false, false, validateInternalIndex);
+    }
+
     /**
      * Validates that all settings are registered and valid.
      *
@@ -296,6 +308,25 @@ public final void validate(
             final boolean validateDependencies,
             final boolean ignorePrivateSettings,
             final boolean ignoreArchivedSettings) {
+        validate(settings, validateDependencies, ignorePrivateSettings, ignoreArchivedSettings, false);
+    }
+
+    /**
+     * Validates that all settings are registered and valid.
+     *
+     * @param settings               the settings
+     * @param validateDependencies   true if dependent settings should be validated
+     * @param ignorePrivateSettings  true if private settings should be ignored during validation
+     * @param ignoreArchivedSettings true if archived settings should be ignored during validation
+     * @param validateInternalIndex  true if index internal settings should be validated
+     * @see Setting#getSettingsDependencies(String)
+     */
+    public final void validate(
+            final Settings settings,
+            final boolean validateDependencies,
+            final boolean ignorePrivateSettings,
+            final boolean ignoreArchivedSettings,
+            final boolean validateInternalIndex) {
         final List<RuntimeException> exceptions = new ArrayList<>();
         for (final String key : settings.keySet()) { // settings iterate in deterministic fashion
             if (isPrivateSetting(key) && ignorePrivateSettings) {
@@ -305,7 +336,7 @@ public final void validate(
                 continue;
             }
             try {
-                validate(key, settings, validateDependencies);
+                validate(key, settings, validateDependencies, validateInternalIndex);
             } catch (final RuntimeException ex) {
                 exceptions.add(ex);
             }
@@ -314,9 +345,27 @@ public final void validate(
     }
 
     /**
-     * Validates that the setting is valid
+     * Validates that the settings is valid.
+     *
+     * @param key the key of the setting to validate
+     * @param settings the settings
+     * @param validateDependencies true if dependent settings should be validated
+     * @throws IllegalArgumentException if the setting is invalid
      */
-    void validate(String key, Settings settings, boolean validateDependencies) {
+    void validate(final String key, final Settings settings, final boolean validateDependencies) {
+        validate(key, settings, validateDependencies, false);
+    }
+
+    /**
+     * Validates that the settings is valid.
+     *
+     * @param key the key of the setting to validate
+     * @param settings the settings
+     * @param validateDependencies true if dependent settings should be validated
+     * @param validateInternalIndex true if internal index settings should be validated
+     * @throws IllegalArgumentException if the setting is invalid
+     */
+    void validate(final String key, final Settings settings, final boolean validateDependencies, final boolean validateInternalIndex) {
         Setting setting = getRaw(key);
         if (setting == null) {
             LevensteinDistance ld = new LevensteinDistance();
@@ -356,6 +405,11 @@ void validate(String key, Settings settings, boolean validateDependencies) {
                     }
                 }
             }
+            // the only time that validateInternalIndex should be true is if this call is coming via the update settings API
+            if (validateInternalIndex && setting.getProperties().contains(Setting.Property.InternalIndex)) {
+                throw new IllegalArgumentException(
+                        "can not update internal setting [" + setting.getKey() + "]; this setting is managed via a dedicated API");
+            }
         }
         setting.get(settings);
     }

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -120,7 +120,13 @@ public enum Property {
          * Mark this setting as not copyable during an index resize (shrink or split). This property can only be applied to settings that
          * also have {@link Property#IndexScope}.
          */
-        NotCopyableOnResize
+        NotCopyableOnResize,
+
+        /**
+         * Indicates an index-level setting that is managed internally. Such a setting can only be added to an index on index creation but
+         * can not be updated via the update API.
+         */
+        InternalIndex
     }
 
     private final Key key;
@@ -152,14 +158,18 @@ private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings
             if (propertiesAsSet.contains(Property.Dynamic) && propertiesAsSet.contains(Property.Final)) {
                 throw new IllegalArgumentException("final setting [" + key + "] cannot be dynamic");
             }
-            if (propertiesAsSet.contains(Property.NotCopyableOnResize) && propertiesAsSet.contains(Property.IndexScope) == false) {
-                throw new IllegalArgumentException(
-                        "non-index-scoped setting [" + key + "] can not have property [" + Property.NotCopyableOnResize + "]");
-            }
+            checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize);
+            checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex);
             this.properties = propertiesAsSet;
         }
     }
 
+    private void checkPropertyRequiresIndexScope(final EnumSet<Property> properties, final Property property) {
+        if (properties.contains(property) && properties.contains(Property.IndexScope) == false) {
+            throw new IllegalArgumentException("non-index-scoped setting [" + key + "] can not have property [" + property + "]");
+        }
+    }
+
     /**
      * Creates a new Setting instance
      * @param key the settings key for this setting.

server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java

@@ -876,4 +876,28 @@ public void testFinalSettingUpdateFail() {
                 Settings.builder().put(currentSettings), Settings.builder(), "node"));
         assertThat(exc.getMessage(), containsString("final node setting [some.final.group.foo]"));
     }
+
+    public void testInternalIndexSettingsFailsValidation() {
+        final Setting<String> indexInternalSetting = Setting.simpleString("index.internal", Property.InternalIndex, Property.IndexScope);
+        final IndexScopedSettings indexScopedSettings =
+                new IndexScopedSettings(Settings.EMPTY, Collections.singleton(indexInternalSetting));
+        final IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> {
+                    final Settings settings = Settings.builder().put("index.internal", "internal").build();
+                    indexScopedSettings.validate(settings, false, /* validateInternalIndex */ true);
+                });
+        final String message = "can not update internal setting [index.internal]; this setting is managed via a dedicated API";
+        assertThat(e, hasToString(containsString(message)));
+    }
+
+    public void testInternalIndexSettingsSkipValidation() {
+        final Setting<String> internalIndexSetting = Setting.simpleString("index.internal", Property.InternalIndex, Property.IndexScope);
+        final IndexScopedSettings indexScopedSettings = 
+                new IndexScopedSettings(Settings.EMPTY, Collections.singleton(internalIndexSetting));
+        // nothing should happen, validation should not throw an exception
+        final Settings settings = Settings.builder().put("index.internal", "internal").build();
+        indexScopedSettings.validate(settings, false, /* validateInternalIndex */ false);
+    }
+
 }

server/src/test/java/org/elasticsearch/common/settings/SettingTests.java

@@ -735,6 +735,13 @@ public void testRejectNonIndexScopedNotCopyableOnResizeSetting() {
         assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [NotCopyableOnResize]")));
     }
 
+    public void testRejectNonIndexScopedIndexInternalSetting() {
+        final IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> Setting.simpleString("foo.bar", Property.InternalIndex));
+        assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [InternalIndex]")));
+    }
+
     public void testTimeValue() {
         final TimeValue random = TimeValue.parseTimeValue(randomTimeValue(), "test");