From c4827445f293c5bac612a43bfed357c9a3f23e0f Mon Sep 17 00:00:00 2001
From: Jay Modi <jaymode@users.noreply.github.com>
Date: Thu, 12 Mar 2020 08:41:30 -0600
Subject: [PATCH 001/285] Deprecate the logstash enabled setting (#53367)

The setting, `xpack.logstash.enabled`, exists to enable or disable the
logstash extensions found within x-pack. In practice, this setting had
no effect on the functionality of the extension. Given this, the
setting is now deprecated in preparation for removal.
---
 .../java/org/elasticsearch/xpack/CcrIntegTestCase.java     | 1 -
 .../org/elasticsearch/xpack/CcrSingleNodeTestCase.java     | 1 -
 .../java/org/elasticsearch/xpack/core/XPackSettings.java   | 2 +-
 .../xpack/ilm/IndexLifecycleInitialisationTests.java       | 1 -
 .../xpack/slm/SnapshotLifecycleInitialisationTests.java    | 1 -
 .../java/org/elasticsearch/xpack/logstash/Logstash.java    | 7 +------
 .../xpack/logstash/LogstashInfoTransportActionTests.java   | 4 ++++
 7 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index 7181dd9b6c9a2..c349ea764101f 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -225,7 +225,6 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
-        builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         // Let cluster state api return quickly in order to speed up auto follow tests:
         builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
index cbfd078452368..9858362c811de 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
@@ -48,7 +48,6 @@ protected Settings nodeSettings() {
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
-        builder.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         // Let cluster state api return quickly in order to speed up auto follow tests:
         builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index f61232cf97b1b..740d1015a7eb6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -82,7 +82,7 @@ private XPackSettings() {
 
     /** Setting for enabling or disabling Logstash extensions. Defaults to true. */
     public static final Setting<Boolean> LOGSTASH_ENABLED = Setting.boolSetting("xpack.logstash.enabled", true,
-            Setting.Property.NodeScope);
+            Setting.Property.NodeScope, Property.Deprecated);
 
     /**
      * Setting for enabling or disabling the index lifecycle extension. Defaults to true.
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 4f4c2913fb2a4..965c5beb5af04 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -100,7 +100,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
 
         // This is necessary to prevent ILM and SLM installing a lifecycle policy, these tests assume a blank slate
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
index 180ff53cf5b97..58f0b632e1321 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
@@ -50,7 +50,6 @@ protected Settings nodeSettings() {
         settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
         settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
         settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
         settings.put(Environment.PATH_REPO_SETTING.getKey(), repositoryLocation);
 
         settings.put(XPackSettings.SNAPSHOT_LIFECYCLE_ENABLED.getKey(), true);
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index 9c7ef923c10d1..3d22402e89779 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
-import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
@@ -37,11 +36,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin {
     private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template";
     private static final String TEMPLATE_VERSION_VARIABLE = "logstash.template.version";
 
-    private final boolean enabled;
-
-    public Logstash(Settings settings) {
-        this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
-    }
+    public Logstash() {}
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
index f78d6424fd148..e3872352b944d 100644
--- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
+++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
@@ -8,11 +8,13 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
 
@@ -42,6 +44,8 @@ public void testEnabledSetting() throws Exception {
         usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new LogstashFeatureSetUsage(out.bytes().streamInput());
         assertThat(serializedUsage.enabled(), is(enabled));
+
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] { XPackSettings.LOGSTASH_ENABLED });
     }
 
     public void testEnabledDefault() throws Exception {

From ae55e06253aa39e8e242e3e2feceab0e8e18e424 Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 12 Mar 2020 09:27:25 -0600
Subject: [PATCH 002/285] Add ComponentTemplate to MetaData (#53290)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add ComponentTemplate to MetaData

This adds a `ComponentTemplate` datastructure that will be used as part of #53101 (Index Templates
v2) to the `MetaData` class. Currently there are no APIs for interacting with this class, so it will
always be an empty map (other than in tests). This infrastructure will be built upon to add APIs in
a subsequent commit.

A `ComponentTemplate` is made up of a `Template`, a version, and a MetaData.Custom class. The
`Template` contains similar information to an `IndexTemplateMetaData` object— settings, mappings,
and alias configuration.
---
 .../elasticsearch/cluster/ClusterModule.java  |   5 +
 .../cluster/metadata/ComponentTemplate.java   | 300 ++++++++++++++++++
 .../metadata/ComponentTemplateMetadata.java   | 170 ++++++++++
 .../cluster/metadata/MetaData.java            |  35 ++
 .../ComponentTemplateMetadataTests.java       |  70 ++++
 .../metadata/ComponentTemplateTests.java      | 163 ++++++++++
 .../cluster/metadata/MetaDataTests.java       |  36 ++-
 .../metadata/ToAndFromJsonMetaDataTests.java  |  16 +
 8 files changed, 794 insertions(+), 1 deletion(-)
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java

diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index e445615e0fc73..70138ecc981bc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -127,6 +128,8 @@ public static List<Entry> getNamedWriteables() {
         registerMetaDataCustom(entries, IndexGraveyard.TYPE, IndexGraveyard::new, IndexGraveyard::readDiffFrom);
         registerMetaDataCustom(entries, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new,
             PersistentTasksCustomMetaData::readDiffFrom);
+        registerMetaDataCustom(entries, ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new,
+            ComponentTemplateMetadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
@@ -145,6 +148,8 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
             IndexGraveyard::fromXContent));
         entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE),
             PersistentTasksCustomMetaData::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(ComponentTemplateMetadata.TYPE),
+            ComponentTemplateMetadata::fromXContent));
         return entries;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
new file mode 100644
index 0000000000000..2b160b1b2fb11
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A component template is a re-usable template as well as metadata about the template. Each
+ * component template is expected to be valid on its own. For example, if a component template
+ * contains a field "foo", it's expected to contain all the necessary settings/mappings/etc for the
+ * "foo" field. These component templates make up the individual pieces composing an index template.
+ */
+public class ComponentTemplate extends AbstractDiffable<ComponentTemplate> implements ToXContentObject {
+    private static final ParseField TEMPLATE = new ParseField("template");
+    private static final ParseField VERSION = new ParseField("version");
+    private static final ParseField METADATA = new ParseField("_meta");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ComponentTemplate, Void> PARSER =
+        new ConstructingObjectParser<>("component_template", false,
+            a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map<String, Object>) a[2]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Template.PARSER, TEMPLATE);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
+    }
+
+    private final Template template;
+    @Nullable
+    private final Long version;
+    @Nullable
+    private final Map<String, Object> metadata;
+
+    static Diff<ComponentTemplate> readComponentTemplateDiffFrom(StreamInput in) throws IOException {
+        return AbstractDiffable.readDiffFrom(ComponentTemplate::new, in);
+    }
+
+    public static ComponentTemplate parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ComponentTemplate(Template template, @Nullable Long version, @Nullable Map<String, Object> metadata) {
+        this.template = template;
+        this.version = version;
+        this.metadata = metadata;
+    }
+
+    public ComponentTemplate(StreamInput in) throws IOException {
+        this.template = new Template(in);
+        this.version = in.readOptionalVLong();
+        if (in.readBoolean()) {
+            this.metadata = in.readMap();
+        } else {
+            this.metadata = null;
+        }
+    }
+
+    public Template template() {
+        return template;
+    }
+
+    @Nullable
+    public Long version() {
+        return version;
+    }
+
+    @Nullable
+    public Map<String, Object> metadata() {
+        return metadata;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        this.template.writeTo(out);
+        out.writeOptionalVLong(this.version);
+        if (this.metadata == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            out.writeMap(this.metadata);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(template, version, metadata);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ComponentTemplate other = (ComponentTemplate) obj;
+        return Objects.equals(template, other.template) &&
+            Objects.equals(version, other.version) &&
+            Objects.equals(metadata, other.metadata);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TEMPLATE.getPreferredName(), this.template);
+        if (this.version != null) {
+            builder.field(VERSION.getPreferredName(), this.version);
+        }
+        if (this.metadata != null) {
+            builder.field(METADATA.getPreferredName(), this.metadata);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    static class Template extends AbstractDiffable<Template> implements ToXContentObject {
+        private static final ParseField SETTINGS = new ParseField("settings");
+        private static final ParseField MAPPINGS = new ParseField("mappings");
+        private static final ParseField ALIASES = new ParseField("aliases");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<Template, Void> PARSER = new ConstructingObjectParser<>("template", false,
+            a -> new Template((Settings) a[0], (CompressedXContent) a[1], (Map<String, AliasMetaData>) a[2]));
+
+        static {
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
+                new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+                Map<String, AliasMetaData> aliasMap = new HashMap<>();
+                while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    AliasMetaData alias = AliasMetaData.Builder.fromXContent(p);
+                    aliasMap.put(alias.alias(), alias);
+                }
+                return aliasMap;
+            }, ALIASES);
+        }
+
+        @Nullable
+        private final Settings settings;
+        @Nullable
+        private final CompressedXContent mappings;
+        @Nullable
+        private final Map<String, AliasMetaData> aliases;
+
+        Template(@Nullable Settings settings, @Nullable CompressedXContent mappings, @Nullable Map<String, AliasMetaData> aliases) {
+            this.settings = settings;
+            this.mappings = mappings;
+            this.aliases = aliases;
+        }
+
+        Template(StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                this.settings = Settings.readSettingsFromStream(in);
+            } else {
+                this.settings = null;
+            }
+            if (in.readBoolean()) {
+                this.mappings = CompressedXContent.readCompressedString(in);
+            } else {
+                this.mappings = null;
+            }
+            if (in.readBoolean()) {
+                this.aliases = in.readMap(StreamInput::readString, AliasMetaData::new);
+            } else {
+                this.aliases = null;
+            }
+        }
+
+        public Settings settings() {
+            return settings;
+        }
+
+        public CompressedXContent mappings() {
+            return mappings;
+        }
+
+        public Map<String, AliasMetaData> aliases() {
+            return aliases;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (this.settings == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                Settings.writeSettingsToStream(this.settings, out);
+            }
+            if (this.mappings == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                this.mappings.writeTo(out);
+            }
+            if (this.aliases == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                out.writeMap(this.aliases, StreamOutput::writeString, (stream, aliasMetaData) -> aliasMetaData.writeTo(stream));
+            }
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(settings, mappings, aliases);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Template other = (Template) obj;
+            return Objects.equals(settings, other.settings) &&
+                Objects.equals(mappings, other.mappings) &&
+                Objects.equals(aliases, other.aliases);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (this.settings != null) {
+                builder.startObject(SETTINGS.getPreferredName());
+                this.settings.toXContent(builder, params);
+                builder.endObject();
+            }
+            if (this.mappings != null) {
+                Map<String, Object> uncompressedMapping =
+                    XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2();
+                if (uncompressedMapping.size() > 0) {
+                    builder.field(MAPPINGS.getPreferredName());
+                    builder.map(uncompressedMapping);
+                }
+            }
+            if (this.aliases != null) {
+                builder.startObject(ALIASES.getPreferredName());
+                for (AliasMetaData alias : this.aliases.values()) {
+                    AliasMetaData.Builder.toXContent(alias, builder, params);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
new file mode 100644
index 0000000000000..241cdca592cb5
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * {@link ComponentTemplateMetadata} is a custom {@link MetaData} implementation for storing a map
+ * of component templates and their names.
+ */
+public class ComponentTemplateMetadata implements MetaData.Custom {
+    public static final String TYPE = "component_template";
+    private static final ParseField COMPONENT_TEMPLATE = new ParseField("component_template");
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ComponentTemplateMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
+        a -> new ComponentTemplateMetadata((Map<String, ComponentTemplate>) a[0]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
+            Map<String, ComponentTemplate> templates = new HashMap<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                String name = p.currentName();
+                templates.put(name, ComponentTemplate.parse(p));
+            }
+            return templates;
+        }, COMPONENT_TEMPLATE);
+    }
+    private final Map<String, ComponentTemplate> componentTemplates;
+
+    public ComponentTemplateMetadata(Map<String, ComponentTemplate> componentTemplates) {
+        this.componentTemplates = componentTemplates;
+    }
+
+    public ComponentTemplateMetadata(StreamInput in) throws IOException {
+        this.componentTemplates = in.readMap(StreamInput::readString, ComponentTemplate::new);
+    }
+
+    public Map<String, ComponentTemplate> componentTemplates() {
+        return this.componentTemplates;
+    }
+
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new ComponentTemplateMetadataDiff((ComponentTemplateMetadata) before, this);
+    }
+
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new ComponentTemplateMetadataDiff(in);
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        // TODO: update after backporting
+        return Version.V_8_0_0;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(this.componentTemplates, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
+    }
+
+    public static ComponentTemplateMetadata fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(COMPONENT_TEMPLATE.getPreferredName());
+        for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {
+            builder.field(template.getKey(), template.getValue());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.componentTemplates);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ComponentTemplateMetadata other = (ComponentTemplateMetadata) obj;
+        return Objects.equals(this.componentTemplates, other.componentTemplates);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    static class ComponentTemplateMetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, ComponentTemplate>> componentTemplateDiff;
+
+        ComponentTemplateMetadataDiff(ComponentTemplateMetadata before, ComponentTemplateMetadata after) {
+            this.componentTemplateDiff = DiffableUtils.diff(before.componentTemplates, after.componentTemplates,
+                DiffableUtils.getStringKeySerializer());
+        }
+
+        ComponentTemplateMetadataDiff(StreamInput in) throws IOException {
+            this.componentTemplateDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                ComponentTemplate::new, ComponentTemplate::readComponentTemplateDiffFrom);
+        }
+
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            return new ComponentTemplateMetadata(componentTemplateDiff.apply(((ComponentTemplateMetadata) part).componentTemplates));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            componentTemplateDiff.writeTo(out);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 00d3a8d708d62..6154e6ecd675f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -73,6 +73,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -661,6 +662,12 @@ public ImmutableOpenMap<String, IndexTemplateMetaData> getTemplates() {
         return this.templates;
     }
 
+    public Map<String, ComponentTemplate> componentTemplates() {
+        return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
+            .map(ComponentTemplateMetadata::componentTemplates)
+            .orElse(Collections.emptyMap());
+    }
+
     public ImmutableOpenMap<String, Custom> customs() {
         return this.customs;
     }
@@ -1052,6 +1059,34 @@ public Builder templates(ImmutableOpenMap<String, IndexTemplateMetaData> templat
             return this;
         }
 
+        public Builder put(String name, ComponentTemplate componentTemplate) {
+            Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
+            // ಠ_ಠ at ImmutableOpenMap
+            Map<String, ComponentTemplate> existingTemplates =
+                Optional.ofNullable((ComponentTemplateMetadata) this.customs.get(ComponentTemplateMetadata.TYPE))
+                    .map(ctm -> new HashMap<>(ctm.componentTemplates()))
+                    .orElse(new HashMap<>());
+            existingTemplates.put(name, componentTemplate);
+            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(existingTemplates));
+            return this;
+        }
+
+        public Builder removeComponentTemplate(String name) {
+            // ಠ_ಠ at ImmutableOpenMap
+            Map<String, ComponentTemplate> existingTemplates =
+                Optional.ofNullable((ComponentTemplateMetadata) this.customs.get(ComponentTemplateMetadata.TYPE))
+                    .map(ctm -> new HashMap<>(ctm.componentTemplates()))
+                    .orElse(new HashMap<>());
+            existingTemplates.remove(name);
+            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(existingTemplates));
+            return this;
+        }
+
+        public Builder componentTemplates(Map<String, ComponentTemplate> componentTemplates) {
+            this.customs.put(ComponentTemplateMetadata.TYPE, new ComponentTemplateMetadata(componentTemplates));
+            return this;
+        }
+
         public Custom getCustom(String type) {
             return customs.get(type);
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
new file mode 100644
index 0000000000000..e8d654603bc12
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ComponentTemplateMetadataTests extends AbstractNamedWriteableTestCase<ComponentTemplateMetadata> {
+
+    @Override
+    protected ComponentTemplateMetadata createTestInstance() {
+        int count = randomIntBetween(0, 3);
+        Map<String, ComponentTemplate> templateMap = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            templateMap.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance());
+        }
+        return new ComponentTemplateMetadata(templateMap);
+    }
+
+    @Override
+    protected ComponentTemplateMetadata mutateInstance(ComponentTemplateMetadata instance) throws IOException {
+        if (instance.componentTemplates().size() == 0) {
+            // Not really much to mutate, so just generate a new one
+            return randomValueOtherThan(instance, this::createTestInstance);
+        }
+        Map<String, ComponentTemplate> templates = new HashMap<>(instance.componentTemplates());
+        Map.Entry<String, ComponentTemplate> newTemplate = instance.componentTemplates().entrySet().iterator().next();
+        if (randomBoolean()) {
+            // Change the key
+            templates.put(randomAlphaOfLength(4), newTemplate.getValue());
+        } else {
+            // Change the value
+            templates.put(newTemplate.getKey(), ComponentTemplateTests.mutateTemplate(newTemplate.getValue()));
+        }
+        return new ComponentTemplateMetadata(templates);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.singletonList(new NamedWriteableRegistry.Entry(ComponentTemplateMetadata.class,
+            ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new)));
+    }
+
+    @Override
+    protected Class<ComponentTemplateMetadata> categoryClass() {
+        return ComponentTemplateMetadata.class;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
new file mode 100644
index 0000000000000..f0c1b523ac9c1
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public class ComponentTemplateTests extends AbstractDiffableSerializationTestCase<ComponentTemplate> {
+    @Override
+    protected ComponentTemplate makeTestChanges(ComponentTemplate testInstance) {
+        try {
+            return mutateInstance(testInstance);
+        } catch (IOException e) {
+            logger.error(e);
+            fail("mutating should not throw an exception, but got: " + e);
+            return null;
+        }
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<ComponentTemplate>> diffReader() {
+        return ComponentTemplate::readComponentTemplateDiffFrom;
+    }
+
+    @Override
+    protected ComponentTemplate doParseInstance(XContentParser parser) throws IOException {
+        return ComponentTemplate.parse(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<ComponentTemplate> instanceReader() {
+        return ComponentTemplate::new;
+    }
+
+    @Override
+    protected ComponentTemplate createTestInstance() {
+        return randomInstance();
+    }
+
+    public static ComponentTemplate randomInstance() {
+        Settings settings = null;
+        CompressedXContent mappings = null;
+        Map<String, AliasMetaData> aliases = null;
+        if (randomBoolean()) {
+            settings = randomSettings();
+        }
+        if (randomBoolean()) {
+            mappings = randomMappings();
+        }
+        if (randomBoolean()) {
+            aliases = randomAliases();
+        }
+        ComponentTemplate.Template template = new ComponentTemplate.Template(settings, mappings, aliases);
+
+        Map<String, Object> meta = null;
+        if (randomBoolean()) {
+            meta = randomMeta();
+        }
+        return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta);
+    }
+
+    private static Map<String, AliasMetaData> randomAliases() {
+        String aliasName = randomAlphaOfLength(5);
+        AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
+            .filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
+            .routing(randomBoolean() ? null : randomAlphaOfLength(3))
+            .isHidden(randomBoolean() ? null : randomBoolean())
+            .writeIndex(randomBoolean() ? null : randomBoolean())
+            .build();
+        return Collections.singletonMap(aliasName, aliasMeta);
+    }
+
+    private static CompressedXContent randomMappings() {
+        try {
+            return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}");
+        } catch (IOException e) {
+            fail("got an IO exception creating fake mappings: " + e);
+            return null;
+        }
+    }
+
+    private static Settings randomSettings() {
+        return Settings.builder()
+            .put(randomAlphaOfLength(4), randomAlphaOfLength(10))
+            .build();
+    }
+
+    private static Map<String, Object> randomMeta() {
+        if (randomBoolean()) {
+            return Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4));
+        } else {
+            return Collections.singletonMap(randomAlphaOfLength(5),
+                Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4)));
+        }
+    }
+
+    @Override
+    protected ComponentTemplate mutateInstance(ComponentTemplate orig) throws IOException {
+        return mutateTemplate(orig);
+    }
+
+    public static ComponentTemplate mutateTemplate(ComponentTemplate orig) {
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                switch (randomIntBetween(0, 2)) {
+                    case 0:
+                        ComponentTemplate.Template ot = orig.template();
+                        return new ComponentTemplate(
+                            new ComponentTemplate.Template(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings),
+                                ot.mappings(), ot.aliases()),
+                            orig.version(), orig.metadata());
+                    case 1:
+                        ComponentTemplate.Template ot2 = orig.template();
+                        return new ComponentTemplate(
+                            new ComponentTemplate.Template(ot2.settings(),
+                                randomValueOtherThan(ot2.mappings(), ComponentTemplateTests::randomMappings), ot2.aliases()),
+                            orig.version(), orig.metadata());
+                    case 2:
+                        ComponentTemplate.Template ot3 = orig.template();
+                        return new ComponentTemplate(
+                            new ComponentTemplate.Template(ot3.settings(), ot3.mappings(),
+                                randomValueOtherThan(ot3.aliases(), ComponentTemplateTests::randomAliases)),
+                            orig.version(), orig.metadata());
+                    default:
+                        throw new IllegalStateException("illegal randomization branch");
+                }
+            case 1:
+                return new ComponentTemplate(orig.template(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong),
+                    orig.metadata());
+            case 2:
+                return new ComponentTemplate(orig.template(), orig.version(),
+                    randomValueOtherThan(orig.metadata(), ComponentTemplateTests::randomMeta));
+            default:
+                throw new IllegalStateException("illegal randomization branch");
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 5bd981f246a56..1d0bd3e36c57b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -723,7 +724,7 @@ public void testFindMappingsWithFilters() throws IOException {
         }
     }
 
-    private IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) {
+    private static IndexMetaData.Builder buildIndexMetaData(String name, String alias, Boolean writeIndex) {
         return IndexMetaData.builder(name)
             .settings(settings(Version.CURRENT)).creationDate(randomNonNegativeLong())
             .putAlias(AliasMetaData.builder(alias).writeIndex(writeIndex))
@@ -906,4 +907,37 @@ public void testBuilderRejectsNullInCustoms() {
         final ImmutableOpenMap<String, MetaData.Custom> map = mapBuilder.build();
         assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key));
     }
+
+    public void testSerialization() throws IOException {
+        final MetaData orig = randomMetaData();
+        final BytesStreamOutput out = new BytesStreamOutput();
+        orig.writeTo(out);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+        final MetaData fromStreamMeta = MetaData.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(),
+            namedWriteableRegistry));
+        assertTrue(MetaData.isGlobalStateEquals(orig, fromStreamMeta));
+    }
+
+    public static MetaData randomMetaData() {
+        return MetaData.builder()
+            .put(buildIndexMetaData("index", "alias", randomBoolean() ? null : randomBoolean()).build(), randomBoolean())
+            .put(IndexTemplateMetaData.builder("template" + randomAlphaOfLength(3))
+                .patterns(Arrays.asList("bar-*", "foo-*"))
+                .settings(Settings.builder()
+                    .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+                    .build())
+                .build())
+            .persistentSettings(Settings.builder()
+                .put("setting" + randomAlphaOfLength(3), randomAlphaOfLength(4))
+                .build())
+            .transientSettings(Settings.builder()
+                .put("other_setting" + randomAlphaOfLength(3), randomAlphaOfLength(4))
+                .build())
+            .clusterUUID("uuid" + randomAlphaOfLength(3))
+            .clusterUUIDCommitted(randomBoolean())
+            .indexGraveyard(IndexGraveyardTests.createRandom())
+            .version(randomNonNegativeLong())
+            .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
+            .build();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index 0338a64b6fe7e..b03c41484893f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
@@ -116,6 +117,11 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         .putAlias(newAliasMetaDataBuilder("alias-bar1"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put("component_template", new ComponentTemplate(
+                    new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
+                        new CompressedXContent("{\"baz\":\"eggplant\"}"),
+                        Collections.singletonMap("alias", AliasMetaData.builder("alias").build())),
+                    5L, Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))))
                 .put(IndexMetaData.builder("test12")
                         .settings(settings(Version.CURRENT)
                                 .put("setting1", "value1")
@@ -285,6 +291,16 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").alias(), equalTo("alias-bar3"));
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").indexRouting(), equalTo("routing-bar"));
         assertThat(parsedMetaData.templates().get("foo").aliases().get("alias-bar3").searchRouting(), equalTo("routing-bar"));
+
+        // component template
+        assertNotNull(parsedMetaData.componentTemplates().get("component_template"));
+        assertThat(parsedMetaData.componentTemplates().get("component_template").version(), is(5L));
+        assertThat(parsedMetaData.componentTemplates().get("component_template").metadata(),
+            equalTo(Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))));
+        assertThat(parsedMetaData.componentTemplates().get("component_template").template(),
+            equalTo(new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
+                new CompressedXContent("{\"baz\":\"eggplant\"}"),
+                Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
     }
 
     private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";

From a8d89fd0a580d88f127cf93a43a3c2378179bff4 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 12 Mar 2020 11:51:16 -0400
Subject: [PATCH 003/285] Fix concurrent requests race over scroll context
 limit (#53449)

Concurrent search scroll requests can lead to more scroll contexts than the limit.
---
 .../elasticsearch/search/SearchService.java   | 15 ++++---
 .../search/SearchServiceTests.java            | 41 +++++++++++++++++++
 2 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index e8eacdafbaa07..c7fa46bf9a353 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -642,7 +642,6 @@ private void onNewContext(SearchContext context) {
         boolean success = false;
         try {
             if (context.scrollContext() != null) {
-                openScrollContexts.incrementAndGet();
                 context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
             context.indexShard().getSearchOperationListener().onNewContext(context);
@@ -661,14 +660,15 @@ private void onNewContext(SearchContext context) {
     final SearchContext createContext(SearchRewriteContext rewriteContext) throws IOException {
         final DefaultSearchContext context = createSearchContext(rewriteContext, defaultSearchTimeout);
         try {
-            if (rewriteContext.request != null && openScrollContexts.get() >= maxOpenScrollContext) {
-                throw new ElasticsearchException(
-                    "Trying to create too many scroll contexts. Must be less than or equal to: [" +
-                        maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
-                        + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
-            }
             final ShardSearchRequest request = rewriteContext.request;
             if (request.scroll() != null) {
+                context.addReleasable(openScrollContexts::decrementAndGet, Lifetime.CONTEXT);
+                if (openScrollContexts.incrementAndGet() > maxOpenScrollContext) {
+                    throw new ElasticsearchException(
+                        "Trying to create too many scroll contexts. Must be less than or equal to: [" +
+                            maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
+                            + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
+                }
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
@@ -768,7 +768,6 @@ private void onFreeContext(SearchContext context) {
         assert activeContexts.containsKey(context.id().getId()) == false;
         context.indexShard().getSearchOperationListener().onFreeContext(context);
         if (context.scrollContext() != null) {
-            openScrollContexts.decrementAndGet();
             context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 1f8f5b424478e..7607d519d1d8c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -539,6 +539,47 @@ public void testMaxOpenScrollContexts() throws RuntimeException, IOException {
             ex.getMessage());
     }
 
+    public void testOpenScrollContextsConcurrently() throws Exception {
+        createIndex("index");
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexShard indexShard = indicesService.indexServiceSafe(resolveIndex("index")).getShard(0);
+
+        final int maxScrollContexts = SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY);
+        final SearchService searchService = getInstanceFromNode(SearchService.class);
+        Thread[] threads = new Thread[randomIntBetween(2, 8)];
+        CountDownLatch latch = new CountDownLatch(threads.length);
+        for (int i = 0; i < threads.length; i++) {
+            threads[i] = new Thread(() -> {
+                latch.countDown();
+                try {
+                    latch.await();
+                    for (; ; ) {
+                        SearchService.SearchRewriteContext rewriteContext =
+                            searchService.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
+                        try {
+                            searchService.createAndPutContext(rewriteContext);
+                        } catch (ElasticsearchException e) {
+                            assertThat(e.getMessage(), equalTo(
+                                "Trying to create too many scroll contexts. Must be less than or equal to: " +
+                                    "[" + maxScrollContexts + "]. " +
+                                    "This limit can be set by changing the [search.max_open_scroll_context] setting."));
+                            return;
+                        }
+                    }
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+            });
+            threads[i].setName("elasticsearch[node_s_0][search]");
+            threads[i].start();
+        }
+        for (Thread thread : threads) {
+            thread.join();
+        }
+        assertThat(searchService.getActiveContexts(), equalTo(maxScrollContexts));
+        searchService.freeAllScrollContexts();
+    }
+
     public static class FailOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
         @Override
         public List<QuerySpec<?>> getQueries() {

From 970f726c1f0a844aa0224611ebdac34f7c89b0aa Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Thu, 12 Mar 2020 12:49:25 -0400
Subject: [PATCH 004/285] [ML] renaming inference processor field
 field_mappings to new name field_map (#53433)

This renames the `inference` processor configuration field `field_mappings` to `field_map`.

`field_mappings` is now deprecated.
---
 .../client/MachineLearningIT.java             |   2 +-
 .../ingest/processors/inference.asciidoc      |   4 +-
 .../ml/qa/ml-with-security/build.gradle       |   3 +
 .../ml/integration/InferenceIngestIT.java     |  14 +--
 .../inference/ingest/InferenceProcessor.java  |  21 +++-
 .../MachineLearningLicensingTests.java        |   2 +-
 ...sportGetTrainedModelsStatsActionTests.java |   2 +-
 .../InferenceProcessorFactoryTests.java       |  18 +--
 .../rest-api-spec/test/ml/inference_crud.yml  |   2 +-
 .../test/ml/inference_processor.yml           | 114 ++++++++++++++++++
 .../test/ml/inference_stats_crud.yml          |   4 +-
 11 files changed, 156 insertions(+), 30 deletions(-)
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 2f923abf04983..e13464aeddabd 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -2265,7 +2265,7 @@ public void testGetTrainedModelsStats() throws Exception {
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc
index 009fbd619aace..b1b110e2b3889 100644
--- a/docs/reference/ingest/processors/inference.asciidoc
+++ b/docs/reference/ingest/processors/inference.asciidoc
@@ -14,7 +14,7 @@ ingested in the pipeline.
 | Name               | Required  | Default                        | Description
 | `model_id`         | yes       | -                              | (String) The ID of the model to load and infer against.
 | `target_field`     | no        | `ml.inference.<processor_tag>` | (String) Field added to incoming documents to contain results objects.
-| `field_mappings`   | yes       | -                              | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
+| `field_map`        | yes       | -                              | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
 | `inference_config` | yes       | -                              | (Object) Contains the inference type and its options. There are two types: <<inference-processor-regression-opt,`regression`>> and <<inference-processor-classification-opt,`classification`>>.
 include::common-options.asciidoc[]
 |======
@@ -26,7 +26,7 @@ include::common-options.asciidoc[]
   "inference": {
     "model_id": "flight_delay_regression-1571767128603",
     "target_field": "FlightDelayMin_prediction_infer",
-    "field_mappings": {},
+    "field_map": {},
     "inference_config": { "regression": {} }
   }
 }
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index ed90561faf1e9..516e5c626fca2 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -139,6 +139,9 @@ integTest.runner {
     'ml/inference_crud/Test put ensemble with empty models',
     'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index',
     'ml/inference_crud/Test put model with empty input.field_names',
+    'ml/inference_processor/Test create processor with missing mandatory fields',
+    'ml/inference_processor/Test create and delete pipeline with inference processor',
+    'ml/inference_processor/Test create processor with deprecated fields',
     'ml/inference_stats_crud/Test get stats given missing trained model',
     'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false',
     'ml/jobs_crud/Test cannot create job with existing categorizer state document',
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
index 8bae148fd55c1..d6c88fbfdb994 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
@@ -120,7 +120,7 @@ public void testSimulate() throws IOException {
             "                \"num_top_feature_importance_values\": 2" +
             "          }},\n" +
             "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -133,7 +133,7 @@ public void testSimulate() throws IOException {
             "          \"target_field\": \"ml.regression\",\n" +
             "          \"model_id\": \"test_regression\",\n" +
             "          \"inference_config\": {\"regression\":{}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -166,7 +166,7 @@ public void testSimulate() throws IOException {
             "        \"inference\": {\n" +
             "          \"model_id\": \"test_classification_missing\",\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -204,7 +204,7 @@ public void testSimulateWithDefaultMappedField() throws IOException {
             "                \"num_top_feature_importance_values\": 2" +
             "          }},\n" +
             "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }\n"+
             "    ]\n" +
@@ -233,7 +233,7 @@ public void testSimulateLangIdent() throws IOException {
             "        \"inference\": {\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
             "          \"model_id\": \"lang_ident_model_1\",\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }\n" +
             "    ]\n" +
@@ -570,7 +570,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "          \"model_id\": \"test_classification\",\n" +
         "          \"tag\": \"classification\",\n" +
         "          \"inference_config\": {\"classification\": {}},\n" +
-        "          \"field_mappings\": {\n" +
+        "          \"field_map\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
@@ -586,7 +586,7 @@ protected NamedXContentRegistry xContentRegistry() {
         "          \"model_id\": \"test_regression\",\n" +
         "          \"tag\": \"regression\",\n" +
         "          \"inference_config\": {\"regression\": {}},\n" +
-        "          \"field_mappings\": {\n" +
+        "          \"field_map\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index d2a2c7896f86b..2a4861c5247f7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -63,6 +64,7 @@ public class InferenceProcessor extends AbstractProcessor {
     public static final String INFERENCE_CONFIG = "inference_config";
     public static final String TARGET_FIELD = "target_field";
     public static final String FIELD_MAPPINGS = "field_mappings";
+    public static final String FIELD_MAP = "field_map";
     private static final String DEFAULT_TARGET_FIELD = "ml.inference";
 
     private final Client client;
@@ -70,7 +72,7 @@ public class InferenceProcessor extends AbstractProcessor {
 
     private final String targetField;
     private final InferenceConfig inferenceConfig;
-    private final Map<String, String> fieldMapping;
+    private final Map<String, String> fieldMap;
     private final InferenceAuditor auditor;
     private volatile boolean previouslyLicensed;
     private final AtomicBoolean shouldAudit = new AtomicBoolean(true);
@@ -81,14 +83,14 @@ public InferenceProcessor(Client client,
                               String targetField,
                               String modelId,
                               InferenceConfig inferenceConfig,
-                              Map<String, String> fieldMapping) {
+                              Map<String, String> fieldMap) {
         super(tag);
         this.client = ExceptionsHelper.requireNonNull(client, "client");
         this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD);
         this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
         this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
         this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG);
-        this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS);
+        this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP);
     }
 
     public String getModelId() {
@@ -126,7 +128,7 @@ void handleResponse(InternalInferModelAction.Response response,
 
     InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) {
         Map<String, Object> fields = new HashMap<>(ingestDocument.getSourceAndMetadata());
-        Model.mapFieldsIfNecessary(fields, fieldMapping);
+        Model.mapFieldsIfNecessary(fields, fieldMap);
         return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
     }
 
@@ -235,7 +237,14 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
             // If multiple inference processors are in the same pipeline, it is wise to tag them
             // The tag will keep default value entries from stepping on each other
             String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField);
-            Map<String, String> fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
+            Map<String, String> fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP);
+            if (fieldMap == null) {
+                fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
+                //TODO Remove in 8.x
+                if (fieldMap != null) {
+                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
+                }
+            }
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
 
             return new InferenceProcessor(client,
@@ -244,7 +253,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
                 targetField,
                 modelId,
                 inferenceConfig,
-                fieldMapping);
+                fieldMap);
         }
 
         // Package private for testing
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
index 0925d252e2019..dfbc4be8e8265 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
@@ -484,7 +484,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() throws Excep
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"modelprocessorlicensetest\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }]}\n";
         // Creating a pipeline should work
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
index 03a011c717292..d776ffd9818d9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
@@ -267,7 +267,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
                     new HashMap<String, Object>() {{
                         put(InferenceProcessor.MODEL_ID, modelId);
                         put("inference_config", Collections.singletonMap("regression", Collections.emptyMap()));
-                        put("field_mappings", Collections.emptyMap());
+                        put("field_map", Collections.emptyMap());
                         put("target_field", randomAlphaOfLength(10));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId + num,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
index 9ed05e56c2c30..cb0519d845b19 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
@@ -131,7 +131,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             ingestService);
 
         Map<String, Object> config = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap()));
@@ -143,7 +143,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]"));
 
         Map<String, Object> config2 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom"));
@@ -154,7 +154,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
             equalTo("inference_config must be an object with one inference type mapped to an object."));
 
         Map<String, Object> config3 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap());
@@ -173,7 +173,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
         processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1"));
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -191,7 +191,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -216,7 +216,7 @@ public void testCreateProcessor() {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -230,7 +230,7 @@ public void testCreateProcessor() {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -251,7 +251,7 @@ public void testCreateProcessorWithDuplicateFields() {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "ml");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(),
@@ -305,7 +305,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
                         put(InferenceProcessor.INFERENCE_CONFIG,
                                 Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap()));
                         put(InferenceProcessor.TARGET_FIELD, "new_field");
-                        put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest"));
+                        put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON);
         }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
index cb5ccd90e1311..8c1f1e581de5b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
@@ -313,7 +313,7 @@ setup:
                   "model_id" : "a-regression-model-0",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
new file mode 100644
index 0000000000000..9c043eee3cd2c
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml
@@ -0,0 +1,114 @@
+# These tests are not part of the ml-with-security tests
+# These are more like ingest processor tests than actual ML focused tests.
+setup:
+  - skip:
+      features: headers
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      ml.put_trained_model:
+        model_id: a-perfect-regression-model
+        body: >
+          {
+            "description": "empty model for tests",
+            "tags": ["regression", "tag1"],
+            "input": {"field_names": ["field1", "field2"]},
+            "definition": {
+               "preprocessors": [],
+               "trained_model": {
+                  "tree": {
+                     "feature_names": ["field1", "field2"],
+                     "tree_structure": [
+                        {"node_index": 0, "leaf_value": 42}
+                     ],
+                     "target_type": "regression"
+                  }
+               }
+            }
+          }
+
+---
+"Test create and delete pipeline with inference processor":
+  - do:
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "inference_config": {"regression": {}},
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+  - do:
+      ingest.delete_pipeline:
+        id: "regression-model-pipeline"
+---
+"Test create processor with missing mandatory fields":
+  - do:
+      catch: /\[model_id\] required property is missing/
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "inference_config": {"regression": {}},
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+  - do:
+      catch: /\[inference_config\] required property is missing/
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+---
+"Test create processor with deprecated fields":
+  - skip:
+      features:
+       - "warnings"
+       - "allowed_warnings"
+  - do:
+      warnings:
+        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "inference_config": {"regression": {}},
+                  "field_mappings": {}
+                }
+              }
+            ]
+          }
+
+  - do:
+      allowed_warnings:
+        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
+      ingest.delete_pipeline:
+        id: "regression-model-pipeline"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
index 5143a690b018a..40d0f6ba01ab7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml
@@ -66,7 +66,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]
@@ -84,7 +84,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]

From 77ec60baa0717bcac7db029ea1647baf8525a9d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Thu, 12 Mar 2020 18:00:36 +0100
Subject: [PATCH 005/285] [DOCS] Adds a warning about reindexing docs with the
 same ID to the PUT DFA docs. (#53490)

---
 docs/reference/ml/ml-shared.asciidoc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index 3439ea5d06da9..ecc5e513e42b0 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -1317,6 +1317,11 @@ The configuration of how to source the analysis data. It requires an
   (Required, string or array) Index or indices on which to perform the 
   analysis. It can be a single index or index pattern as well as an array of 
   indices or patterns.
++
+--
+WARNING: If your source indices contain documents with the same IDs, only the 
+document that is indexed last appears in the destination index.
+--
   
 `query`:::
   (Optional, object) The {es} query domain-specific language 

From 5ccb3b675ed28d45f1140b0ae04d583f4831a976 Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 12 Mar 2020 18:09:15 +0100
Subject: [PATCH 006/285] Ignore isJoda flag from 7x nodes (#53481)

when upgrading from 7.7+ ES will send out a flag indicating if a pattern is of joda style. This is only used to support joda style indices in 7.x, in 8 we no longer support this. All indices in 8 should use java style pattern. Hence we can ignore this flag. Similarly when writing from v8 to v7.7+ we should always send false flag.

relates #52555
relates #53478
closes #53477
---
 build.gradle                                      |  4 ++--
 .../org/elasticsearch/search/DocValueFormat.java  | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index a263aa2dab69c..2dbf94ae7dabb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/issues/53477" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
index b09c8983c1394..7f681bfa47c92 100644
--- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -201,6 +202,13 @@ public DateTime(StreamInput in) throws IOException {
             String zoneId = in.readString();
             this.timeZone = ZoneId.of(zoneId);
             this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt());
+            if (in.getVersion().onOrAfter(Version.V_7_7_0) && in.getVersion().before(Version.V_8_0_0)) {
+                /* when deserialising from 7.7+ nodes expect a flag indicating if a pattern is of joda style
+                   This is only used to support joda style indices in 7.x, in 8 we no longer support this.
+                   All indices in 8 should use java style pattern. Hence we can ignore this flag.
+                */
+                in.readBoolean();
+            }
         }
 
         @Override
@@ -213,6 +221,13 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(formatter.pattern());
             out.writeString(timeZone.getId());
             out.writeVInt(resolution.ordinal());
+            if (out.getVersion().onOrAfter(Version.V_7_7_0) && out.getVersion().before(Version.V_8_0_0)) {
+                /* when serializing to 7.7+  send out a flag indicating if a pattern is of joda style
+                   This is only used to support joda style indices in 7.x, in 8 we no longer support this.
+                   All indices in 8 should use java style pattern. Hence this flag is always false.
+                */
+                out.writeBoolean(false);
+            }
         }
 
         @Override

From c27eab72bbba23163aed2bdff7287f912084930f Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 12 Mar 2020 10:10:40 -0700
Subject: [PATCH 007/285] Remove Java 13 from runtime testing matrix

With Java 14 soon to be GA 13 will be EOL so there's no need to continue
testing on it.

Signed-off-by: Mark Vieira <portugee@gmail.com>
---
 .ci/matrix-runtime-javas.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
index 4a0065b27afd4..b8c543cea0b08 100644
--- a/.ci/matrix-runtime-javas.yml
+++ b/.ci/matrix-runtime-javas.yml
@@ -7,7 +7,6 @@
 
 ES_RUNTIME_JAVA:
   - java11
-  - openjdk13
   - openjdk14
   - openjdk15
   - zulu11

From facd525b0a30daa00794201db7af3883d903d6a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Thu, 12 Mar 2020 19:11:21 +0100
Subject: [PATCH 008/285] Mask wildcard query special characters on keyword
 queries (#53127)

Wildcard queries on keyword fields get normalized, however this normalization
step should exclude the two special characters * and ? in order to keep the
wildcard query itself intact.

Closes #46300
---
 .../index/mapper/KeywordFieldMapper.java      |   1 +
 .../index/mapper/StringFieldType.java         |  44 +++++--
 .../index/mapper/TypeFieldMapper.java         |  61 +--------
 .../index/mapper/TypeFieldTypeTests.java      |  37 +-----
 .../index/query/RangeQueryBuilderTests.java   |   7 --
 .../query/WildcardQueryBuilderTests.java      |  11 +-
 .../search/query/SearchQueryIT.java           | 118 +++++++++++++++++-
 7 files changed, 165 insertions(+), 114 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 7fde4359742e3..524a1eb79f133 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -204,6 +204,7 @@ protected KeywordFieldType(KeywordFieldType ref) {
             this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
         }
 
+        @Override
         public KeywordFieldType clone() {
             return new KeywordFieldType(this);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 4ddda3df0af2d..05bf6b61d1de1 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -21,8 +21,6 @@
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
@@ -31,6 +29,7 @@
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -38,6 +37,8 @@
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 
@@ -47,6 +48,8 @@
  * can be implemented. */
 public abstract class StringFieldType extends TermBasedFieldType {
 
+    private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)");
+
     public StringFieldType() {}
 
     protected StringFieldType(MappedFieldType ref) {
@@ -92,16 +95,41 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
 
     @Override
     public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        Query termQuery = termQuery(value, context);
-        if (termQuery instanceof MatchNoDocsQuery || termQuery instanceof MatchAllDocsQuery) {
-            return termQuery;
-        }
-
+        failIfNotIndexed();
         if (context.allowExpensiveQueries() == false) {
             throw new ElasticsearchException("[wildcard] queries cannot be executed when '" +
                     ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false.");
         }
-        Term term = MappedFieldType.extractTerm(termQuery);
+
+        Term term;
+        if (searchAnalyzer() != null) {
+            // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
+            // is a char_filter that would otherwise remove them
+            Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
+            BytesRefBuilder sb = new BytesRefBuilder();
+            int last = 0;
+
+            while (wildcardMatcher.find()) {
+                if (wildcardMatcher.start() > 0) {
+                    String chunk = value.substring(last, wildcardMatcher.start());
+
+                    BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
+                    sb.append(normalized);
+                }
+                // append the matched group - without normalizing
+                sb.append(new BytesRef(wildcardMatcher.group()));
+
+                last = wildcardMatcher.end();
+            }
+            if (last < value.length()) {
+                String chunk = value.substring(last);
+                BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
+                sb.append(normalized);
+            }
+            term = new Term(name(), sb.toBytesRef());
+        } else {
+            term = new Term(name(), indexedValueForSearch(value));
+        }
 
         WildcardQuery query = new WildcardQuery(term);
         QueryParsers.setRewriteMethod(query, method);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index e5c33b83f24e5..d32d4fa9d3f11 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -30,13 +30,11 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -90,7 +88,7 @@ public MetadataFieldMapper getDefault(ParserContext context) {
         }
     }
 
-    public static final class TypeFieldType extends StringFieldType {
+    public static final class TypeFieldType extends ConstantFieldType {
 
         TypeFieldType() {
         }
@@ -121,61 +119,8 @@ public ValuesSourceType getValuesSourceType() {
         }
 
         @Override
-        public boolean isSearchable() {
-            return true;
-        }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new MatchAllDocsQuery();
-        }
-
-        @Override
-        public Query termQuery(Object value, QueryShardContext context) {
-            return termsQuery(Arrays.asList(value), context);
-        }
-
-        @Override
-        public Query termsQuery(List<?> values, QueryShardContext context) {
-            DocumentMapper mapper = context.getMapperService().documentMapper();
-            if (mapper == null) {
-                return new MatchNoDocsQuery("No types");
-            }
-            BytesRef indexType = indexedValueForSearch(mapper.type());
-            if (values.stream()
-                    .map(this::indexedValueForSearch)
-                    .anyMatch(indexType::equals)) {
-                if (context.getMapperService().hasNested()) {
-                    // type filters are expected not to match nested docs
-                    return Queries.newNonNestedFilter();
-                } else {
-                    return new MatchAllDocsQuery();
-                }
-            } else {
-                return new MatchNoDocsQuery("Type list does not contain the index type");
-            }
-        }
-
-        @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
-            Query result = new MatchAllDocsQuery();
-            String type = context.getMapperService().documentMapper().type();
-            if (type != null) {
-                BytesRef typeBytes = new BytesRef(type);
-                if (lowerTerm != null) {
-                    int comp = indexedValueForSearch(lowerTerm).compareTo(typeBytes);
-                    if (comp > 0 || (comp == 0 && includeLower == false)) {
-                        result = new MatchNoDocsQuery("[_type] was lexicographically smaller than lower bound of range");
-                    }
-                }
-                if (upperTerm != null) {
-                    int comp = indexedValueForSearch(upperTerm).compareTo(typeBytes);
-                    if (comp < 0 || (comp == 0 && includeUpper == false)) {
-                        result = new MatchNoDocsQuery("[_type] was lexicographically greater than upper bound of range");
-                    }
-                }
-            }
-            return result;
+        protected boolean matches(String pattern, QueryShardContext context) {
+            return pattern.equals(MapperService.SINGLE_MAPPING_NAME);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
index 79a93c04faa1f..0e00bf6677c8f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java
@@ -21,14 +21,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.test.VersionUtils;
 import org.mockito.Mockito;
 
 public class TypeFieldTypeTests extends FieldTypeTestCase {
@@ -39,40 +32,14 @@ protected MappedFieldType createDefaultFieldType() {
 
     public void testTermsQuery() throws Exception {
         QueryShardContext context = Mockito.mock(QueryShardContext.class);
-        Version indexVersionCreated = VersionUtils.randomIndexCompatibleVersion(random());
-        Settings indexSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build();
-        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
-        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
-        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
-        Mockito.when(context.indexVersionCreated()).thenReturn(indexVersionCreated);
-
-        MapperService mapperService = Mockito.mock(MapperService.class);
-        Mockito.when(mapperService.documentMapper()).thenReturn(null);
-        Mockito.when(context.getMapperService()).thenReturn(mapperService);
 
         TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType();
         ft.setName(TypeFieldMapper.NAME);
-        Query query = ft.termQuery("my_type", context);
-        assertEquals(new MatchNoDocsQuery(), query);
 
-        DocumentMapper mapper = Mockito.mock(DocumentMapper.class);
-        Mockito.when(mapper.type()).thenReturn("my_type");
-        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
-        query = ft.termQuery("my_type", context);
+        Query query = ft.termQuery("_doc", context);
         assertEquals(new MatchAllDocsQuery(), query);
 
-        Mockito.when(mapperService.hasNested()).thenReturn(true);
-        query = ft.termQuery("my_type", context);
-        assertEquals(Queries.newNonNestedFilter(), query);
-
-        mapper = Mockito.mock(DocumentMapper.class);
-        Mockito.when(mapper.type()).thenReturn("other_type");
-        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
-        query = ft.termQuery("my_type", context);
+        query = ft.termQuery("other_type", context);
         assertEquals(new MatchNoDocsQuery(), query);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index c9c43a1e90186..852c64fd6fdfd 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -556,13 +556,6 @@ public void testParseRelation() {
         assertEquals(ShapeRelation.INTERSECTS, builder.relation());
     }
 
-    public void testTypeField() throws IOException {
-        RangeQueryBuilder builder = QueryBuilders.rangeQuery("_type")
-            .from("value1");
-        builder.doToQuery(createShardContext());
-        assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
-    }
-
     /**
      * Range queries should generally be cacheable, at least the ones we create randomly.
      * This test makes sure we also test the non-cacheable cases regularly.
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 9482e0bb87d49..3eff0d1f819a5 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -75,7 +76,9 @@ protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query quer
 
             assertThat(wildcardQuery.getField(), equalTo(expectedFieldName));
             assertThat(wildcardQuery.getTerm().field(), equalTo(expectedFieldName));
-            assertThat(wildcardQuery.getTerm().text(), equalTo(queryBuilder.value()));
+            // wildcard queries get normalized
+            String text = wildcardQuery.getTerm().text().toLowerCase(Locale.ROOT);
+            assertThat(text, equalTo(text));
         } else {
             Query expected = new MatchNoDocsQuery("unknown field [" + expectedFieldName + "]");
             assertEquals(expected, query);
@@ -138,14 +141,14 @@ public void testTypeField() throws IOException {
         builder.doToQuery(createShardContext());
         assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
     }
-    
+
     public void testRewriteIndexQueryToMatchNone() throws IOException {
         WildcardQueryBuilder query = new WildcardQueryBuilder("_index", "does_not_exist");
         QueryShardContext queryShardContext = createShardContext();
         QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
-    }   
-    
+    }
+
     public void testRewriteIndexQueryNotMatchNone() throws IOException {
         String fullIndexName = getIndex().getName();
         String firstHalfOfIndexName = fullIndexName.substring(0,fullIndexName.length()/2);
diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 51bb41f238950..404e8f92ec8f4 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search.query;
 
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.join.ScoreMode;
@@ -31,6 +33,7 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.lucene.search.SpanBooleanQueryRewriteWithMaxClause;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
@@ -38,6 +41,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.NormalizingCharFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
@@ -46,11 +52,14 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WildcardQueryBuilder;
 import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -62,16 +71,20 @@
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
+import java.io.Reader;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
 
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -121,7 +134,7 @@ public class SearchQueryIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class);
+        return Arrays.asList(InternalSettingsPlugin.class, MockAnalysisPlugin.class);
     }
 
     @Override
@@ -1763,6 +1776,107 @@ public void testFieldAliasesForMetaFields() throws Exception {
 
    }
 
+   /**
+    * Test that wildcard queries on keyword fields get normalized
+    */
+    public void testWildcardQueryNormalizationOnKeywordField() {
+       assertAcked(prepareCreate("test")
+               .setSettings(Settings.builder()
+                       .put("index.analysis.normalizer.lowercase_normalizer.type", "custom")
+                       .putList("index.analysis.normalizer.lowercase_normalizer.filter", "lowercase")
+                       .build())
+                .setMapping("field1", "type=keyword,normalizer=lowercase_normalizer"));
+       client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
+       refresh();
+
+        {
+            WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
+            SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+            assertHitCount(searchResponse, 1L);
+
+            wildCardQuery = wildcardQuery("field1", "bb*");
+            searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+            assertHitCount(searchResponse, 1L);
+        }
+   }
+
+    /**
+     * Test that wildcard queries on text fields get normalized
+     */
+     public void testWildcardQueryNormalizationOnTextField() {
+        assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder()
+                        .put("index.analysis.analyzer.lowercase_analyzer.type", "custom")
+                        .put("index.analysis.analyzer.lowercase_analyzer.tokenizer", "standard")
+                        .putList("index.analysis.analyzer.lowercase_analyzer.filter", "lowercase")
+                        .build())
+                 .setMapping("field1", "type=text,analyzer=lowercase_analyzer"));
+        client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
+        refresh();
+
+         {
+             WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
+             SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+             assertHitCount(searchResponse, 1L);
+
+             wildCardQuery = wildcardQuery("field1", "bb*");
+             searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+             assertHitCount(searchResponse, 1L);
+         }
+    }
+
+    /**
+     * Reserved characters should be excluded when the normalization is applied for keyword fields.
+     * See https://github.com/elastic/elasticsearch/issues/46300 for details.
+     */
+    public void testWildcardQueryNormalizationKeywordSpecialCharacters() {
+        assertAcked(prepareCreate("test")
+                .setSettings(Settings.builder().put("index.analysis.char_filter.no_wildcard.type", "mock_pattern_replace")
+                        .put("index.analysis.normalizer.no_wildcard.type", "custom")
+                        .put("index.analysis.normalizer.no_wildcard.char_filter", "no_wildcard").build())
+                .setMapping("field", "type=keyword,normalizer=no_wildcard"));
+        client().prepareIndex("test").setId("1").setSource("field", "label-1").get();
+        refresh();
+
+        WildcardQueryBuilder wildCardQuery = wildcardQuery("field", "la*");
+        SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+        assertHitCount(searchResponse, 1L);
+
+        wildCardQuery = wildcardQuery("field", "la*el-?");
+        searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+        assertHitCount(searchResponse, 1L);
+    }
+
+    public static class MockAnalysisPlugin extends Plugin implements AnalysisPlugin {
+
+        @Override
+        public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
+            return singletonMap("mock_pattern_replace", (indexSettings, env, name, settings) -> {
+                class Factory implements NormalizingCharFilterFactory {
+
+                    private final Pattern pattern = Regex.compile("[\\*\\?]", null);
+
+                    @Override
+                    public String name() {
+                        return name;
+                    }
+
+                    @Override
+                    public Reader create(Reader reader) {
+                        return new PatternReplaceCharFilter(pattern, "", reader);
+                    }
+                }
+                return new Factory();
+            });
+        }
+
+        @Override
+        public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+            return singletonMap("keyword", (indexSettings, environment, name, settings) -> TokenizerFactory.newFactory(name,
+                    () -> new MockTokenizer(MockTokenizer.KEYWORD, false)));
+        }
+    }
+
     /**
      * Test correct handling {@link SpanBooleanQueryRewriteWithMaxClause#rewrite(IndexReader, MultiTermQuery)}. That rewrite method is e.g.
      * set for fuzzy queries with "constant_score" rewrite nested inside a `span_multi` query and would cause NPEs due to an unset

From 58340c2dbef564939b59f762eb312404bbba28b8 Mon Sep 17 00:00:00 2001
From: Tom Veasey <tveasey@users.noreply.github.com>
Date: Thu, 12 Mar 2020 18:39:29 +0000
Subject: [PATCH 009/285] [ML] Adds the class_assignment_objective parameter to
 classification (#52763)

Adds a new parameter for classification that enables choosing whether to assign labels to
maximise accuracy or to maximise the minimum class recall.

Fixes #52427.
---
 .../client/ml/dataframe/Classification.java   | 51 ++++++++++++++++--
 .../client/MachineLearningIT.java             |  2 +
 .../MlClientDocumentationIT.java              |  6 ++-
 .../ml/dataframe/ClassificationTests.java     |  1 +
 .../ml/put-data-frame-analytics.asciidoc      |  3 +-
 .../apis/put-dfanalytics.asciidoc             |  4 ++
 docs/reference/ml/ml-shared.asciidoc          |  8 +++
 .../ml/dataframe/analyses/Classification.java | 53 ++++++++++++++++--
 .../ml/job/results/ReservedFieldNames.java    |  1 +
 .../xpack/core/ml/config_index_mappings.json  |  3 ++
 .../DataFrameAnalyticsConfigTests.java        |  2 +
 .../analyses/ClassificationTests.java         | 54 +++++++++++++------
 .../ml/integration/ClassificationIT.java      |  7 +--
 .../test/ml/data_frame_analytics_crud.yml     |  3 ++
 .../90_ml_data_frame_analytics_crud.yml       | 25 +++++++++
 .../90_ml_data_frame_analytics_crud.yml       | 33 ++++++++++++
 .../90_ml_data_frame_analytics_crud.yml       | 26 +++++++++
 17 files changed, 250 insertions(+), 32 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
index 6fc533c0f722e..48c75e6e34e6d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java
@@ -22,10 +22,12 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Objects;
 
 public class Classification implements DataFrameAnalysis {
@@ -49,6 +51,7 @@ public static Builder builder(String dependentVariable) {
     static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
     static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
+    static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
@@ -67,7 +70,8 @@ public static Builder builder(String dependentVariable) {
                 (String) a[7],
                 (Double) a[8],
                 (Integer) a[9],
-                (Long) a[10]));
+                (Long) a[10],
+                (ClassAssignmentObjective) a[11]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
@@ -81,6 +85,12 @@ public static Builder builder(String dependentVariable) {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_CLASSES);
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ClassAssignmentObjective.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
     }
 
     private final String dependentVariable;
@@ -92,13 +102,15 @@ public static Builder builder(String dependentVariable) {
     private final Integer numTopFeatureImportanceValues;
     private final String predictionFieldName;
     private final Double trainingPercent;
+    private final ClassAssignmentObjective classAssignmentObjective;
     private final Integer numTopClasses;
     private final Long randomizeSeed;
 
     private Classification(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
                            @Nullable Integer maxTrees, @Nullable Double featureBagFraction,
                            @Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
-                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed) {
+                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed,
+                           @Nullable ClassAssignmentObjective classAssignmentObjective) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
@@ -108,6 +120,7 @@ private Classification(String dependentVariable, @Nullable Double lambda, @Nulla
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
         this.predictionFieldName = predictionFieldName;
         this.trainingPercent = trainingPercent;
+        this.classAssignmentObjective = classAssignmentObjective;
         this.numTopClasses = numTopClasses;
         this.randomizeSeed = randomizeSeed;
     }
@@ -157,6 +170,10 @@ public Long getRandomizeSeed() {
         return randomizeSeed;
     }
 
+    public ClassAssignmentObjective getClassAssignmentObjective() {
+        return classAssignmentObjective;
+    }
+
     public Integer getNumTopClasses() {
         return numTopClasses;
     }
@@ -192,6 +209,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (randomizeSeed != null) {
             builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
         }
+        if (classAssignmentObjective != null) {
+            builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
+        }
         if (numTopClasses != null) {
             builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         }
@@ -202,7 +222,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public int hashCode() {
         return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
-            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses);
+            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses, classAssignmentObjective);
     }
 
     @Override
@@ -220,7 +240,8 @@ public boolean equals(Object o) {
             && Objects.equals(predictionFieldName, that.predictionFieldName)
             && Objects.equals(trainingPercent, that.trainingPercent)
             && Objects.equals(randomizeSeed, that.randomizeSeed)
-            && Objects.equals(numTopClasses, that.numTopClasses);
+            && Objects.equals(numTopClasses, that.numTopClasses)
+            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective);
     }
 
     @Override
@@ -228,6 +249,19 @@ public String toString() {
         return Strings.toString(this);
     }
 
+    public enum ClassAssignmentObjective {
+        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
+
+        public static ClassAssignmentObjective fromString(String value) {
+            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
     public static class Builder {
         private String dependentVariable;
         private Double lambda;
@@ -240,6 +274,7 @@ public static class Builder {
         private Double trainingPercent;
         private Integer numTopClasses;
         private Long randomizeSeed;
+        private ClassAssignmentObjective classAssignmentObjective;
 
         private Builder(String dependentVariable) {
             this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -295,9 +330,15 @@ public Builder setNumTopClasses(Integer numTopClasses) {
             return this;
         }
 
+        public Builder setClassAssignmentObjective(ClassAssignmentObjective classAssignmentObjective) {
+            this.classAssignmentObjective = classAssignmentObjective;
+            return this;
+        }
+
         public Classification build() {
             return new Classification(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
-                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed);
+                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed,
+                classAssignmentObjective);
         }
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index e13464aeddabd..17a9c0cd1cfca 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -1336,6 +1336,8 @@ public void testPutDataFrameAnalyticsConfig_GivenClassification() throws Excepti
                 .setPredictionFieldName("my_dependent_variable_prediction")
                 .setTrainingPercent(80.0)
                 .setRandomizeSeed(42L)
+                .setClassAssignmentObjective(
+                    org.elasticsearch.client.ml.dataframe.Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY)
                 .setNumTopClasses(1)
                 .setLambda(1.0)
                 .setGamma(1.0)
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index 74e34889897e3..adf78daacc7d5 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -139,6 +139,7 @@
 import org.elasticsearch.client.ml.datafeed.DatafeedStats;
 import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig;
+import org.elasticsearch.client.ml.dataframe.Classification;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
@@ -2969,7 +2970,7 @@ public void testPutDataFrameAnalytics() throws Exception {
             // end::put-data-frame-analytics-outlier-detection-customized
 
             // tag::put-data-frame-analytics-classification
-            DataFrameAnalysis classification = org.elasticsearch.client.ml.dataframe.Classification.builder("my_dependent_variable") // <1>
+            DataFrameAnalysis classification = Classification.builder("my_dependent_variable") // <1>
                 .setLambda(1.0) // <2>
                 .setGamma(5.5) // <3>
                 .setEta(5.5) // <4>
@@ -2979,7 +2980,8 @@ public void testPutDataFrameAnalytics() throws Exception {
                 .setPredictionFieldName("my_prediction_field_name") // <8>
                 .setTrainingPercent(50.0) // <9>
                 .setRandomizeSeed(1234L) // <10>
-                .setNumTopClasses(1) // <11>
+                .setClassAssignmentObjective(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY) // <11>
+                .setNumTopClasses(1) // <12>
                 .build();
             // end::put-data-frame-analytics-classification
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
index 906003e200975..0970222c513b9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java
@@ -36,6 +36,7 @@ public static Classification randomClassification() {
             .setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
             .setTrainingPercent(randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true))
             .setRandomizeSeed(randomBoolean() ? null : randomLong())
+            .setClassAssignmentObjective(randomBoolean() ? null : randomFrom(Classification.ClassAssignmentObjective.values()))
             .setNumTopClasses(randomBoolean() ? null : randomIntBetween(0, 10))
             .build();
     }
diff --git a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
index 4be2011340210..cf88d65ae9314 100644
--- a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
+++ b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
@@ -121,7 +121,8 @@ include-tagged::{doc-tests-file}[{api}-classification]
 <8> The name of the prediction field in the results object.
 <9> The percentage of training-eligible rows to be used in training. Defaults to 100%.
 <10> The seed to be used by the random generator that picks which rows are used in training.
-<11> The number of top classes to be reported in the results. Defaults to 2.
+<11> The optimization objective to target when assigning class labels. Defaults to maximize_minimum_recall.
+<12> The number of top classes to be reported in the results. Defaults to 2.
 
 ===== Regression
 
diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
index 2a3822b0ee888..6a011dcb04f48 100644
--- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
@@ -136,6 +136,10 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=gamma]
 (Optional, double) 
 include::{docdir}/ml/ml-shared.asciidoc[tag=lambda]
 
+`analysis`.`classification`.`class_assignment_objective`::::
+(Optional, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=class-assignment-objective]
+
 `analysis`.`classification`.`num_top_classes`::::
 (Optional, integer)
 include::{docdir}/ml/ml-shared.asciidoc[tag=num-top-classes]
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index ecc5e513e42b0..16c9f486d8d63 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -339,6 +339,14 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=mode]
 include::{docdir}/ml/ml-shared.asciidoc[tag=time-span]
 end::chunking-config[]
 
+tag::class-assignment-objective[]
+Defines the objective to optimize when assigning class labels. Available
+objectives are `maximize_accuracy` and `maximize_minimum_recall`. When maximizing
+accuracy class labels are chosen to maximize the number of correct predictions.
+When maximizing minimum recall labels are chosen to maximize the minimum recall
+for any class. Defaults to maximize_minimum_recall.
+end::class-assignment-objective[]
+
 tag::custom-rules[]
 An array of custom rule objects, which enable you to customize the way detectors
 operate. For example, a rule may dictate to the detector conditions under which
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index d9289a5bcf1ac..0f365ad671b76 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.FieldAliasMapper;
@@ -21,6 +22,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -37,6 +39,7 @@ public class Classification implements DataFrameAnalysis {
 
     public static final ParseField DEPENDENT_VARIABLE = new ParseField("dependent_variable");
     public static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
+    public static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     public static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     public static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
     public static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
@@ -54,12 +57,19 @@ private static ConstructingObjectParser<Classification, Void> createParser(boole
                 (String) a[0],
                 new BoostedTreeParams((Double) a[1], (Double) a[2], (Double) a[3], (Integer) a[4], (Double) a[5], (Integer) a[6]),
                 (String) a[7],
-                (Integer) a[8],
-                (Double) a[9],
-                (Long) a[10]));
+                (ClassAssignmentObjective) a[8],
+                (Integer) a[9],
+                (Double) a[10],
+                (Long) a[11]));
         parser.declareString(constructorArg(), DEPENDENT_VARIABLE);
         BoostedTreeParams.declareFields(parser);
         parser.declareString(optionalConstructorArg(), PREDICTION_FIELD_NAME);
+        parser.declareField(optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ClassAssignmentObjective.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
         parser.declareInt(optionalConstructorArg(), NUM_TOP_CLASSES);
         parser.declareDouble(optionalConstructorArg(), TRAINING_PERCENT);
         parser.declareLong(optionalConstructorArg(), RANDOMIZE_SEED);
@@ -89,6 +99,7 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
     private final String dependentVariable;
     private final BoostedTreeParams boostedTreeParams;
     private final String predictionFieldName;
+    private final ClassAssignmentObjective classAssignmentObjective;
     private final int numTopClasses;
     private final double trainingPercent;
     private final long randomizeSeed;
@@ -96,6 +107,7 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
     public Classification(String dependentVariable,
                           BoostedTreeParams boostedTreeParams,
                           @Nullable String predictionFieldName,
+                          @Nullable ClassAssignmentObjective classAssignmentObjective,
                           @Nullable Integer numTopClasses,
                           @Nullable Double trainingPercent,
                           @Nullable Long randomizeSeed) {
@@ -108,19 +120,26 @@ public Classification(String dependentVariable,
         this.dependentVariable = ExceptionsHelper.requireNonNull(dependentVariable, DEPENDENT_VARIABLE);
         this.boostedTreeParams = ExceptionsHelper.requireNonNull(boostedTreeParams, BoostedTreeParams.NAME);
         this.predictionFieldName = predictionFieldName == null ? dependentVariable + "_prediction" : predictionFieldName;
+        this.classAssignmentObjective = classAssignmentObjective == null ?
+            ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL : classAssignmentObjective;
         this.numTopClasses = numTopClasses == null ? DEFAULT_NUM_TOP_CLASSES : numTopClasses;
         this.trainingPercent = trainingPercent == null ? 100.0 : trainingPercent;
         this.randomizeSeed = randomizeSeed == null ? Randomness.get().nextLong() : randomizeSeed;
     }
 
     public Classification(String dependentVariable) {
-        this(dependentVariable, BoostedTreeParams.builder().build(), null, null, null, null);
+        this(dependentVariable, BoostedTreeParams.builder().build(), null, null, null, null, null);
     }
 
     public Classification(StreamInput in) throws IOException {
         dependentVariable = in.readString();
         boostedTreeParams = new BoostedTreeParams(in);
         predictionFieldName = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            classAssignmentObjective = in.readEnum(ClassAssignmentObjective.class);
+        } else {
+            classAssignmentObjective = ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL;
+        }
         numTopClasses = in.readOptionalVInt();
         trainingPercent = in.readDouble();
         if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -142,6 +161,10 @@ public String getPredictionFieldName() {
         return predictionFieldName;
     }
 
+    public ClassAssignmentObjective getClassAssignmentObjective() {
+        return classAssignmentObjective;
+    }
+
     public int getNumTopClasses() {
         return numTopClasses;
     }
@@ -164,6 +187,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(dependentVariable);
         boostedTreeParams.writeTo(out);
         out.writeOptionalString(predictionFieldName);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeEnum(classAssignmentObjective);
+        }
         out.writeOptionalVInt(numTopClasses);
         out.writeDouble(trainingPercent);
         if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -178,6 +204,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         boostedTreeParams.toXContent(builder, params);
+        builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
         builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         if (predictionFieldName != null) {
             builder.field(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
@@ -195,6 +222,7 @@ public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
+        params.put(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
         params.put(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         if (predictionFieldName != null) {
             params.put(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
@@ -303,6 +331,7 @@ public boolean equals(Object o) {
         return Objects.equals(dependentVariable, that.dependentVariable)
             && Objects.equals(boostedTreeParams, that.boostedTreeParams)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
+            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective)
             && Objects.equals(numTopClasses, that.numTopClasses)
             && trainingPercent == that.trainingPercent
             && randomizeSeed == that.randomizeSeed;
@@ -310,6 +339,20 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(dependentVariable, boostedTreeParams, predictionFieldName, numTopClasses, trainingPercent, randomizeSeed);
+        return Objects.hash(dependentVariable, boostedTreeParams, predictionFieldName, classAssignmentObjective,
+                            numTopClasses, trainingPercent, randomizeSeed);
+    }
+
+    public enum ClassAssignmentObjective {
+        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
+
+        public static ClassAssignmentObjective fromString(String value) {
+            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
index b4a8054cca85a..6a06608638ff9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
@@ -317,6 +317,7 @@ public final class ReservedFieldNames {
             Classification.NAME.getPreferredName(),
             Classification.DEPENDENT_VARIABLE.getPreferredName(),
             Classification.PREDICTION_FIELD_NAME.getPreferredName(),
+            Classification.CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(),
             Classification.NUM_TOP_CLASSES.getPreferredName(),
             Classification.TRAINING_PERCENT.getPreferredName(),
             BoostedTreeParams.LAMBDA.getPreferredName(),
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
index 5ee5957e49331..fe0c559b12245 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json
@@ -43,6 +43,9 @@
               "max_trees" : {
                 "type" : "integer"
               },
+              "class_assignment_objective" : {
+                "type" : "keyword"
+              },
               "num_top_classes" : {
                 "type" : "integer"
               },
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
index 7dd0f6f9b0ca9..1c733e04504c8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
@@ -155,12 +155,14 @@ protected void assertOnBWCObject(DataFrameAnalyticsConfig bwcSerializedObject, D
             bwcAnalysis = new Classification(bwcClassification.getDependentVariable(),
                 bwcClassification.getBoostedTreeParams(),
                 bwcClassification.getPredictionFieldName(),
+                bwcClassification.getClassAssignmentObjective(),
                 bwcClassification.getNumTopClasses(),
                 bwcClassification.getTrainingPercent(),
                 42L);
             testAnalysis = new Classification(testClassification.getDependentVariable(),
                 testClassification.getBoostedTreeParams(),
                 testClassification.getPredictionFieldName(),
+                testClassification.getClassAssignmentObjective(),
                 testClassification.getNumTopClasses(),
                 testClassification.getTrainingPercent(),
                 42L);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 0898a3fb476e8..e7b2dbbc09f95 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -54,17 +54,20 @@ public static Classification createRandom() {
         String dependentVariableName = randomAlphaOfLength(10);
         BoostedTreeParams boostedTreeParams = BoostedTreeParamsTests.createRandom();
         String predictionFieldName = randomBoolean() ? null : randomAlphaOfLength(10);
+        Classification.ClassAssignmentObjective classAssignmentObjective = randomBoolean() ?
+            null : randomFrom(Classification.ClassAssignmentObjective.values());
         Integer numTopClasses = randomBoolean() ? null : randomIntBetween(0, 1000);
         Double trainingPercent = randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true);
         Long randomizeSeed = randomBoolean() ? null : randomLong();
-        return new Classification(dependentVariableName, boostedTreeParams, predictionFieldName, numTopClasses, trainingPercent,
-            randomizeSeed);
+        return new Classification(dependentVariableName, boostedTreeParams, predictionFieldName, classAssignmentObjective,
+            numTopClasses, trainingPercent, randomizeSeed);
     }
 
     public static Classification mutateForVersion(Classification instance, Version version) {
         return new Classification(instance.getDependentVariable(),
             BoostedTreeParamsTests.mutateForVersion(instance.getBoostedTreeParams(), version),
             instance.getPredictionFieldName(),
+            version.onOrAfter(Version.V_8_0_0) ? instance.getClassAssignmentObjective() : null,
             instance.getNumTopClasses(),
             instance.getTrainingPercent(),
             instance.getRandomizeSeed());
@@ -80,12 +83,14 @@ protected void assertOnBWCObject(Classification bwcSerializedObject, Classificat
         Classification newBwc = new Classification(bwcSerializedObject.getDependentVariable(),
             bwcSerializedObject.getBoostedTreeParams(),
             bwcSerializedObject.getPredictionFieldName(),
+            bwcSerializedObject.getClassAssignmentObjective(),
             bwcSerializedObject.getNumTopClasses(),
             bwcSerializedObject.getTrainingPercent(),
             42L);
         Classification newInstance = new Classification(testInstance.getDependentVariable(),
             testInstance.getBoostedTreeParams(),
             testInstance.getPredictionFieldName(),
+            testInstance.getClassAssignmentObjective(),
             testInstance.getNumTopClasses(),
             testInstance.getTrainingPercent(),
             42L);
@@ -99,71 +104,85 @@ protected Writeable.Reader<Classification> instanceReader() {
 
     public void testConstructor_GivenTrainingPercentIsLessThanOne() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 0.999, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 0.999, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[training_percent] must be a double in [1, 100]"));
     }
 
     public void testConstructor_GivenTrainingPercentIsGreaterThan100() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 100.0001, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 100.0001, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[training_percent] must be a double in [1, 100]"));
     }
 
     public void testConstructor_GivenNumTopClassesIsLessThanZero() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", -1, 1.0, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, -1, 1.0, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[num_top_classes] must be an integer in [0, 1000]"));
     }
 
     public void testConstructor_GivenNumTopClassesIsGreaterThan1000() {
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", 1001, 1.0, randomLong()));
+            () -> new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1001, 1.0, randomLong()));
 
         assertThat(e.getMessage(), equalTo("[num_top_classes] must be an integer in [0, 1000]"));
     }
 
     public void testGetPredictionFieldName() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 50.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 50.0, randomLong());
         assertThat(classification.getPredictionFieldName(), equalTo("result"));
 
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, null, 3, 50.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, null, null, 3, 50.0, randomLong());
         assertThat(classification.getPredictionFieldName(), equalTo("foo_prediction"));
     }
 
+    public void testClassAssignmentObjective() {
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result",
+            Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY, 7, 1.0, randomLong());
+        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY));
+
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result",
+        Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL, 7, 1.0, randomLong());
+        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL));
+
+        // class_assignment_objective == null, default applied
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 7, 1.0, randomLong());
+        assertThat(classification.getClassAssignmentObjective(), equalTo(Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL));
+    }
+
     public void testGetNumTopClasses() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 7, 1.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 7, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(7));
 
         // Boundary condition: num_top_classes == 0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 0, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 0, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(0));
 
         // Boundary condition: num_top_classes == 1000
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 1000, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1000, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(1000));
 
         // num_top_classes == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, null, 1.0, randomLong());
         assertThat(classification.getNumTopClasses(), equalTo(2));
     }
 
     public void testGetTrainingPercent() {
-        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 50.0, randomLong());
+        Classification classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 50.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(50.0));
 
         // Boundary condition: training_percent == 1.0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 1.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 1.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(1.0));
 
         // Boundary condition: training_percent == 100.0
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, 100.0, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, 100.0, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(100.0));
 
         // training_percent == null, default applied
-        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", 3, null, randomLong());
+        classification = new Classification("foo", BOOSTED_TREE_PARAMS, "result", null, 3, null, randomLong());
         assertThat(classification.getTrainingPercent(), equalTo(100.0));
     }
 
@@ -178,6 +197,7 @@ public void testGetParams() {
             equalTo(
                 Map.of(
                     "dependent_variable", "foo",
+                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "foo_prediction",
                     "prediction_field_type", "bool")));
@@ -186,6 +206,7 @@ public void testGetParams() {
             equalTo(
                 Map.of(
                     "dependent_variable", "bar",
+                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "bar_prediction",
                     "prediction_field_type", "int")));
@@ -194,6 +215,7 @@ public void testGetParams() {
             equalTo(
                 Map.of(
                     "dependent_variable", "baz",
+                    "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "baz_prediction",
                     "prediction_field_type", "string")));
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index 4db035f9b5f70..d2f90eed635ea 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -88,6 +88,7 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
                 null,
                 null,
                 null,
+                null,
                 null));
         registerAnalytics(config);
         putAnalytics(config);
@@ -189,7 +190,7 @@ public <T> void testWithOnlyTrainingRowsAndTrainingPercentIsFifty(String jobId,
                 sourceIndex,
                 destIndex,
                 null,
-                new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, numTopClasses, 50.0, null));
+                new Classification(dependentVariable, BoostedTreeParams.builder().build(), null, null, numTopClasses, 50.0, null));
         registerAnalytics(config);
         putAnalytics(config);
 
@@ -438,7 +439,7 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
             .build();
 
         DataFrameAnalyticsConfig firstJob = buildAnalytics(firstJobId, sourceIndex, firstJobDestIndex, null,
-            new Classification(dependentVariable, boostedTreeParams, null, 1, 50.0, null));
+            new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, null));
         registerAnalytics(firstJob);
         putAnalytics(firstJob);
 
@@ -447,7 +448,7 @@ public void testTwoJobsWithSameRandomizeSeedUseSameTrainingSet() throws Exceptio
 
         long randomizeSeed = ((Classification) firstJob.getAnalysis()).getRandomizeSeed();
         DataFrameAnalyticsConfig secondJob = buildAnalytics(secondJobId, sourceIndex, secondJobDestIndex, null,
-            new Classification(dependentVariable, boostedTreeParams, null, 1, 50.0, randomizeSeed));
+            new Classification(dependentVariable, boostedTreeParams, null, null, 1, 50.0, randomizeSeed));
 
         registerAnalytics(secondJob);
         putAnalytics(secondJob);
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index 8884941ddfa98..a5a99b30391ea 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -1834,6 +1834,7 @@ setup:
                 "eta": 0.5,
                 "max_trees": 400,
                 "feature_bag_fraction": 0.3,
+                "class_assignment_objective": "maximize_accuracy",
                 "training_percent": 60.3,
                 "randomize_seed": 24
               }
@@ -1853,6 +1854,7 @@ setup:
       "prediction_field_name": "foo_prediction",
       "training_percent": 60.3,
       "randomize_seed": 24,
+      "class_assignment_objective": "maximize_accuracy",
       "num_top_classes": 2
     }
   }}
@@ -1896,6 +1898,7 @@ setup:
       "prediction_field_name": "foo_prediction",
       "training_percent": 100.0,
       "randomize_seed": 24,
+      "class_assignment_objective": "maximize_minimum_recall",
       "num_top_classes": 2
     }
   }}
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index af304afc57db6..5049f3360a42e 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -92,6 +92,31 @@
   - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
   - match: { data_frame_analytics.0.state: "stopped" }
 
+---
+"Get old classification job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
+  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
+  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
+
+---
+"Get old classification job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
 ---
 "Put an outlier_detection job on the mixed cluster":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index fe160bba15f23..923a56395e8a4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -19,6 +19,16 @@ setup:
             "user": "Kimchy"
           }
 
+  - do:
+      index:
+        index: bwc_ml_classification_job_source
+        body: >
+          {
+            "numeric_field_1": 1.0,
+            "foo": "a",
+            "user": "Kimchy"
+          }
+
   - do:
       indices.refresh:
         index: bwc_ml_*
@@ -64,3 +74,26 @@ setup:
             }
           }
   - match: { id: "old_cluster_regression_job" }
+
+---
+"Put classification job on the old cluster":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "old_cluster_classification_job"
+        body: >
+          {
+            "source": {
+              "index": "bwc_ml_classification_job_source",
+              "query": {"term" : { "user.keyword" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "old_cluster_classification_job_results"
+            },
+            "analysis": {
+              "classification":{
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "old_cluster_classification_job" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index 14438883f0da1..b197d0c5cf433 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -52,6 +52,32 @@
   - match: { data_frame_analytics.0.id: "old_cluster_regression_job" }
   - match: { data_frame_analytics.0.state: "stopped" }
 
+---
+"Get old classification job":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.source.index: ["bwc_ml_classification_job_source"] }
+  - match: { data_frame_analytics.0.source.query: {"term": { "user.keyword": "Kimchy" }} }
+  - match: { data_frame_analytics.0.dest.index: "old_cluster_classification_job_results" }
+  - match: { data_frame_analytics.0.analysis.classification.dependent_variable: "foo" }
+  - match: { data_frame_analytics.0.analysis.classification.training_percent: 100.0 }
+  - match: { data_frame_analytics.0.analysis.classification.class_assignment_objective: "maximize_minimum_recall" }
+  - is_true: data_frame_analytics.0.analysis.classification.randomize_seed
+
+---
+"Get old classification job stats":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "old_cluster_classification_job"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "old_cluster_classification_job" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
 ---
 "Get mixed cluster outlier_detection job":
   - skip:

From 9ad05976170292d55d8785dc15005208bf3970d9 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 12 Mar 2020 21:17:11 +0100
Subject: [PATCH 010/285] Removes old Lucene's experimental flag from analyzer
 documentations (#53217)

This change removes the Lucene's experimental flag from the documentations of the following
tokenizer/filters:
  * Simple Pattern Split Tokenizer
  * Simple Pattern tokenizer
  * Flatten Graph Token Filter
  * Word Delimiter Graph Token Filter

The flag is still present in Lucene codebase but we're fully supporting these tokenizers/filters
in ES for a long time now so the docs flag is misleading.

Co-authored-by: James Rodewig <james.rodewig@elastic.co>
---
 .../analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc    | 2 --
 .../analysis/tokenizers/simplepattern-tokenizer.asciidoc        | 2 --
 .../analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc   | 2 --
 3 files changed, 6 deletions(-)

diff --git a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
index bcff83c5e9950..d75ce6f003979 100644
--- a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Flatten graph</titleabbrev>
 ++++
 
-experimental[This functionality is marked as experimental in Lucene]
-
 The `flatten_graph` token filter accepts an arbitrary graph token
 stream, such as that produced by
 <<analysis-synonym-graph-tokenfilter>>, and flattens it into a single
diff --git a/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
index 5da001640a027..67c0cefc98957 100644
--- a/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/simplepattern-tokenizer.asciidoc
@@ -1,8 +1,6 @@
 [[analysis-simplepattern-tokenizer]]
 === Simple Pattern Tokenizer
 
-experimental[This functionality is marked as experimental in Lucene]
-
 The `simple_pattern` tokenizer uses a regular expression to capture matching
 text as terms. The set of regular expression features it supports is more
 limited than the <<analysis-pattern-tokenizer,`pattern`>> tokenizer, but the
diff --git a/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
index 55be14c45638a..3f24233334e57 100644
--- a/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/simplepatternsplit-tokenizer.asciidoc
@@ -1,8 +1,6 @@
 [[analysis-simplepatternsplit-tokenizer]]
 === Simple Pattern Split Tokenizer
 
-experimental[This functionality is marked as experimental in Lucene]
-
 The `simple_pattern_split` tokenizer uses a regular expression to split the
 input into terms at pattern matches. The set of regular expression features it
 supports is more limited than the <<analysis-pattern-tokenizer,`pattern`>>

From 88d05d1e33bd05e3d40faa905eedb6e3f0018c7a Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 12 Mar 2020 22:25:04 +0100
Subject: [PATCH 011/285] Fix race condition when deleting an async search
 (#53513)

Deleting an async search id can throw a ResourceNotFoundException even if the query was successfully cancelled.
We delete the stored response automatically if the query is cancelled so that creates a race with the delete action that also ensures that the task is removed. This change ensures that we ignore missing async search ids in the async search index if they were successfully cancelled.

Relates #53360
Relates #49931
---
 .../xpack/search/AsyncSearchIndexService.java             | 3 ++-
 .../xpack/search/TransportDeleteAsyncSearchAction.java    | 8 +++++---
 .../xpack/search/TransportSubmitAsyncSearchAction.java    | 2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
index b48ad768e1207..d1ec9fac7c06f 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
@@ -201,13 +201,14 @@ void updateExpirationTime(String docId,
      * Deletes the provided <code>searchId</code> from the index if present.
      */
     void deleteResponse(AsyncSearchId searchId,
+                        boolean failIfNotFound,
                         ActionListener<AcknowledgedResponse> listener) {
         DeleteRequest request = new DeleteRequest(INDEX).id(searchId.getDocId());
         createIndexIfNecessary(
             ActionListener.wrap(v -> client.delete(request,
                 ActionListener.wrap(
                     resp -> {
-                        if (resp.status() == RestStatus.NOT_FOUND) {
+                        if (resp.status() == RestStatus.NOT_FOUND && failIfNotFound) {
                             listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
                         } else {
                             listener.onResponse(new AcknowledgedResponse(true));
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
index 0fd514a744d96..14bf4eafec517 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java
@@ -61,10 +61,12 @@ protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, Act
     private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener<AcknowledgedResponse> listener) throws IOException {
         AsyncSearchTask task = store.getTask(taskManager, searchId);
         if (task != null) {
-            task.cancelTask(() -> store.deleteResponse(searchId, listener));
+            task.cancelTask(() -> store.deleteResponse(searchId, false, listener));
         } else {
-            // check if the response can be retrieved by the user (handle security) and then delete.
-            store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, listener), listener::onFailure));
+            // the task is not running anymore so we throw a not found exception if
+            // the search id is also not present in the index (already deleted) or if the user
+            // is not allowed to access it.
+            store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, true, listener), listener::onFailure));
         }
     }
 }
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
index 5835f851862d6..a89f166866a9c 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
@@ -167,7 +167,7 @@ private void onFinalResponse(CancellableTask submitTask,
                                  Runnable nextAction) {
         if (submitTask.isCancelled() || searchTask.isCancelled()) {
             // the user cancelled the submit so we ensure that there is nothing stored in the response index.
-            store.deleteResponse(searchTask.getSearchId(), ActionListener.wrap(() -> {
+            store.deleteResponse(searchTask.getSearchId(), false, ActionListener.wrap(() -> {
                 taskManager.unregister(searchTask);
                 nextAction.run();
             }));

From 428ed2a2d0fcc6da5a3fdae157b8e95449cc74ad Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 12 Mar 2020 15:32:55 -0600
Subject: [PATCH 012/285] Disable BWC Tests for ComponentTemplateMetadata
 backport (#53516)

This disables the BWC tests for the ComponentTemplateMetadata class added in #53290 and #53489
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 2dbf94ae7dabb..acf9d48770442 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53515" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From b825f6461f1de4b25a72990c6a21945872ba3cba Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 12 Mar 2020 16:28:29 -0600
Subject: [PATCH 013/285] Update minimal supported version for
 ComponentTemplateMetadata (#53515)

This can be merged once #53489 is merged.
---
 .../cluster/metadata/ComponentTemplateMetadata.java            | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
index 241cdca592cb5..1b1e6b346ef9a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -93,8 +93,7 @@ public String getWriteableName() {
 
     @Override
     public Version getMinimalSupportedVersion() {
-        // TODO: update after backporting
-        return Version.V_8_0_0;
+        return Version.V_7_7_0;
     }
 
     @Override

From 99a4fa354d4a2e729131f96be7ca296c8e8347d3 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 12 Mar 2020 20:14:30 -0400
Subject: [PATCH 014/285] Update jackson-databind to 2.8.11.6 (#53522)

This commit upgrades the jackson-databind depdendency to
2.8.11.6. Additionally, we revert a previous change that put
ingest-geoip on the version of jackson-databind from the version
properties file. This is because upgrading ingest-geoip to a later
version of jackson-databind also requires an upgrade to the geoip2
dependency which is currently blocked. Therefore, if we can get to a
point where we otherwise upgrade our Jackson dependencies, we do not
want ingest-geoip to automatically come along with it.
---
 buildSrc/version.properties                                  | 2 +-
 modules/ingest-geoip/build.gradle                            | 5 +++--
 .../ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1 | 1 -
 .../ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 | 1 +
 .../licenses/jackson-databind-2.8.11.4.jar.sha1              | 1 -
 .../licenses/jackson-databind-2.8.11.6.jar.sha1              | 1 +
 .../licenses/jackson-databind-2.8.11.4.jar.sha1              | 1 -
 .../licenses/jackson-databind-2.8.11.6.jar.sha1              | 1 +
 8 files changed, 7 insertions(+), 6 deletions(-)
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index e9278d0c4a554..dea67de1657d1 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -11,7 +11,7 @@ jts               = 1.15.0
 # you should also inspect that version to see if it can be advanced along with
 # the com.maxmind.geoip2:geoip2 dependency
 jackson           = 2.8.11
-jacksondatabind   = 2.8.11.4
+jacksondatabind   = 2.8.11.6
 snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index ce9dc7d574c12..6789f96d883b8 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -28,8 +28,9 @@ dependencies {
   // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
   compile('com.maxmind.geoip2:geoip2:2.9.0')
   // geoip2 dependencies:
-  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}")
+  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
+  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
+  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
   compile('com.maxmind.db:maxmind-db:1.2.2')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
deleted file mode 100644
index 5203969bcf5c0..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
deleted file mode 100644
index 5203969bcf5c0..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
deleted file mode 100644
index 5203969bcf5c0..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.8.11.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-596d6923ff4cf7ea72ded3ac32903b9c618ce9f1
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file

From 37c739c0483f1b7202ba7325e459d2b91ee07695 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 13 Mar 2020 01:27:26 +0100
Subject: [PATCH 015/285] Fix pre-sorting of shards in the can_match phase
 (#53397)

This commit fixes a bug on sorted queries with a primary sort field
that uses different types in the requested indices. In this scenario
the returned min/max values to sort the shards are not comparable so
we should avoid the sorting rather than throwing an obscure exception.
---
 .../search/CanMatchPreFilterSearchPhase.java  | 15 +++-
 .../elasticsearch/search/sort/MinAndMax.java  |  4 +-
 .../CanMatchPreFilterSearchPhaseTests.java    | 75 +++++++++++++++++++
 3 files changed, 89 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
index f4e6e3e492ed0..70eeb2a811e5b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
@@ -32,11 +32,9 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.transport.Transport;
 
-import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
@@ -128,7 +126,18 @@ private static List<SearchShardIterator> sortShards(GroupShardsIterator<SearchSh
     }
 
     private static boolean shouldSortShards(MinAndMax<?>[] minAndMaxes) {
-        return Arrays.stream(minAndMaxes).anyMatch(Objects::nonNull);
+        Class<?> clazz = null;
+        for (MinAndMax<?> minAndMax : minAndMaxes) {
+            if (clazz == null) {
+                clazz = minAndMax == null ? null : minAndMax.getMin().getClass();
+            } else if (minAndMax != null && clazz != minAndMax.getMin().getClass()) {
+                // we don't support sort values that mix different types (e.g.: long/double, numeric/keyword).
+                // TODO: we could fail the request because there is a high probability
+                //  that the merging of topdocs will fail later for the same reason ?
+                return false;
+            }
+        }
+        return clazz != null;
     }
 
     private static Comparator<Integer> shardComparator(GroupShardsIterator<SearchShardIterator> shardsIts,
diff --git a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
index ab02d6df7993b..703ae6939aa8f 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java
@@ -55,14 +55,14 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Return the minimum value.
      */
-    T getMin() {
+    public T getMin() {
         return minValue;
     }
 
     /**
      * Return the maximum value.
      */
-    T getMax() {
+    public T getMax() {
         return maxValue;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
index fb894d8e44d0b..3d46a3e13a434 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
@@ -54,6 +55,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class CanMatchPreFilterSearchPhaseTests extends ESTestCase {
 
     public void testFilterShards() throws InterruptedException {
@@ -350,4 +353,76 @@ public void run() {
             }
         }
     }
+
+    public void testInvalidSortShards() throws InterruptedException {
+        final TransportSearchAction.SearchTimeProvider timeProvider =
+            new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), System::nanoTime);
+
+        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
+        lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
+        lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
+
+        for (SortOrder order : SortOrder.values()) {
+            int numShards = randomIntBetween(2, 20);
+            List<ShardId> shardIds = new ArrayList<>();
+            Set<ShardId> shardToSkip = new HashSet<>();
+
+            SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+                @Override
+                public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task,
+                                         ActionListener<SearchService.CanMatchResponse> listener) {
+                    final MinAndMax<?> minMax;
+                    if (request.shardId().id() == numShards-1) {
+                        minMax = new MinAndMax<>(new BytesRef("bar"), new BytesRef("baz"));
+                    } else {
+                        Long min = randomLong();
+                        Long max = randomLongBetween(min, Long.MAX_VALUE);
+                        minMax = new MinAndMax<>(min, max);
+                    }
+                    boolean canMatch = frequently();
+                    synchronized (shardIds) {
+                        shardIds.add(request.shardId());
+                        if (canMatch == false) {
+                            shardToSkip.add(request.shardId());
+                        }
+                    }
+                    new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(canMatch, minMax))).start();
+                }
+            };
+
+            AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
+            CountDownLatch latch = new CountDownLatch(1);
+            GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("logs",
+                new OriginalIndices(new String[]{"logs"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
+                numShards, randomBoolean(), primaryNode, replicaNode);
+            final SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp").order(order)));
+            searchRequest.allowPartialSearchResults(true);
+
+            CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger,
+                searchTransportService,
+                (clusterAlias, node) -> lookup.get(node),
+                Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
+                Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(),
+                searchRequest, null, shardsIter, timeProvider, ClusterState.EMPTY_STATE, null,
+                (iter) -> new SearchPhase("test") {
+                    @Override
+                    public void run() {
+                        result.set(iter);
+                        latch.countDown();
+                    }
+                }, SearchResponse.Clusters.EMPTY);
+
+            canMatchPhase.start();
+            latch.await();
+            int shardId = 0;
+            for (SearchShardIterator i : result.get()) {
+                assertThat(i.shardId().id(), equalTo(shardId++));
+                assertEquals(shardToSkip.contains(i.shardId()), i.skip());
+            }
+            assertThat(result.get().size(), equalTo(numShards));
+        }
+    }
 }

From f0ec89882f59c0d4af7ae4fb85780d6de1892622 Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 12 Mar 2020 20:09:53 -0600
Subject: [PATCH 016/285] Re-enable BWC tests after ComponentTemplateMetadata
 backport (#53520)

This can be merged after #53515 is merged.

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index acf9d48770442..2dbf94ae7dabb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53515" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From febe7af62efae12d0faeab2c9ce391e2376c3171 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Fri, 13 Mar 2020 04:07:40 -0400
Subject: [PATCH 017/285] [DOCS] Clarify `max_shingle_size` parm def (#53480)

Rewrites the `search_as_you_type` field datatype's `max_shingle_size`
mapping parameter to improve clarity and better communicate trade-offs
regarding index size.

Relates to [elastic/kibana#55161][0].

Closes #51774.

[0]: https://github.com/elastic/kibana/pull/55161#discussion_r368107177
---
 .../mapping/types/search-as-you-type.asciidoc     | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
index 75cb76a4faff4..da4cb14ba9904 100644
--- a/docs/reference/mapping/types/search-as-you-type.asciidoc
+++ b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -162,15 +162,18 @@ GET my_index/_search
 The following parameters are accepted in a mapping for the `search_as_you_type`
 field and are specific to this field type
 
-[horizontal]
-
 `max_shingle_size`::
++
+--
+(Optional, integer)
+Largest shingle size to create. Valid values are `2` (inclusive) to `4`
+(inclusive). Defaults to `3`.
 
-    The largest shingle size to index the input with and create subfields for,
-    creating one subfield for each shingle size between 2 and
-    `max_shingle_size`. Accepts integer values between 2 and 4 inclusive. This
-    option defaults to 3.
+A subfield is created for each integer between `2` and this value. For example,
+a value of `3` creates two subfields: `my_field._2gram` and `my_field._3gram`
 
+More subfields enables more specific queries but increases index size.
+--
 
 [[general-params]]
 ==== Parameters of the field type as a text field

From 7a182948f6c0aa8d98c412863f37bf1590d425be Mon Sep 17 00:00:00 2001
From: William Brafford <william.brafford@elastic.co>
Date: Fri, 13 Mar 2020 06:41:37 -0400
Subject: [PATCH 018/285] Use snake case for nodes stats/info metric names
 (#53446)

The REST API uses "thread_pool" as the name of the thread pool metric.
If we use this name internally when we serialize nodes stats and info
requests, we won't need to do any fancy logic to check for and switch
out "threadPool", which was the previous internal name.
---
 build.gradle                                                  | 4 ++--
 .../action/admin/cluster/node/info/NodesInfoRequest.java      | 2 +-
 .../action/admin/cluster/node/stats/NodesStatsRequest.java    | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.gradle b/build.gradle
index 2dbf94ae7dabb..2b6039c282682 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53446" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
index 4ea78a17fd0e9..6f33f75692294 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
@@ -286,7 +286,7 @@ enum Metrics {
         OS("os"),
         PROCESS("process"),
         JVM("jvm"),
-        THREAD_POOL("threadPool"),
+        THREAD_POOL("thread_pool"),
         TRANSPORT("transport"),
         HTTP("http"),
         PLUGINS("plugins"),
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 875af87793cf2..4524a2c082c65 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -319,7 +319,7 @@ private enum Metric {
         OS("os"),
         PROCESS("process"),
         JVM("jvm"),
-        THREAD_POOL("threadPool"),
+        THREAD_POOL("thread_pool"),
         FS("fs"),
         TRANSPORT("transport"),
         HTTP("http"),

From b1682fcba1c6ab23bf4d5b842fb4b914a073a386 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Fri, 13 Mar 2020 12:11:34 +0000
Subject: [PATCH 019/285] [ML] Parse single named object in config classes
 (#53472)

---
 .../client/ml/inference/TrainedModelDefinition.java   |  8 +-------
 .../ml/inference/trainedmodel/ensemble/Ensemble.java  |  6 +-----
 .../core/ml/inference/TrainedModelDefinition.java     |  3 +--
 .../ml/inference/trainedmodel/ensemble/Ensemble.java  | 11 +----------
 4 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
index e01d08d019fdc..1f6eed5b3fac1 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java
@@ -43,9 +43,8 @@ public class TrainedModelDefinition implements ToXContentObject {
             true,
             TrainedModelDefinition.Builder::new);
     static {
-        PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
+        PARSER.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> p.namedObject(TrainedModel.class, n, null),
-            (modelDocBuilder) -> { /* Noop does not matter client side*/ },
             TRAINED_MODEL);
         PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> p.namedObject(PreProcessor.class, n, null),
@@ -124,11 +123,6 @@ public Builder setTrainedModel(TrainedModel trainedModel) {
             return this;
         }
 
-        private Builder setTrainedModel(List<TrainedModel> trainedModel) {
-            assert trainedModel.size() == 1;
-            return setTrainedModel(trainedModel.get(0));
-        }
-
         public TrainedModelDefinition build() {
             return new TrainedModelDefinition(this.trainedModel, this.preProcessors);
         }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
index 45d8afb32e6b9..4f16422cc7bd7 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -56,9 +56,8 @@ public class Ensemble implements TrainedModel {
                     p.namedObject(TrainedModel.class, n, null),
             (ensembleBuilder) -> { /* Noop does not matter client side */ },
             TRAINED_MODELS);
-        PARSER.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
+        PARSER.declareNamedObject(Ensemble.Builder::setOutputAggregator,
             (p, c, n) -> p.namedObject(OutputAggregator.class, n, null),
-            (ensembleBuilder) -> { /* Noop does not matter client side */ },
             AGGREGATE_OUTPUT);
         PARSER.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         PARSER.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -194,9 +193,6 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
-        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
-            this.setOutputAggregator(outputAggregators.get(0));
-        }
 
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
index f15d8296bd531..4a11c361d9457 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinition.java
@@ -53,11 +53,10 @@ private static ObjectParser<TrainedModelDefinition.Builder, Void> createParser(b
         ObjectParser<TrainedModelDefinition.Builder, Void> parser = new ObjectParser<>(NAME,
             ignoreUnknownFields,
             TrainedModelDefinition.Builder::builderForParser);
-        parser.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
+        parser.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> ignoreUnknownFields ?
                 p.namedObject(LenientlyParsedTrainedModel.class, n, null) :
                 p.namedObject(StrictlyParsedTrainedModel.class, n, null),
-            (modelDocBuilder) -> { /* Noop does not matter as we will throw if more than one is defined */ },
             TRAINED_MODEL);
         parser.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> ignoreUnknownFields ?
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
index 5c9996ba21588..5955ecda6f276 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -73,11 +73,10 @@ private static ObjectParser<Ensemble.Builder, Void> createParser(boolean lenient
                     p.namedObject(StrictlyParsedTrainedModel.class, n, null),
             (ensembleBuilder) -> ensembleBuilder.setModelsAreOrdered(true),
             TRAINED_MODELS);
-        parser.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
+        parser.declareNamedObject(Ensemble.Builder::setOutputAggregator,
             (p, c, n) ->
                 lenient ? p.namedObject(LenientlyParsedOutputAggregator.class, n, null) :
                     p.namedObject(StrictlyParsedOutputAggregator.class, n, null),
-            (ensembleBuilder) -> {/*Noop as it could be an array or object, it just has to be a one*/},
             AGGREGATE_OUTPUT);
         parser.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         parser.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -414,14 +413,6 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
-        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
-            if (outputAggregators.size() != 1) {
-                throw ExceptionsHelper.badRequestException("[{}] must have exactly one aggregator defined.",
-                    AGGREGATE_OUTPUT.getPreferredName());
-            }
-            this.setOutputAggregator(outputAggregators.get(0));
-        }
-
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);
         }

From f873398956e9686ba7f2f4a365eadc1adb7a0975 Mon Sep 17 00:00:00 2001
From: William Brafford <william.brafford@elastic.co>
Date: Fri, 13 Mar 2020 08:55:14 -0400
Subject: [PATCH 020/285] =?UTF-8?q?Re=C3=ABnable=20BWC=20tests=20(#53538)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 2b6039c282682..2dbf94ae7dabb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53446" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From eade4a2b4676b69b1ff438ecb07d9afdce770292 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 13 Mar 2020 14:37:20 +0100
Subject: [PATCH 021/285] Fix broken link in client documentation (#51834)

This change fixes the link to the RankFeatureQueryBuilder javadocs
in the documentation.

Closes #51800
---
 docs/Versions.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index ac839e3b9cb95..f7d22f233b840 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -29,7 +29,7 @@ ifeval::["{release-state}"=="unreleased"]
 :rest-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}-SNAPSHOT
 :rest-client-sniffer-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}-SNAPSHOT
 :rest-high-level-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}-SNAPSHOT
-:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}-SNAPSHOT
+:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}-SNAPSHOT
 :painless-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}-SNAPSHOT
 :parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
 :percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
@@ -44,7 +44,7 @@ ifeval::["{release-state}"!="unreleased"]
 :rest-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}
 :rest-client-sniffer-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}
 :rest-high-level-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}
-:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}
+:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}
 :painless-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}
 :parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
 :percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}

From 314145294ea926e069c6f8629dfc622a7f31a0fb Mon Sep 17 00:00:00 2001
From: Marios Trivyzas <matriv@gmail.com>
Date: Fri, 13 Mar 2020 14:42:09 +0100
Subject: [PATCH 022/285] Fix YAML test for search.allow_expensive_queries
 (#53541)

Remove excessive testing and keep only the checks for when the queries
are disallowed. Fix also the check for the initial value of the setting
to be conmbatible with Go client tests.
---
 .../test/search/320_disallow_queries.yml      | 185 +-----------------
 1 file changed, 1 insertion(+), 184 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
index e0b6827aa7995..7135c8642736c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/320_disallow_queries.yml
@@ -55,94 +55,7 @@ teardown:
       cluster.get_settings:
         flat_settings: true
 
-  - match: {search.allow_expensive_queries: null}
-
-  ### Prefix
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            prefix:
-              text:
-                value: out
-
-  - match: { hits.total.value: 3 }
-
-  ### Fuzzy
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            fuzzy:
-              text:
-                value: outwide
-
-  - match: { hits.total.value: 3 }
-
-
-  ### Regexp
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            regexp:
-              text:
-                value: .*ou.*id.*
-
-  - match: { hits.total.value: 3 }
-
-  ### Wildcard
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            wildcard:
-              text:
-                value: out?ide
-
-  - match: { hits.total.value: 3 }
-
-  ### Range on text
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text:
-                gte: "theres"
-
-  - match: { hits.total.value: 2 }
-
-  ### Range on keyword
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text.raw:
-                gte : "Outside it is cold and wet"
-
-  - match: { hits.total.value: 2 }
-
-  ### Nested
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            nested:
-              path: "nested1"
-              query:
-                bool:
-                  must: [{"match": {"nested1.foo": "bar2"}}]
-
-  - match: { hits.total.value: 1 }
+  - is_false: search.allow_expensive_queries
 
   ### Update setting to false
   - do:
@@ -232,99 +145,3 @@ teardown:
               query:
                 bool:
                   must: [{"match" : {"nested1.foo" : "bar2"}}]
-
-  ### Revert setting to true
-  - do:
-      cluster.put_settings:
-        body:
-          transient:
-            search.allow_expensive_queries: "true"
-        flat_settings: true
-
-  - match: {transient: {search.allow_expensive_queries: "true"}}
-
-  ### Prefix
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            prefix:
-              text:
-                value: out
-
-  - match: { hits.total.value: 3 }
-
-  ### Fuzzy
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            fuzzy:
-              text:
-                value: outwide
-
-  - match: { hits.total.value: 3 }
-
-  ### Regexp
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            regexp:
-              text:
-                value: .*ou.*id.*
-
-  - match: { hits.total.value: 3 }
-
-  ### Wildcard
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            wildcard:
-              text:
-                value: out?ide
-
-  - match: { hits.total.value: 3 }
-
-  ### Range on text
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text:
-                gte: "theres"
-
-  - match: { hits.total.value: 2 }
-
-  ### Range on keyword
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            range:
-              text.raw:
-                gte: "Outside it is cold and wet"
-
-  - match: { hits.total.value: 2 }
-
-  ### Nested
-  - do:
-      search:
-        index: test
-        body:
-          query:
-            nested:
-              path: "nested1"
-              query:
-                bool:
-                  must: [{"match": {"nested1.foo": "bar2"}}]
-
-  - match: { hits.total.value: 1 }

From a8080102936a128c41456be68e06065dd743b167 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 13 Mar 2020 15:13:26 +0100
Subject: [PATCH 023/285] Reenable watcher rest tests (#53532)

Also log a message instead of failing if there are active watches at a beginning of a test.

Relates to #53177
---
 .../org/elasticsearch/smoketest/WatcherYamlRestIT.java |  2 --
 .../xpack/watcher/WatcherYamlSuiteTestCase.java        | 10 ++++++++--
 ...keTestWatcherWithSecurityClientYamlTestSuiteIT.java |  2 --
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/WatcherYamlRestIT.java b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/WatcherYamlRestIT.java
index 8f94e7c61d374..5ec99c01b97b5 100644
--- a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/WatcherYamlRestIT.java
+++ b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/WatcherYamlRestIT.java
@@ -5,14 +5,12 @@
  */
 package org.elasticsearch.smoketest;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.xpack.watcher.WatcherYamlSuiteTestCase;
 
 /**
  * Runs the YAML rest tests against an external cluster
  */
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53177")
 public class WatcherYamlRestIT extends WatcherYamlSuiteTestCase {
     public WatcherYamlRestIT(ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
diff --git a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
index 96dde81bf8a45..7dd064f7dbc91 100644
--- a/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
+++ b/x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java
@@ -14,11 +14,11 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -54,7 +54,10 @@ public final void startWatcher() throws Exception {
                     throw new AssertionError("waiting until starting state reached started state");
                 case "started":
                     int watcherCount = (int) response.evaluate("stats.0.watch_count");
-                    assertThat(watcherCount, equalTo(0));
+                    if (watcherCount > 0) {
+                        logger.info("expected 0 active watches, but got [{}], deleting watcher indices again", watcherCount);
+                        deleteWatcherIndices();
+                    }
                     // all good here, we are done
                     break;
                 default:
@@ -88,7 +91,10 @@ public final void stopWatcher() throws Exception {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         }, 60, TimeUnit.SECONDS);
+        deleteWatcherIndices();
+    }
 
+    private static void deleteWatcherIndices() throws IOException {
         Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
         deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
         adminClient().performRequest(deleteWatchesIndexRequest);
diff --git a/x-pack/plugin/watcher/qa/with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java b/x-pack/plugin/watcher/qa/with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java
index e16ccc70ca9cb..5bebf4ebc23d4 100644
--- a/x-pack/plugin/watcher/qa/with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java
+++ b/x-pack/plugin/watcher/qa/with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java
@@ -7,7 +7,6 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -19,7 +18,6 @@
 
 import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53177")
 public class SmokeTestWatcherWithSecurityClientYamlTestSuiteIT extends WatcherYamlSuiteTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";

From fb158dfb3ea36eb24335bd2e004305c1033c9e2f Mon Sep 17 00:00:00 2001
From: bellengao <gbl_long@163.com>
Date: Fri, 13 Mar 2020 22:35:02 +0800
Subject: [PATCH 024/285] Fix inaccurate total hit count in _search template
 api (#53155)

When 'rest_track_total_hits_as_int' is set to true, the total hits count in the response should be accurate. So we should set trackTotalHits to true if need when parsing the inline script of a search template request.

Closes #52801
---
 .../mustache/RestSearchTemplateAction.java    |  1 -
 .../TransportSearchTemplateAction.java        | 21 +++++++++++++++++++
 .../test/lang_mustache/30_search_template.yml | 13 ++++++++++++
 .../50_multi_search_template.yml              |  8 +++++++
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
index fe8e13d7ee239..dfc0f3d0228b2 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -72,7 +72,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             searchTemplateRequest = SearchTemplateRequest.fromXContent(parser);
         }
         searchTemplateRequest.setRequest(searchRequest);
-        RestSearchAction.checkRestTotalHits(request, searchRequest);
 
         return channel -> client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel));
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
index a78cfbd2b6eec..586deae9b7309 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
@@ -32,11 +32,13 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -110,8 +112,27 @@ static SearchRequest convert(SearchTemplateRequest searchTemplateRequest, Search
             builder.parseXContent(parser, false);
             builder.explain(searchTemplateRequest.isExplain());
             builder.profile(searchTemplateRequest.isProfile());
+            checkRestTotalHitsAsInt(searchRequest, builder);
             searchRequest.source(builder);
         }
         return searchRequest;
     }
+
+    private static void checkRestTotalHitsAsInt(SearchRequest searchRequest, SearchSourceBuilder searchSourceBuilder) {
+        if (searchRequest.source() == null) {
+            searchRequest.source(new SearchSourceBuilder());
+        }
+        Integer trackTotalHitsUpTo = searchRequest.source().trackTotalHitsUpTo();
+        // trackTotalHitsUpTo is set to Integer.MAX_VALUE when `rest_total_hits_as_int` is true
+        if (trackTotalHitsUpTo != null) {
+            if (searchSourceBuilder.trackTotalHitsUpTo() == null) {
+                // trackTotalHitsUpTo should be set here, ensure that we can get an accurate total hits count
+                searchSourceBuilder.trackTotalHitsUpTo(trackTotalHitsUpTo);
+            } else if (searchSourceBuilder.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_ACCURATE
+                && searchSourceBuilder.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
+                throw new IllegalArgumentException("[" + RestSearchAction.TOTAL_HITS_AS_INT_PARAM + "] cannot be used " +
+                    "if the tracking of total hits is not accurate, got " + searchSourceBuilder.trackTotalHitsUpTo());
+            }
+        }
+    }
 }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
index f4d4f3a97e5e0..a9d3c2da68617 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
@@ -40,6 +40,19 @@
 
   - match: { hits.total: 2 }
 
+---
+"Test with invalid track_total_hits":
+
+  - do:
+      catch:              bad_request
+      search_template:
+        rest_total_hits_as_int: true
+        body: { "source" : { "query": { "match_{{template}}": {} }, "track_total_hits": "{{trackTotalHits}}" }, "params" : { "template" : "all", "trackTotalHits" : 1 } }
+
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
+
 ---
 "Missing template search request":
 
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
index 8f72583b61d7c..e92e10b9ad276 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
@@ -94,6 +94,11 @@ setup:
           - source: '{"query": {"{{query_type}}": {} }' # Unknown query type
             params:
               query_type: "unknown"
+          # Search 4 has an unsupported track_total_hits
+          - index: index_*
+          - source: '{"query": {"match_all": {} }, "track_total_hits": "{{trackTotalHits}}" }'
+            params:
+              trackTotalHits: 1
 
   - match:  { responses.0.hits.total:     2  }
   - match:  { responses.1.error.root_cause.0.type: json_e_o_f_exception }
@@ -101,6 +106,9 @@ setup:
   - match:  { responses.2.hits.total:     1  }
   - match:  { responses.3.error.root_cause.0.type: parsing_exception }
   - match:  { responses.3.error.root_cause.0.reason: "/unknown.query.\\[unknown\\]/" }
+  - match:  { responses.4.error.root_cause.0.type: illegal_argument_exception }
+  - match:  { responses.4.error.root_cause.0.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
+
 
 ---
 "Multi-search template with invalid request":

From 548fd9494b46c81b0caf69c27e9b5d200bc8260a Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Fri, 13 Mar 2020 15:37:29 +0100
Subject: [PATCH 025/285] Move async search yaml tests to x-pack yaml test
 folder (#53537)

The yaml tests for async search currently sit in its qa folder. There is no reason though for them to live in a separate folder as they don't require particular setup. This commit moves them to the main folder together with the other x-pack yaml tests so that they will be run by the client test runners too.
---
 .../plugin/async-search/qa/rest/build.gradle  | 30 ----------------
 .../xpack/search/AsyncSearchRestIT.java       | 35 -------------------
 .../xpack/search/AsyncSearchSecurityIT.java   |  3 +-
 .../xpack/search/AsyncSearchRestTestCase.java | 15 --------
 .../test/async_search}/10_basic.yml           |  0
 5 files changed, 2 insertions(+), 81 deletions(-)
 delete mode 100644 x-pack/plugin/async-search/qa/rest/build.gradle
 delete mode 100644 x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java
 delete mode 100644 x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java
 rename x-pack/plugin/{async-search/qa/rest/src/test/resources/rest-api-spec/test/search => src/test/resources/rest-api-spec/test/async_search}/10_basic.yml (100%)

diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle
deleted file mode 100644
index fbe97dcb7aba5..0000000000000
--- a/x-pack/plugin/async-search/qa/rest/build.gradle
+++ /dev/null
@@ -1,30 +0,0 @@
-import org.elasticsearch.gradle.test.RestIntegTestTask
-
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-test'
-
-restResources {
-  restApi {
-    includeXpack 'async_search'
-  }
-}
-
-dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-  testCompile project(path: xpackModule('async-search'), configuration: 'runtime')
-}
-
-task restTest(type: RestIntegTestTask) {
-  mustRunAfter(precommit)
-}
-
-testClusters.restTest {
-  testDistribution = 'DEFAULT'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.monitoring.enabled', 'false'
-  setting 'xpack.security.enabled', 'true'
-  user username: 'async-search-user', password: 'async-search-password'
-}
-
-check.dependsOn restTest
-test.enabled = false
diff --git a/x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java b/x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java
deleted file mode 100644
index 89bef2430f557..0000000000000
--- a/x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.search;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-
-public class AsyncSearchRestIT extends ESClientYamlSuiteTestCase {
-
-    public AsyncSearchRestIT(final ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        final String userAuthHeaderValue = basicAuthHeaderValue("async-search-user",
-            new SecureString("async-search-password".toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", userAuthHeaderValue).build();
-    }
-}
diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
index d1760acda2b8d..05e0132deb7d3 100644
--- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
+++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -31,7 +32,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-public class AsyncSearchSecurityIT extends AsyncSearchRestTestCase {
+public class AsyncSearchSecurityIT extends ESRestTestCase {
     /**
      * All tests run as a superuser but use <code>es-security-runas-user</code> to become a less privileged user.
      */
diff --git a/x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java b/x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java
deleted file mode 100644
index f176efdda4201..0000000000000
--- a/x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.search;
-
-import org.elasticsearch.test.rest.ESRestTestCase;
-
-public class AsyncSearchRestTestCase extends ESRestTestCase {
-    @Override
-    protected boolean preserveClusterUponCompletion() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
similarity index 100%
rename from x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml
rename to x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml

From 6f1fdcf980dd3f3a89604fcde02aff46f9192ddd Mon Sep 17 00:00:00 2001
From: Jakob Reiter <jakommo@users.noreply.github.com>
Date: Fri, 13 Mar 2020 15:47:43 +0100
Subject: [PATCH 026/285] Remove additional ] (#53420)

There was one ] too much, which broke the headline formatting of the next paragraph
---
 x-pack/docs/en/security/authentication/saml-guide.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
index c100ef161c755..1cf13fbd2a15d 100644
--- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc
+++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
@@ -879,7 +879,7 @@ an {es} access token and a refresh token. The access token is used as credential
 for subsequent calls to {es}. The refresh token enables the user to get new {es}
 access tokens after the current one expires.
 
-[[saml-no-kibana-realm]]]
+[[saml-no-kibana-realm]]
 ==== SAML realm
 
 You must create a SAML realm and configure it accordingly

From 1fc3fe3d32f41eab2101c0536751b7c47e63cc48 Mon Sep 17 00:00:00 2001
From: Marios Trivyzas <matriv@gmail.com>
Date: Fri, 13 Mar 2020 16:17:30 +0100
Subject: [PATCH 027/285] Fix Term Vectors with artificial docs and keyword
 fields (#53504)

Previously, Term Vectors API was returning empty results for
artificial documents with keyword fields. Checking only for `string()`
on `IndexableField` is not enough, since for `KeywordFieldType`
`binaryValue()` must be used instead.

Fixes #53494
---
 .../index/mapper/ParseContext.java            | 18 ---------------
 .../index/termvectors/TermVectorsService.java | 23 ++++++++++++++++++-
 .../index/mapper/DateFieldMapperTests.java    |  3 ++-
 .../index/mapper/DocumentParserTests.java     | 14 +++++++----
 .../mapper/FieldNamesFieldMapperTests.java    |  3 ++-
 .../index/mapper/IpFieldMapperTests.java      |  3 ++-
 .../index/mapper/KeywordFieldMapperTests.java |  4 ++++
 .../index/mapper/NumberFieldMapperTests.java  |  4 ++--
 8 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index b2c536a900e33..d675735834f7b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -120,24 +120,6 @@ public IndexableField[] getFields(String name) {
             return f.toArray(new IndexableField[f.size()]);
         }
 
-        /**
-         * Returns an array of values of the field specified as the method parameter.
-         * This method returns an empty array when there are no
-         * matching fields.  It never returns null.
-         * If you want the actual numeric field instances back, use {@link #getFields}.
-         * @param name the name of the field
-         * @return a <code>String[]</code> of field values
-         */
-        public final String[] getValues(String name) {
-            List<String> result = new ArrayList<>();
-            for (IndexableField field : fields) {
-                if (field.name().equals(name) && field.stringValue() != null) {
-                    result.add(field.stringValue());
-                }
-            }
-            return result.toArray(new String[result.size()]);
-        }
-
         public IndexableField getField(String name) {
             for (IndexableField field : fields) {
                 if (field.name().equals(name)) {
diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 400e5355c0942..65df5e2b35268 100644
--- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -56,6 +57,7 @@
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -313,7 +315,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
             else {
                 seenFields.add(field.name());
             }
-            String[] values = doc.getValues(field.name());
+            String[] values = getValues(doc.getFields(field.name()));
             documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values)));
         }
         return generateTermVectors(indexShard,
@@ -321,6 +323,25 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
                 request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
+    /**
+     * Returns an array of values of the field specified as the method parameter.
+     * This method returns an empty array when there are no
+     * matching fields.  It never returns null.
+     * @param fields The <code>IndexableField</code> to get the values from
+     * @return a <code>String[]</code> of field values
+     */
+    public static String[] getValues(IndexableField[] fields) {
+        List<String> result = new ArrayList<>();
+        for (IndexableField field : fields) {
+            if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) {
+                result.add(field.binaryValue().utf8ToString());
+            } else if (field.fieldType() instanceof StringFieldType) {
+                result.add(field.stringValue());
+            }
+        }
+        return result.toArray(new String[0]);
+    }
+
     private static ParsedDocument parseDocument(IndexShard indexShard, String index, BytesReference doc,
                                                 XContentType xContentType, String routing) {
         MapperService mapperService = indexShard.mapperService();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index b276f7fc711af..d63a34ab501e3 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -202,7 +203,7 @@ private void testIgnoreMalfomedForValue(String value, String expectedException)
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(0, fields.length);
-        assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
+        assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
     }
 
     public void testChangeFormat() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 8e54a92a22228..86861901f7a07 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -154,11 +154,15 @@ public void testDotsWithExistingMapper() throws Exception {
             .endObject());
         ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", bytes, XContentType.JSON));
         assertNull(doc.dynamicMappingsUpdate()); // no update!
-        String[] values = doc.rootDoc().getValues("foo.bar.baz");
-        assertEquals(3, values.length);
-        assertEquals("123", values[0]);
-        assertEquals("456", values[1]);
-        assertEquals("789", values[2]);
+
+        IndexableField[] fields = doc.rootDoc().getFields("foo.bar.baz");
+        assertEquals(6, fields.length);
+        assertEquals(123, fields[0].numericValue());
+        assertEquals("123", fields[1].stringValue());
+        assertEquals(456, fields[2].numericValue());
+        assertEquals("456", fields[3].stringValue());
+        assertEquals(789, fields[4].numericValue());
+        assertEquals("789", fields[5].stringValue());
     }
 
     public void testDotsWithExistingNestedMapper() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index 953a24ce00fa8..e37523ee730c0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -52,7 +53,7 @@ private static SortedSet<String> set(String... values) {
     }
 
     void assertFieldNames(Set<String> expected, ParsedDocument doc) {
-        String[] got = doc.rootDoc().getValues("_field_names");
+        String[] got = TermVectorsService.getValues(doc.rootDoc().getFields("_field_names"));
         assertEquals(expected, set(got));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index a34adaca2f144..37ea96e3df976 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -194,7 +195,7 @@ public void testIgnoreMalformed() throws Exception {
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(0, fields.length);
-        assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
+        assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
     }
 
     public void testNullValue() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 0e25c0a6a212b..d5f0414708d64 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -128,6 +129,9 @@ public void testDefaults() throws Exception {
         fieldType = fields[1].fieldType();
         assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
         assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+
+        // used by TermVectorsService
+        assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field")));
     }
 
     public void testIgnoreAbove() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index a1322c6614456..7adaf1b2a3233 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.randomizedtesting.annotations.Timeout;
-
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.Strings;
@@ -32,6 +31,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec;
+import org.elasticsearch.index.termvectors.TermVectorsService;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -250,7 +250,7 @@ public void testIgnoreMalformed() throws Exception {
 
                 IndexableField[] fields = doc.rootDoc().getFields("field");
                 assertEquals(0, fields.length);
-                assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored"));
+                assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored")));
             }
         }
     }

From 375a211889136305a7534c1418d3c216fd154c9b Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 13 Mar 2020 11:19:36 -0400
Subject: [PATCH 028/285] Handle no such remote cluster exception in ccr
 (#53415)

A remote client can throw a NoSuchRemoteClusterException while fetching
the cluster state from the leader cluster. We also need to handle that
exception when retrying to add a retention lease to the leader shard.

Closes #53225
---
 .../ccr/action/ShardFollowTasksExecutor.java  | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
index cbc4e4abc2bfe..e523d452c75b1 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
@@ -132,16 +132,7 @@ protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer h
                 final Index followerIndex = params.getFollowShardId().getIndex();
                 final Index leaderIndex = params.getLeaderShardId().getIndex();
                 final Supplier<TimeValue> timeout = () -> isStopped() ? TimeValue.MINUS_ONE : waitForMetadataTimeOut;
-
-                final Client remoteClient;
-                try {
-                    remoteClient = remoteClient(params);
-                } catch (NoSuchRemoteClusterException e) {
-                    errorHandler.accept(e);
-                    return;
-                }
-
-                CcrRequests.getIndexMetadata(remoteClient, leaderIndex, minRequiredMappingVersion, 0L, timeout, ActionListener.wrap(
+                final ActionListener<IndexMetaData> listener = ActionListener.wrap(
                     indexMetaData -> {
                         if (indexMetaData.mapping() == null) {
                             assert indexMetaData.getMappingVersion() == 1;
@@ -155,7 +146,12 @@ protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer h
                             errorHandler));
                     },
                     errorHandler
-                ));
+                );
+                try {
+                    CcrRequests.getIndexMetadata(remoteClient(params), leaderIndex, minRequiredMappingVersion, 0L, timeout, listener);
+                } catch (NoSuchRemoteClusterException e) {
+                    errorHandler.accept(e);
+                }
             }
 
             @Override
@@ -424,21 +420,27 @@ protected Scheduler.Cancellable scheduleBackgroundRetentionLeaseRenewal(final Lo
                                         "{} background adding retention lease [{}] while following",
                                         params.getFollowShardId(),
                                         retentionLeaseId);
-                                CcrRetentionLeases.asyncAddRetentionLease(
+                                try {
+                                    final ActionListener<RetentionLeaseActions.Response> wrappedListener = ActionListener.wrap(
+                                        r -> {},
+                                        inner -> {
+                                            /*
+                                             * If this fails that the retention lease already exists, something highly unusual is
+                                             * going on. Log it, and renew again after another renew interval has passed.
+                                             */
+                                            final Throwable innerCause = ExceptionsHelper.unwrapCause(inner);
+                                            logRetentionLeaseFailure(retentionLeaseId, innerCause);
+                                        });
+                                    CcrRetentionLeases.asyncAddRetentionLease(
                                         params.getLeaderShardId(),
                                         retentionLeaseId,
                                         followerGlobalCheckpoint.getAsLong(),
                                         remoteClient(params),
-                                        ActionListener.wrap(
-                                                r -> {},
-                                                inner -> {
-                                                    /*
-                                                     * If this fails that the retention lease already exists, something highly unusual is
-                                                     * going on. Log it, and renew again after another renew interval has passed.
-                                                     */
-                                                    final Throwable innerCause = ExceptionsHelper.unwrapCause(inner);
-                                                    logRetentionLeaseFailure(retentionLeaseId, innerCause);
-                                                }));
+                                        wrappedListener);
+                                } catch (NoSuchRemoteClusterException rce) {
+                                    // we will attempt to renew again after another renew interval has passed
+                                    logRetentionLeaseFailure(retentionLeaseId, rce);
+                                }
                             } else {
                                  // if something else happened, we will attempt to renew again after another renew interval has passed
                             }

From 0f8de60bcdb074d158499959b182b91cad29bbc9 Mon Sep 17 00:00:00 2001
From: Tom Veasey <tveasey@users.noreply.github.com>
Date: Fri, 13 Mar 2020 15:28:47 +0000
Subject: [PATCH 029/285] [ML] Mute data frame analytics BWC tests (#53546)

Until #52763 is backported to 7.x.
---
 .../test/mixed_cluster/90_ml_data_frame_analytics_crud.yml   | 5 +++++
 .../test/old_cluster/90_ml_data_frame_analytics_crud.yml     | 3 +++
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml     | 5 +++++
 3 files changed, 13 insertions(+)

diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index 5049f3360a42e..be388da19be07 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
+
 ---
 "Get old outlier_detection job":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index 923a56395e8a4..10ab98a738e1c 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
 
   - do:
       index:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index b197d0c5cf433..aa4ef717263f9 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
+
 ---
 "Get old cluster outlier_detection job":
 

From 7aa661c2eb8c87021e7b2f914b7bde24d54f87e2 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Fri, 13 Mar 2020 09:32:15 -0700
Subject: [PATCH 030/285] [DOCS] Adds operations_behind to transform stats
 (#53518)

---
 docs/reference/transform/apis/get-transform-stats.asciidoc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc
index e23a6fb2fd54b..1175fba036bb4 100644
--- a/docs/reference/transform/apis/get-transform-stats.asciidoc
+++ b/docs/reference/transform/apis/get-transform-stats.asciidoc
@@ -104,6 +104,10 @@ upper bound of data that is included in the checkpoint.
 `checkpointing`.`next`.`timestamp_millis`::::
 (date) The timestamp of the checkpoint, which indicates when the checkpoint was
 created.
+`checkpointing`.`operations_behind`:::
+(integer) The number of operations that have occurred on the source index but
+have not been applied to the destination index yet. A high number can indicate
+that the {transform} is failing to keep up. 
 
 `id`::
 (string)

From b1ac51d0dd6bd68af71ee2ce23021edcaf37ac5c Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Fri, 13 Mar 2020 10:02:59 -0700
Subject: [PATCH 031/285] Ensure jdk download uses a unique maven group
 (#53531)

The maven coordinates for jdks in the jdk download plugin contain a
group name local to our build. However, this still needs to be unique in
order for the group filtering to a single ivy repository to work. This
commit adjusts the group name to include both the jdk vendor as well as
major version.

Note: this change was already included in the backport of #53358
---
 .../java/org/elasticsearch/gradle/JdkDownloadPlugin.java  | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index a0fe4eb081137..b63d8f65ea04f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -143,7 +143,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
                     repo.patternLayout(layout -> layout.artifact(artifactPattern));
                 });
                 repositories.exclusiveContent(exclusiveContentRepository -> {
-                    exclusiveContentRepository.filter(config -> config.includeGroup(jdk.getVendor()));
+                    exclusiveContentRepository.filter(config -> config.includeGroup(groupName(jdk)));
                     exclusiveContentRepository.forRepositories(ivyRepo);
                 });
             }
@@ -254,7 +254,11 @@ private static String dependencyNotation(Jdk jdk) {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return jdk.getVendor() + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+    }
+
+    private static String groupName(Jdk jdk) {
+        return jdk.getVendor() + "_" + jdk.getMajor();
     }
 
     private static String configName(String... parts) {

From 59c2eb462fbbafc468640b790a5afeede6aeb9ac Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Fri, 13 Mar 2020 13:17:35 -0400
Subject: [PATCH 032/285] [ML] [Inference] fix number inference models returned
 in x-pack info call (#53540)

the ML portion of the x-pack info API was erroneously counting configuration documents and definition documents. The underlying implementation of our storage separates the two out.

This PR filters the query so that only trained model config documents are counted.
---
 .../xpack/ml/MachineLearningUsageTransportAction.java         | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
index 4f731d9804c4b..7f36db7a5b893 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
@@ -40,6 +41,7 @@
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
@@ -113,6 +115,8 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat
                 addInferenceIngestUsage(response, inferenceUsage);
                 SearchRequestBuilder requestBuilder = client.prepareSearch(InferenceIndexConstants.INDEX_PATTERN)
                     .setSize(0)
+                    .setQuery(QueryBuilders.boolQuery()
+                        .filter(QueryBuilders.termQuery(InferenceIndexConstants.DOC_TYPE.getPreferredName(), TrainedModelConfig.NAME)))
                     .setTrackTotalHits(true);
                 ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(),
                     ClientHelper.ML_ORIGIN,

From d7bbc9df1d3edf47f8d1c2f0983c88c64d78a4b4 Mon Sep 17 00:00:00 2001
From: Gordon Brown <gordon.brown@elastic.co>
Date: Fri, 13 Mar 2020 11:57:00 -0600
Subject: [PATCH 033/285] Allow _cat indices & aliases to use indices options
 (#53248)

This commit adjusts the _cat/indices and _cat/aliases APIs to allow
specifying indices options, so that these APIs can handle hidden
indices/aliases in the same way as other APIs.

Also adds the hidden option to the expand_wildcards parameter
in the YAML spec for every API that accepts it.
---
 .../rest-api-spec/api/cat.aliases.json        |  12 +
 .../rest-api-spec/api/cat.indices.json        |  12 +
 .../rest-api-spec/api/cluster.health.json     |   1 +
 .../rest-api-spec/api/cluster.state.json      |   1 +
 .../resources/rest-api-spec/api/count.json    |   1 +
 .../rest-api-spec/api/delete_by_query.json    |   1 +
 .../rest-api-spec/api/field_caps.json         |   1 +
 .../api/indices.clear_cache.json              |   1 +
 .../rest-api-spec/api/indices.close.json      |   1 +
 .../rest-api-spec/api/indices.delete.json     |   1 +
 .../rest-api-spec/api/indices.exists.json     |   1 +
 .../api/indices.exists_alias.json             |   1 +
 .../api/indices.exists_type.json              |   1 +
 .../rest-api-spec/api/indices.flush.json      |   1 +
 .../rest-api-spec/api/indices.forcemerge.json |   1 +
 .../rest-api-spec/api/indices.get.json        |   1 +
 .../rest-api-spec/api/indices.get_alias.json  |   3 +-
 .../api/indices.get_field_mapping.json        |   1 +
 .../api/indices.get_mapping.json              |   1 +
 .../api/indices.get_settings.json             |   1 +
 .../api/indices.get_upgrade.json              |   1 +
 .../rest-api-spec/api/indices.open.json       |   1 +
 .../api/indices.put_mapping.json              |   1 +
 .../api/indices.put_settings.json             |   1 +
 .../rest-api-spec/api/indices.refresh.json    |   1 +
 .../rest-api-spec/api/indices.segments.json   |   1 +
 .../api/indices.shard_stores.json             |   1 +
 .../rest-api-spec/api/indices.stats.json      |   1 +
 .../rest-api-spec/api/indices.upgrade.json    |   1 +
 .../api/indices.validate_query.json           |   1 +
 .../rest-api-spec/api/rank_eval.json          |   1 +
 .../resources/rest-api-spec/api/search.json   |   1 +
 .../rest-api-spec/api/search_shards.json      |   1 +
 .../rest-api-spec/api/search_template.json    |   1 +
 .../rest-api-spec/api/update_by_query.json    |   1 +
 .../test/cat.aliases/40_hidden.yml            | 150 +++++++++++
 .../test/cat.indices/20_hidden.yml            | 243 ++++++++++++++++++
 .../rest/action/cat/RestAliasAction.java      |   2 +
 .../rest/action/cat/RestIndicesAction.java    |  14 +-
 x-pack/plugin/build.gradle                    |   2 +
 .../test/rest/CatIndicesWithSecurityIT.java   | 127 +++++++++
 x-pack/plugin/src/test/resources/roles.yml    |   7 +
 42 files changed, 599 insertions(+), 5 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
 create mode 100644 x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
 create mode 100644 x-pack/plugin/src/test/resources/roles.yml

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index d37273cfd41bf..6ee594b099c43 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -53,6 +53,18 @@
         "type":"boolean",
         "description":"Verbose mode. Display column headers",
         "default":false
+      },
+      "expand_wildcards":{
+        "type":"enum",
+        "options":[
+          "open",
+          "closed",
+          "hidden",
+          "none",
+          "all"
+        ],
+        "default": ["all"],
+        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index 76c749cc7d3bd..d4a7eb3b051d2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -107,6 +107,18 @@
         "type":"boolean",
         "description":"If set to true segment stats will include stats for segments that are not currently loaded into memory",
         "default":false
+      },
+      "expand_wildcards":{
+        "type":"enum",
+        "options":[
+          "open",
+          "closed",
+          "hidden",
+          "none",
+          "all"
+        ],
+        "default": "all",
+        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
index 7911a8e244218..2a21ff3725b6a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
@@ -33,6 +33,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
index d6f0e9ababc6d..017705082d189 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
@@ -97,6 +97,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index 4084d3be608a4..93a450afff88a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -47,6 +47,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index 93f6d63bbc2d2..9be09b49cc54c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -70,6 +70,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
index eed6f8bf082a2..d56c3313a0bf0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
@@ -47,6 +47,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
index b1b1d8214c47d..64c10a520c7c4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -53,6 +53,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
index 539088199d683..f26c8e77a06a6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
index d77d47f0d739b..53fdf44bb36a1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
index d6bdcef95815f..7539f44a81eed 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
@@ -39,6 +39,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index a19786a7938f9..66e5ce92cbbe5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -51,6 +51,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
index 2a09d8bdfa679..c854d0e8fd841 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
@@ -39,6 +39,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
index 1eada8f831db6..35138b920466f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
@@ -51,6 +51,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
index 5eaf62840d841..6036b75bb83e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
@@ -45,6 +45,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index ccacf91d93095..90a1274ecb059 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -39,6 +39,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
index 800ea8d30d68d..e238c4fc38afc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
@@ -69,10 +69,11 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
-        "default":"all",
+        "default": ["all"],
         "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
       },
       "local":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
index 4c5191f2f320a..0e71b6d395777 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
@@ -55,6 +55,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
index 17274e8bbd78a..5a27f6d06b83c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
@@ -41,6 +41,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index c19fa3bf5262f..4a1dea974f750 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -73,6 +73,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
index 862fdde4ac395..68cfdf25aeea2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
@@ -49,6 +49,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
index 14050d626df83..1dab468ce4ff4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
index c95f5ea3b48f3..18a42ec11a45e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
@@ -44,6 +44,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
index 75bf3a340c311..66fe23bab8ba2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
@@ -53,6 +53,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
index 7eee4eeec5fd0..950e0a62489fa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
index b99b19799dab9..83430a9a85600 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
@@ -41,6 +41,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
index b397963f99d2f..7e48e99916171 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
@@ -51,6 +51,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index 0940313154a74..0a8960f2f9e89 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -137,6 +137,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
index 93628fdd106d0..406fbacda307c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
@@ -45,6 +45,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
index 5ff6f073b77ec..3becec003a9e6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
@@ -69,6 +69,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
index c87b850ab148f..eadf240192394 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
@@ -43,6 +43,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 2411c309c7dff..1e0c232efa055 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -89,6 +89,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
index 70ee592520be9..74b7055b4c4b0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
@@ -55,6 +55,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index f3286cd932c4e..1d239376b3fe8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -47,6 +47,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index 591371798d440..299c4f705d292 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -70,6 +70,7 @@
         "options":[
           "open",
           "closed",
+          "hidden",
           "none",
           "all"
         ],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
new file mode 100644
index 0000000000000..6866ff595fefd
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
@@ -0,0 +1,150 @@
+---
+"Test cat aliases output with a hidden index with a hidden alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            test_alias:
+              is_hidden: true
+
+  - do:
+      cat.aliases: {}
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+  - do:
+      cat.aliases:
+        name: test_alias
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+
+  - do:
+      cat.aliases:
+        expand_wildcards: ["open","closed"]
+
+  - match:
+      $body: |
+        /^
+        $/
+---
+"Test cat aliases output with a hidden index with a visible alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            test_alias: {}
+  - do:
+      cat.aliases: {}
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+  - do:
+      cat.aliases:
+        name: test_alias
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+---
+"Test cat aliases output with a visible index with a hidden alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+          aliases:
+            test_alias:
+              is_hidden: true
+  - do:
+      cat.aliases: {}
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
+
+  - do:
+      cat.aliases:
+        name: test_alias
+
+  - match:
+      $body: |
+        /^
+            test_alias          \s+
+            test                \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+            -                   \s+
+        $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
new file mode 100644
index 0000000000000..3a4fe28c85996
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
@@ -0,0 +1,243 @@
+---
+"Test cat indices output for hidden index":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+  - do:
+      cat.indices: {}
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        expand_wildcards: ["all"]
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+---
+"Test cat indices output for dot-hidden index and dot-prefixed pattern":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+  - do:
+      indices.create:
+        index: .index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+  - do:
+      cat.indices: {}
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        index: ".*"
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           \.index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+---
+"Test cat indices output with a hidden index with a visible alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            alias1: {}
+  - do:
+      cat.indices:
+        index: "i*"
+        # Can't use a bare wildcard here because Security replaces wildcards
+        # it with all matching authorized indices/aliases, including the visible
+        # alias
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        expand_wildcards: ["open", "hidden"]
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+  - do:
+      cat.indices:
+        index: alias1
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+---
+"Test cat indices output with a hidden index with a hidden alias":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            alias1:
+              is_hidden: true
+  - do:
+      cat.indices: {}
+
+  - match:
+      $body: |
+        /^$/
+
+  - do:
+      cat.indices:
+        expand_wildcards: ["all"]
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+
+  - do:
+      cat.indices:
+        index: alias1
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
+---
+"Test cat indices output with a hidden index, dot-hidden alias and dot pattern":
+  - skip:
+      version: "- 7.99.99"
+      reason: "hidden indices and aliases were added in 7.7.0"
+      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            index:
+              hidden: true
+          aliases:
+            .alias1:
+              is_hidden: true
+  - do:
+      cat.indices: {}
+  - match:
+      $body: |
+        /^$/
+  - do:
+      cat.indices:
+        index: ".*"
+  - match:
+      $body: |
+        /^(green  \s+
+           open   \s+
+           index1 \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           0      \s+
+           0      \s+
+           0      \s+
+           (\d+|\d+[.]\d+)(kb|b) \s+
+           (\d+|\d+[.]\d+)(kb|b) \s*
+         )
+         $/
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index e03b0c98e0e98..90f20811c3dc9 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -21,6 +21,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
@@ -52,6 +53,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
                 new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) :
                 new GetAliasesRequest();
+        getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
         getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
 
         return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestResponseListener<GetAliasesResponse>(channel) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index a70fcec89b0c8..d760d56e8d542 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -90,7 +90,7 @@ protected void documentation(StringBuilder sb) {
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final IndicesOptions indicesOptions = IndicesOptions.strictExpand();
+        final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand());
         final boolean local = request.paramAsBoolean("local", false);
         final TimeValue masterNodeTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT);
         final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false);
@@ -109,6 +109,12 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) {
                     final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, listener);
                     groupedListener.onResponse(getSettingsResponse);
 
+                    // The list of indices that will be returned is determined by the indices returned from the Get Settings call.
+                    // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the
+                    // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so
+                    // force the IndicesOptions for all the sub-requests to be as inclusive as possible.
+                    final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
+
                     // Indices that were successfully resolved during the get settings request might be deleted when the subsequent cluster
                     // state, cluster health and indices stats requests execute. We have to distinguish two cases:
                     // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
@@ -117,11 +123,11 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) {
                     //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
                     // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve the
                     // index names with the same indices options that we used for the initial cluster state request (strictExpand).
-                    sendIndicesStatsRequest(indices, indicesOptions, includeUnloadedSegments, client,
+                    sendIndicesStatsRequest(indices, subRequestIndicesOptions, includeUnloadedSegments, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
-                    sendClusterStateRequest(indices, indicesOptions, local, masterNodeTimeout, client,
+                    sendClusterStateRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
-                    sendClusterHealthRequest(indices, indicesOptions, local, masterNodeTimeout, client,
+                    sendClusterHealthRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
                         ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
                 }
 
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index 85bf1a7500d46..e2867b1d715a8 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -105,6 +105,8 @@ testClusters.integTest {
   keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
   user username: "x_pack_rest_user", password: "x-pack-test-password"
+  user username: "cat_test_user", password: "cat-test-password", role: "cat_test_role"
   extraConfigFile nodeKey.name, nodeKey
   extraConfigFile nodeCert.name, nodeCert
+  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
 }
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
new file mode 100644
index 0000000000000..ab9171b7bc064
--- /dev/null
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/CatIndicesWithSecurityIT.java
@@ -0,0 +1,127 @@
+/*
+ *
+ *  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ *  * or more contributor license agreements. Licensed under the Elastic License;
+ *  * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+
+package org.elasticsearch.xpack.test.rest;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.matchesRegex;
+
+public class CatIndicesWithSecurityIT extends ESRestTestCase {
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("cat_test_user", new SecureString("cat-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    public void testHiddenIndexWithVisibleAlias() throws IOException {
+        // Create the index and alias
+        {
+            final Request createRequest = new Request("PUT", ".index_hidden");
+            createRequest.setJsonEntity(
+                "{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, \"aliases\": {\"index_allowed\": {}}}");
+            final Response createResponse = adminClient().performRequest(createRequest);
+            assertOK(createResponse);
+            ensureGreen("index_allowed");
+        }
+
+        // And it should be visible by a user that has permissions for only the alias
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
+        }
+    }
+
+    public void testHiddenIndexWithHiddenAlias() throws IOException {
+        // Create the index and alias
+        {
+            final Request createRequest = new Request("PUT", ".index_hidden");
+            createRequest.setJsonEntity("{\"settings\": {\"index.hidden\": true, \"number_of_replicas\":  0}, "
+                + "\"aliases\": {\"index_allowed\": {\"is_hidden\":  true}}}");
+            final Response createResponse = adminClient().performRequest(createRequest);
+            assertOK(createResponse);
+            ensureGreen("index_allowed");
+        }
+
+        // It should *not* be visible
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
+        }
+
+        // But we should be able to see the index if we ask for hidden things
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
+        }
+    }
+
+    public void testVisibleIndexWithHiddenAlias() throws IOException {
+        // Create the index and alias
+        {
+            final Request createRequest = new Request("PUT", "visible_index");
+            createRequest.setJsonEntity("{\"settings\": {\"number_of_replicas\":  0}, "
+                + "\"aliases\": {\"index_allowed\": {\"is_hidden\":  true}}}");
+            final Response createResponse = adminClient().performRequest(createRequest);
+            assertOK(createResponse);
+            ensureGreen("index_allowed");
+        }
+
+        // It should *not* be visible, because this user only has access through the *hidden* alias
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
+        }
+
+        // But we should be able to see the index if we ask for hidden things
+        {
+            final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
+            final Response catResponse = client().performRequest(catRequest);
+            assertOK(catResponse);
+
+            final String resp = EntityUtils.toString(catResponse.getEntity());
+            assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+visible_index.*$"));
+        }
+    }
+}
diff --git a/x-pack/plugin/src/test/resources/roles.yml b/x-pack/plugin/src/test/resources/roles.yml
new file mode 100644
index 0000000000000..5f8677860d079
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/roles.yml
@@ -0,0 +1,7 @@
+# Required for CatIndicesWithSecurityIT
+cat_test_role:
+  cluster:
+    - monitor
+  indices:
+    - names: [ "index_allowed" ]
+      privileges: [ "read", "write", "monitor" ]

From f6d3784b495af1988b202f5a0339917e8bdb3509 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Fri, 13 Mar 2020 12:28:17 -0700
Subject: [PATCH 034/285] migrate tests from MissingIT to agg tests (#53448)

Move the remaining tests for the missing aggregation into its
AggregatorTestCase out of its integration test and remove the IT
---
 .../search/aggregations/bucket/MissingIT.java | 184 --------------
 .../missing/MissingAggregatorTests.java       | 233 ++++++++++++++----
 2 files changed, 189 insertions(+), 228 deletions(-)
 delete mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
deleted file mode 100644
index ddd8c1920b99e..0000000000000
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.missing.Missing;
-import org.elasticsearch.search.aggregations.metrics.Avg;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.missing;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.core.IsNull.notNullValue;
-
-@ESIntegTestCase.SuiteScopeTestCase
-public class MissingIT extends ESIntegTestCase {
-
-    static int numDocs, numDocsMissing, numDocsUnmapped;
-
-    @Override
-    public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("idx")
-                .setMapping("tag", "type=keyword").get());
-        List<IndexRequestBuilder> builders = new ArrayList<>();
-        numDocs = randomIntBetween(5, 20);
-        numDocsMissing = randomIntBetween(1, numDocs - 1);
-        for (int i = 0; i < numDocsMissing; i++) {
-            builders.add(client().prepareIndex("idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("value", i)
-                    .endObject()));
-        }
-        for (int i = numDocsMissing; i < numDocs; i++) {
-            builders.add(client().prepareIndex("idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("tag", "tag1")
-                    .endObject()));
-        }
-
-        createIndex("unmapped_idx");
-        numDocsUnmapped = randomIntBetween(2, 5);
-        for (int i = 0; i < numDocsUnmapped; i++) {
-            builders.add(client().prepareIndex("unmapped_idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("value", i)
-                    .endObject()));
-        }
-
-        prepareCreate("empty_bucket_idx").setMapping("value", "type=integer").get();
-        for (int i = 0; i < 2; i++) {
-            builders.add(client().prepareIndex("empty_bucket_idx").setId(""+i).setSource(jsonBuilder()
-                    .startObject()
-                    .field("value", i*2)
-                    .endObject()));
-        }
-
-        indexRandom(true, builders);
-        ensureSearchable();
-    }
-
-    public void testUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("unmapped_idx")
-                .addAggregation(missing("missing_tag").field("tag"))
-                .get();
-
-        assertSearchResponse(response);
-
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsUnmapped));
-    }
-
-    public void testPartiallyUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "unmapped_idx")
-                .addAggregation(missing("missing_tag").field("tag"))
-                .get();
-
-        assertSearchResponse(response);
-
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
-    }
-
-    public void testSimple() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(missing("missing_tag").field("tag"))
-                .get();
-
-        assertSearchResponse(response);
-
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing));
-    }
-
-    public void testWithSubAggregation() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "unmapped_idx")
-                .addAggregation(missing("missing_tag").field("tag")
-                        .subAggregation(avg("avg_value").field("value")))
-                .get();
-
-        assertSearchResponse(response);
-
-        assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-
-        Missing missing = response.getAggregations().get("missing_tag");
-        assertThat(missing, notNullValue());
-        assertThat(missing.getName(), equalTo("missing_tag"));
-        assertThat(missing.getDocCount(), equalTo((long) numDocsMissing + numDocsUnmapped));
-        assertThat((long) ((InternalAggregation)missing).getProperty("_count"), equalTo((long) numDocsMissing + numDocsUnmapped));
-        assertThat(missing.getAggregations().asList().isEmpty(), is(false));
-
-        long sum = 0;
-        for (int i = 0; i < numDocsMissing; ++i) {
-            sum += i;
-        }
-        for (int i = 0; i < numDocsUnmapped; ++i) {
-            sum += i;
-        }
-        Avg avgValue = missing.getAggregations().get("avg_value");
-        assertThat(avgValue, notNullValue());
-        assertThat(avgValue.getName(), equalTo("avg_value"));
-        assertThat(avgValue.getValue(), equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
-        assertThat((double) ((InternalAggregation)missing).getProperty("avg_value.value"),
-                equalTo((double) sum / (numDocsMissing + numDocsUnmapped)));
-    }
-
-    public void testEmptyAggregation() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0)
-                        .subAggregation(missing("missing").field("value")))
-                .get();
-
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
-        Histogram histo = searchResponse.getAggregations().get("histo");
-        assertThat(histo, Matchers.notNullValue());
-        Histogram.Bucket bucket = histo.getBuckets().get(1);
-        assertThat(bucket, Matchers.notNullValue());
-
-        Missing missing = bucket.getAggregations().get("missing");
-        assertThat(missing, Matchers.notNullValue());
-        assertThat(missing.getName(), equalTo("missing"));
-        assertThat(missing.getDocCount(), is(0L));
-    }
-
-
-}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
index 04b1bbe10c4a7..3665fcded7280 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.search.aggregations.bucket.missing;
 
 import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -31,6 +33,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
@@ -44,6 +47,11 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
+import org.elasticsearch.search.aggregations.metrics.InternalSum;
+import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -53,6 +61,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -64,6 +73,10 @@
 import static java.util.Collections.singletonMap;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.common.lucene.search.Queries.newMatchAllQuery;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
+import static org.hamcrest.collection.IsEmptyCollection.empty;
 
 public class MissingAggregatorTests extends AggregatorTestCase {
 
@@ -91,9 +104,9 @@ public void testMatchNoDocs() throws IOException {
                     writer.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong())));
                 }
             },
-            internalMissing -> {
-                assertEquals(0, internalMissing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(0, missing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(missing));
             },
             singleton(fieldType)
         );
@@ -118,9 +131,9 @@ public void testMatchAllDocs() throws IOException {
                     writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
                 }
             },
-            internalMissing -> {
-                assertEquals(numDocs, internalMissing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(numDocs, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
             List.of(aggFieldType, anotherFieldType)
         );
@@ -152,9 +165,9 @@ public void testMatchSparse() throws IOException {
             newMatchAllQuery(),
             builder,
             writer -> writer.addDocuments(docs),
-            internalMissing -> {
-                assertEquals(finalDocsMissingAggField, internalMissing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingAggField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
             List.of(aggFieldType, anotherFieldType)
         );
@@ -191,11 +204,110 @@ public void testMatchSparseRangeField() throws IOException {
             newMatchAllQuery(),
             builder,
             writer -> writer.addDocuments(docs),
-            internalMissing -> {
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingAggField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
+            },
+            List.of(aggFieldType, anotherFieldType)
+        );
+    }
+
+    public void testSubAggregation() throws IOException {
+        final MappedFieldType missingAggFieldType = new KeywordFieldMapper.Builder("_name").fieldType();
+        missingAggFieldType.setName("missing_agg_field");
+        missingAggFieldType.setHasDocValues(true);
+        final MappedFieldType sumAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        sumAggFieldType.setName("sum_agg_field");
+
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("missing", null)
+            .field(missingAggFieldType.name())
+            .subAggregation(
+              new SumAggregationBuilder("sum")
+                .field(sumAggFieldType.name())
+            );
+
+        final int numDocs = randomIntBetween(10, 20);
+        int docsMissingAggField = 0;
+        long expectedSum = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            final Set<IndexableField> doc = new HashSet<>();
+            final long sumFieldValue = randomLongBetween(0, 1000);
+            doc.add(new SortedNumericDocValuesField(sumAggFieldType.name(), sumFieldValue));
+            if (randomBoolean()) {
+                doc.add(new SortedDocValuesField(missingAggFieldType.name(), new BytesRef(randomUnicodeOfLengthBetween(2, 20))));
+            } else {
+                docsMissingAggField++;
+                expectedSum += sumFieldValue;
+            }
+            docs.add(doc);
+        }
+        final int finalDocsMissingAggField = docsMissingAggField;
+        final long finalExpectedSum = expectedSum;
+
+        testCase(
+            newMatchAllQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            (InternalMissing internalMissing) -> {
                 assertEquals(finalDocsMissingAggField, internalMissing.getDocCount());
                 assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+
+                assertThat(internalMissing.getAggregations().asList(), not(empty()));
+                final InternalSum internalSum = internalMissing.getAggregations().get("sum");
+                assertEquals(finalExpectedSum, internalSum.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(internalSum));
             },
-            List.of(aggFieldType, anotherFieldType)
+            List.of(missingAggFieldType, sumAggFieldType)
+        );
+    }
+
+    public void testEmptyBucket() throws IOException {
+        final MappedFieldType histoAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        histoAggFieldType.setName("histo_agg_field");
+        final MappedFieldType missingAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        missingAggFieldType.setName("missing_agg_field");
+
+        final HistogramAggregationBuilder builder = new HistogramAggregationBuilder("histo")
+            .field(histoAggFieldType.name())
+            .interval(1)
+            .minDocCount(0)
+            .subAggregation(
+              new MissingAggregationBuilder("missing", null)
+                .field(missingAggFieldType.name())
+            );
+
+        testCaseWithReduce(
+            newMatchAllQuery(),
+            builder,
+            writer -> {
+                writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 0)));
+                writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 2)));
+            },
+            (InternalHistogram histogram) -> {
+                assertThat(histogram.getBuckets(), hasSize(3));
+
+                {
+                    assertThat(histogram.getBuckets().get(0), notNullValue());
+                    final InternalMissing missing = histogram.getBuckets().get(0).getAggregations().get("missing");
+                    assertEquals(1, missing.getDocCount());
+                    assertTrue(AggregationInspectionHelper.hasValue(missing));
+                }
+                {
+                    assertThat(histogram.getBuckets().get(1), notNullValue());
+                    final InternalMissing missing = histogram.getBuckets().get(1).getAggregations().get("missing");
+                    assertEquals(0, missing.getDocCount());
+                    assertFalse(AggregationInspectionHelper.hasValue(missing));
+                }
+                {
+                    assertThat(histogram.getBuckets().get(2), notNullValue());
+                    final InternalMissing missing = histogram.getBuckets().get(2).getAggregations().get("missing");
+                    assertEquals(1, missing.getDocCount());
+                    assertTrue(AggregationInspectionHelper.hasValue(missing));
+                }
+            },
+            List.of(histoAggFieldType, missingAggFieldType),
+            true
         );
     }
 
@@ -215,9 +327,9 @@ public void testUnmappedWithoutMissingParam() throws IOException {
                     writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
                 }
             },
-            internalMissing -> {
-                assertEquals(numDocs, internalMissing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(numDocs, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
             singleton(aggFieldType)
         );
@@ -240,14 +352,44 @@ public void testUnmappedWithMissingParam() throws IOException {
                     writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong())));
                 }
             },
-            internalMissing -> {
-                assertEquals(0, internalMissing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(0, missing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(missing));
             },
             singleton(aggFieldType)
         );
     }
 
+    public void testSingleValuedFieldPartiallyUnmapped() throws IOException {
+        final int numDocs = randomIntBetween(10, 20);
+
+        final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType();
+        fieldType.setName("field");
+        final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null)
+            .field(fieldType.name());
+
+        try (Directory mappedDirectory = newDirectory(); Directory unmappedDirectory = newDirectory()) {
+            try (RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory)) {
+                for (int i = 0; i < numDocs; i++) {
+                    mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong())));
+                }
+            }
+
+            new RandomIndexWriter(random(), unmappedDirectory).close();
+
+            try (IndexReader mappedReader = DirectoryReader.open(mappedDirectory);
+                 IndexReader unmappedReader = DirectoryReader.open(unmappedDirectory);
+                 MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) {
+
+                final IndexSearcher searcher = newSearcher(multiReader, true, true);
+
+                final InternalMissing internalMissing = searchAndReduce(searcher, newMatchAllQuery(), builder, fieldType);
+                assertEquals(0, internalMissing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
+            }
+        }
+    }
+
     public void testMissingParam() throws IOException {
         final int numDocs = randomIntBetween(10, 20);
 
@@ -268,9 +410,9 @@ public void testMissingParam() throws IOException {
                     writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())));
                 }
             },
-            internalMissing -> {
-                assertEquals(0, internalMissing.getDocCount());
-                assertFalse(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(0, missing.getDocCount());
+                assertFalse(AggregationInspectionHelper.hasValue(missing));
             },
             List.of(aggFieldType, anotherFieldType)
         );
@@ -306,9 +448,9 @@ public void testMultiValuedField() throws IOException {
             newMatchAllQuery(),
             builder,
             writer -> writer.addDocuments(docs),
-            internalMissing -> {
-                assertEquals(finalDocsMissingAggField, internalMissing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingAggField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
             List.of(aggFieldType, anotherFieldType)
         );
@@ -349,9 +491,9 @@ private void valueScriptTestCase(Script script) throws IOException {
             newMatchAllQuery(),
             builder,
             writer -> writer.addDocuments(docs),
-            internalMissing -> {
-                assertEquals(finalDocsMissingField, internalMissing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsMissingField, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
             List.of(aggFieldType, anotherFieldType)
         );
@@ -395,29 +537,32 @@ private void fieldScriptTestCase(Script script, long threshold) throws IOExcepti
             newMatchAllQuery(),
             builder,
             writer -> writer.addDocuments(docs),
-            internalMissing -> {
-                assertEquals(finalDocsBelowThreshold, internalMissing.getDocCount());
-                assertTrue(AggregationInspectionHelper.hasValue(internalMissing));
+            (InternalMissing missing) -> {
+                assertEquals(finalDocsBelowThreshold, missing.getDocCount());
+                assertTrue(AggregationInspectionHelper.hasValue(missing));
             },
             singleton(aggFieldType)
         );
     }
 
-    private void testCase(Query query,
-                          MissingAggregationBuilder builder,
-                          CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
-                          Consumer<InternalMissing> verify,
-                          Collection<MappedFieldType> fieldTypes) throws IOException {
+    private <B extends AggregationBuilder, I extends InternalAggregation> void testCase(
+            Query query,
+            B builder,
+            CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
+            Consumer<I> verify,
+            Collection<MappedFieldType> fieldTypes) throws IOException {
+
         testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false);
         testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true);
     }
 
-    private void testCaseWithReduce(Query query,
-                                    MissingAggregationBuilder builder,
-                                    CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
-                                    Consumer<InternalMissing> verify,
-                                    Collection<MappedFieldType> fieldTypes,
-                                    boolean reduced) throws IOException {
+    private <B extends AggregationBuilder, I extends InternalAggregation> void testCaseWithReduce(
+            Query query,
+            B builder,
+            CheckedConsumer<RandomIndexWriter, IOException> writeIndex,
+            Consumer<I> verify,
+            Collection<MappedFieldType> fieldTypes,
+            boolean reduced) throws IOException {
 
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
@@ -427,13 +572,13 @@ private void testCaseWithReduce(Query query,
             try (IndexReader indexReader = DirectoryReader.open(directory)) {
                 final IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
                 final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]);
-                final InternalMissing missing;
+                final I result;
                 if (reduced) {
-                    missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray);
+                    result = searchAndReduce(indexSearcher, query, builder, fieldTypesArray);
                 } else {
-                    missing = search(indexSearcher, query, builder, fieldTypesArray);
+                    result = search(indexSearcher, query, builder, fieldTypesArray);
                 }
-                verify.accept(missing);
+                verify.accept(result);
             }
         }
     }

From 9e48ca75c37e9d2081d05097a11255baf1b78568 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Fri, 13 Mar 2020 13:09:22 -0700
Subject: [PATCH 035/285] Lower docker-compose version to allow for older
 Docker engine support (#53526)

---
 distribution/docker/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml
index 245056382f304..08e39ff8dd750 100644
--- a/distribution/docker/docker-compose.yml
+++ b/distribution/docker/docker-compose.yml
@@ -1,5 +1,5 @@
 # Only used for testing the docker images
-version: '3.7'
+version: '3.4'
 services:
   elasticsearch-default-1:
     image: elasticsearch:test

From 8ccdaa3a354573a795a39f5f6bf17bd4797150ae Mon Sep 17 00:00:00 2001
From: Tim Brooks <tim@uncontended.net>
Date: Fri, 13 Mar 2020 15:01:01 -0600
Subject: [PATCH 036/285] Align remote info api with new settings (#53441)

Currently the remote info api has added a number of possible fields
(proxy, num_socket_connections, etc) that are available in proxy mode.
These fields are not aligned with what the settings are named. This
commit modifies this API to align with the settings.
---
 .../org/elasticsearch/client/cluster/ProxyModeInfo.java     | 6 +++---
 .../elasticsearch/client/cluster/RemoteConnectionInfo.java  | 6 +++---
 docs/reference/cluster/remote-info.asciidoc                 | 6 +++---
 .../resources/rest-api-spec/test/multi_cluster/20_info.yml  | 6 +++---
 .../elasticsearch/transport/ProxyConnectionStrategy.java    | 6 +++---
 .../transport/RemoteClusterConnectionTests.java             | 4 ++--
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
index 0fc4f240eb8ef..79758ec709c63 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
@@ -23,9 +23,9 @@
 
 public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
     static final String NAME = "proxy";
-    static final String ADDRESS = "address";
-    static final String NUM_SOCKETS_CONNECTED = "num_sockets_connected";
-    static final String MAX_SOCKET_CONNECTIONS = "max_socket_connections";
+    static final String PROXY_ADDRESS = "proxy_address";
+    static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
+    static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
     private final String address;
     private final int maxSocketConnections;
     private final int numSocketsConnected;
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
index 2bf99c61085c4..474b991cd434a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
@@ -66,9 +66,9 @@ public final class RemoteConnectionInfo {
         PARSER.declareString(constructorArg(), new ParseField(INITIAL_CONNECT_TIMEOUT));
         PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
 
-        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.ADDRESS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_SOCKET_CONNECTIONS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_SOCKETS_CONNECTED));
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
 
         PARSER.declareStringArray(optionalConstructorArg(), new ParseField(SniffModeInfo.SEEDS));
         PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.MAX_CONNECTIONS_PER_CLUSTER));
diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc
index 8e81a0444886c..05931ea2d6a16 100644
--- a/docs/reference/cluster/remote-info.asciidoc
+++ b/docs/reference/cluster/remote-info.asciidoc
@@ -51,13 +51,13 @@ by the configured remote cluster alias.
     Maximum number of connections maintained for the remote cluster when sniff
     mode is configured.
 
-`address`::
+`proxy_address`::
     Address for remote connections when proxy mode is configured.
 
-`num_sockets_connected`::
+`num_proxy_sockets_connected`::
     Number of open socket connections to the remote cluster when proxy mode
     is configured.
 
-`max_socket_connections`::
+`max_proxy_socket_connections`::
     The maximum number of socket connections to the remote cluster when proxy
     mode is configured.
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
index 0e5236f9b1171..f8a31c5ec9214 100644
--- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml
@@ -80,9 +80,9 @@
       cluster.remote_info: {}
 
   - match: { test_remote_cluster.connected: true }
-  - match: { test_remote_cluster.address: $remote_ip }
-  - gt:    { test_remote_cluster.num_sockets_connected: 0}
-  - match: { test_remote_cluster.max_socket_connections: 10}
+  - match: { test_remote_cluster.proxy_address: $remote_ip }
+  - gt:    { test_remote_cluster.num_proxy_sockets_connected: 0}
+  - match: { test_remote_cluster.max_proxy_socket_connections: 10}
   - match: { test_remote_cluster.initial_connect_timeout: "30s" }
   - match: { test_remote_cluster.mode: "proxy" }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index 5ce6de7998801..2d314a591b57a 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -272,9 +272,9 @@ private ProxyModeInfo(StreamInput input) throws IOException {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field("address", address);
-            builder.field("num_sockets_connected", numSocketsConnected);
-            builder.field("max_socket_connections", maxSocketConnections);
+            builder.field("proxy_address", address);
+            builder.field("num_proxy_sockets_connected", numSocketsConnected);
+            builder.field("max_proxy_socket_connections", maxSocketConnections);
             return builder;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index f8a4fe4b9429b..2e3102cb7e926 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -417,8 +417,8 @@ public void testRenderConnectionInfoXContent() throws IOException {
                 "\"num_nodes_connected\":2,\"max_connections_per_cluster\":3,\"initial_connect_timeout\":\"30m\"," +
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         } else {
-            assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"address\":\"seed:1\"," +
-                "\"num_sockets_connected\":16,\"max_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
+            assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"proxy_address\":\"seed:1\"," +
+                "\"num_proxy_sockets_connected\":16,\"max_proxy_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         }
     }

From 1cdb5b3ecfd6c09a1588fdae90e4422f63c15768 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 13 Mar 2020 14:51:12 -0700
Subject: [PATCH 037/285] Move AStatement mutable members into isolated
 Input/Output objects (#53348)

This is the follow up to #53075, isolating the mutable data on the
statement nodes as the referenced change did the expression nodes.

This is a step toward the long-term goal of making the "user" trees nodes
immutable. This change isolates the mutable data for statement nodes in
the "user" tree during the semantic (analysis) phase by moving the
mutable data into Input and Output objects. These objects are created
locally during the semantic phase to share information required for
semantic checking between parent-child nodes.

Note that for this change, Input and Output are still stored in a mutable
way on the statement nodes. This will not be the case once the semantic
(analysis) phase and ir generation (write) phase are combined in a future
change.

Relates #49869
---
 .../painless/node/AExpression.java            |   1 +
 .../painless/node/AStatement.java             | 131 ++++++++++--------
 .../elasticsearch/painless/node/ELambda.java  |   9 +-
 .../elasticsearch/painless/node/SBlock.java   |  36 +++--
 .../elasticsearch/painless/node/SBreak.java   |  17 ++-
 .../elasticsearch/painless/node/SCatch.java   |  32 +++--
 .../painless/node/SContinue.java              |  17 ++-
 .../painless/node/SDeclBlock.java             |  11 +-
 .../painless/node/SDeclaration.java           |   9 +-
 .../org/elasticsearch/painless/node/SDo.java  |  24 ++--
 .../elasticsearch/painless/node/SEach.java    |  22 +--
 .../painless/node/SExpression.java            |  24 ++--
 .../org/elasticsearch/painless/node/SFor.java |  29 ++--
 .../painless/node/SFunction.java              |   9 +-
 .../org/elasticsearch/painless/node/SIf.java  |  22 +--
 .../elasticsearch/painless/node/SIfElse.java  |  43 +++---
 .../elasticsearch/painless/node/SReturn.java  |  15 +-
 .../painless/node/SSubEachArray.java          |   7 +-
 .../painless/node/SSubEachIterable.java       |   7 +-
 .../elasticsearch/painless/node/SThrow.java   |  15 +-
 .../org/elasticsearch/painless/node/STry.java |  49 ++++---
 .../elasticsearch/painless/node/SWhile.java   |  26 ++--
 22 files changed, 337 insertions(+), 218 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 13c79d6b8ded6..189cb94dcd70b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -91,6 +91,7 @@ public static class Output {
     AExpression prefix;
 
     // TODO: remove placeholders once analysis and write are combined into build
+    // TODO: https://github.com/elastic/elasticsearch/issues/53561
     // This are used to support the transition from a mutable to immutable state.
     // Currently, the IR tree is built during the user tree "write" phase, so
     // these are stored on the node to set during the "semantic" phase and then
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index 3ad2b0b8a0800..d8d12d8a14698 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -30,66 +30,77 @@
  */
 public abstract class AStatement extends ANode {
 
-    /**
-     * Set to true when the final statement in an {@link SClass} is reached.
-     * Used to determine whether or not an auto-return is necessary.
-     */
-    boolean lastSource = false;
-
-    /**
-     * Set to true when a loop begins.  Used by {@link SBlock} to help determine
-     * when the final statement of a loop is reached.
-     */
-    boolean beginLoop = false;
-
-    /**
-     * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
-     * to determine if a break/continue statement is legal.
-     */
-    boolean inLoop = false;
-
-    /**
-     * Set to true when on the last statement of a loop.  Used by {@link SContinue}
-     * to prevent extraneous continue statements.
-     */
-    boolean lastLoop = false;
-
-    /**
-     * Set to true if a statement would cause the method to exit.  Used to
-     * determine whether or not an auto-return is necessary.
-     */
-    boolean methodEscape = false;
-
-    /**
-     * Set to true if a statement would cause a loop to exit.  Used to
-     * prevent unreachable statements.
-     */
-    boolean loopEscape = false;
-
-    /**
-     * Set to true if all current paths escape from the current {@link SBlock}.
-     * Used during the analysis phase to prevent unreachable statements and
-     * the writing phase to prevent extraneous bytecode gotos from being written.
-     */
-    boolean allEscape = false;
-
-    /**
-     * Set to true if any continue statement occurs in a loop.  Used to prevent
-     * unnecessary infinite loops.
-     */
-    boolean anyContinue = false;
+    public static class Input {
+
+        /**
+         * Set to true when the final statement in an {@link SClass} is reached.
+         * Used to determine whether or not an auto-return is necessary.
+         */
+        boolean lastSource = false;
+
+        /**
+         * Set to true when a loop begins.  Used by {@link SBlock} to help determine
+         * when the final statement of a loop is reached.
+         */
+        boolean beginLoop = false;
+
+        /**
+         * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
+         * to determine if a break/continue statement is legal.
+         */
+        boolean inLoop = false;
+
+        /**
+         * Set to true when on the last statement of a loop.  Used by {@link SContinue}
+         * to prevent extraneous continue statements.
+         */
+        boolean lastLoop = false;
+    }
 
-    /**
-     * Set to true if any break statement occurs in a loop.  Used to prevent
-     * extraneous loops.
-     */
-    boolean anyBreak = false;
+    public static class Output {
+
+        /**
+         * Set to true if a statement would cause the method to exit.  Used to
+         * determine whether or not an auto-return is necessary.
+         */
+        boolean methodEscape = false;
+
+        /**
+         * Set to true if a statement would cause a loop to exit.  Used to
+         * prevent unreachable statements.
+         */
+        boolean loopEscape = false;
+
+        /**
+         * Set to true if all current paths escape from the current {@link SBlock}.
+         * Used during the analysis phase to prevent unreachable statements and
+         * the writing phase to prevent extraneous bytecode gotos from being written.
+         */
+        boolean allEscape = false;
+
+        /**
+         * Set to true if any continue statement occurs in a loop.  Used to prevent
+         * unnecessary infinite loops.
+         */
+        boolean anyContinue = false;
+
+        /**
+         * Set to true if any break statement occurs in a loop.  Used to prevent
+         * extraneous loops.
+         */
+        boolean anyBreak = false;
+
+        /**
+         * Set to the approximate number of statements in a loop block to prevent
+         * infinite loops during runtime.
+         */
+        int statementCount = 0;
+    }
 
-    /**
-     * Set to the approximate number of statements in a loop block to prevent
-     * infinite loops during runtime.
-     */
-    int statementCount = 0;
+    // TODO: remove placeholders once analysis and write are combined into build
+    // TODO: https://github.com/elastic/elasticsearch/issues/53561
+    Input input;
+    Output output;
 
     /**
      * Standard constructor with location used for error tracking.
@@ -101,7 +112,9 @@ public abstract class AStatement extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(ScriptRoot scriptRoot, Scope scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        throw new UnsupportedOperationException();
+    }
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 4619ff6522724..a4cb19d048c62 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -165,17 +165,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         if (block.statements.isEmpty()) {
             throw createError(new IllegalArgumentException("cannot generate empty lambda"));
         }
-        block.lastSource = true;
-        block.analyze(scriptRoot, lambdaScope);
-        captures = new ArrayList<>(lambdaScope.getCaptures());
+        AStatement.Input blockInput = new AStatement.Input();
+        blockInput.lastSource = true;
+        AStatement.Output blockOutput = block.analyze(scriptRoot, lambdaScope, blockInput);
 
-        if (block.methodEscape == false) {
+        if (blockOutput.methodEscape == false) {
             throw createError(new IllegalArgumentException("not all paths return a value for lambda"));
         }
 
         maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
 
         // prepend capture list to lambda's arguments
+        captures = new ArrayList<>(lambdaScope.getCaptures());
         this.typeParameters = new ArrayList<>(captures.size() + typeParameters.size());
         parameterNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 20ee3ec572d57..8e6db345e0b30 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -44,7 +44,10 @@ public SBlock(Location location, List<AStatement> statements) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
         }
@@ -54,22 +57,25 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         for (AStatement statement : statements) {
             // Note that we do not need to check after the last statement because
             // there is no statement that can be unreachable after the last.
-            if (allEscape) {
+            if (output.allEscape) {
                 throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
-            statement.inLoop = inLoop;
-            statement.lastSource = lastSource && statement == last;
-            statement.lastLoop = (beginLoop || lastLoop) && statement == last;
-            statement.analyze(scriptRoot, scope);
-
-            methodEscape = statement.methodEscape;
-            loopEscape = statement.loopEscape;
-            allEscape = statement.allEscape;
-            anyContinue |= statement.anyContinue;
-            anyBreak |= statement.anyBreak;
-            statementCount += statement.statementCount;
+            Input statementInput = new Input();
+            statementInput.inLoop = input.inLoop;
+            statementInput.lastSource = input.lastSource && statement == last;
+            statementInput.lastLoop = (input.beginLoop || input.lastLoop) && statement == last;
+            Output statementOutput = statement.analyze(scriptRoot, scope, statementInput);
+
+            output.methodEscape = statementOutput.methodEscape;
+            output.loopEscape = statementOutput.loopEscape;
+            output.allEscape = statementOutput.allEscape;
+            output.anyContinue |= statementOutput.anyContinue;
+            output.anyBreak |= statementOutput.anyBreak;
+            output.statementCount += statementOutput.statementCount;
         }
+
+        return output;
     }
 
     @Override
@@ -81,8 +87,8 @@ BlockNode write(ClassNode classNode) {
         }
 
         blockNode.setLocation(location);
-        blockNode.setAllEscape(allEscape);
-        blockNode.setStatementCount(statementCount);
+        blockNode.setAllEscape(output.allEscape);
+        blockNode.setStatementCount(output.statementCount);
 
         return blockNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 8c2d2d3c03403..4b036153319ea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -35,15 +35,20 @@ public SBreak(Location location) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!inLoop) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
         }
 
-        loopEscape = true;
-        allEscape = true;
-        anyBreak = true;
-        statementCount = 1;
+        output.loopEscape = true;
+        output.allEscape = true;
+        output.anyBreak = true;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index ed960776a19f2..f00c0b18a70fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -46,8 +46,11 @@ public SCatch(Location location, DType baseException, SDeclaration declaration,
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        declaration.analyze(scriptRoot, scope);
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        declaration.analyze(scriptRoot, scope, new Input());
 
         Class<?> baseType = baseException.resolveType(scriptRoot.getPainlessLookup()).getType();
         Class<?> type = scope.getVariable(location, declaration.name).getType();
@@ -59,18 +62,21 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         if (block != null) {
-            block.lastSource = lastSource;
-            block.inLoop = inLoop;
-            block.lastLoop = lastLoop;
-            block.analyze(scriptRoot, scope);
-
-            methodEscape = block.methodEscape;
-            loopEscape = block.loopEscape;
-            allEscape = block.allEscape;
-            anyContinue = block.anyContinue;
-            anyBreak = block.anyBreak;
-            statementCount = block.statementCount;
+            Input blockInput = new Input();
+            blockInput.lastSource = input.lastSource;
+            blockInput.inLoop = input.inLoop;
+            blockInput.lastLoop = input.lastLoop;
+            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+
+            output.methodEscape = blockOutput.methodEscape;
+            output.loopEscape = blockOutput.loopEscape;
+            output.allEscape = blockOutput.allEscape;
+            output.anyContinue = blockOutput.anyContinue;
+            output.anyBreak = blockOutput.anyBreak;
+            output.statementCount = blockOutput.statementCount;
         }
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index fe19365095777..1f5c752f8ccef 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -35,18 +35,23 @@ public SContinue(Location location) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
-        if (!inLoop) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
+        if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
         }
 
-        if (lastLoop) {
+        if (input.lastLoop) {
             throw createError(new IllegalArgumentException("Extraneous continue statement."));
         }
 
-        allEscape = true;
-        anyContinue = true;
-        statementCount = 1;
+        output.allEscape = true;
+        output.anyContinue = true;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index fa85f57346908..47bbe0d123ecb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -44,12 +44,17 @@ public SDeclBlock(Location location, List<SDeclaration> declarations) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         for (SDeclaration declaration : declarations) {
-            declaration.analyze(scriptRoot, scope);
+            declaration.analyze(scriptRoot, scope, new Input());
         }
 
-        statementCount = declarations.size();
+        output.statementCount = declarations.size();
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 538de2751611c..2a28b4d029558 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -48,18 +48,23 @@ public SDeclaration(Location location, DType type, String name, boolean requires
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         DResolvedType resolvedType = type.resolveType(scriptRoot.getPainlessLookup());
         type = resolvedType;
 
         if (expression != null) {
-            Input expressionInput = new Input();
+            AExpression.Input expressionInput = new AExpression.Input();
             expressionInput.expected = resolvedType.getType();
             expression.analyze(scriptRoot, scope, expressionInput);
             expression.cast();
         }
 
         scope.defineVariable(location, resolvedType.getType(), name, false);
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 89c5aa4be4ce1..8d31f325a049f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -45,18 +45,22 @@ public SDo(Location location, SBlock block, AExpression condition) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         scope = scope.newLocalScope();
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
-        block.beginLoop = true;
-        block.inLoop = true;
-        block.analyze(scriptRoot, scope);
+        Input blockInput = new Input();
+        blockInput.beginLoop = true;
+        blockInput.inLoop = true;
+        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
 
-        if (block.loopEscape && !block.anyContinue) {
+        if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
@@ -72,13 +76,15 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                 throw createError(new IllegalArgumentException("Extraneous do while loop."));
             }
 
-            if (!block.anyBreak) {
-                methodEscape = true;
-                allEscape = true;
+            if (blockOutput.anyBreak == false) {
+                output.methodEscape = true;
+                output.allEscape = true;
             }
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 0772331d11b64..b5c439233b64d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -53,7 +53,10 @@ public SEach(Location location, String type, String name, AExpression expression
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, new AExpression.Input());
         expression.input.expected = expressionOutput.actual;
         expression.cast();
@@ -76,22 +79,25 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
         }
 
-        sub.analyze(scriptRoot, scope);
+        sub.analyze(scriptRoot, scope, input);
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous for each loop."));
         }
 
-        block.beginLoop = true;
-        block.inLoop = true;
-        block.analyze(scriptRoot, scope);
-        block.statementCount = Math.max(1, block.statementCount);
+        Input blockInput = new Input();
+        blockInput.beginLoop = true;
+        blockInput.inLoop = true;
+        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
 
-        if (block.loopEscape && !block.anyContinue) {
+        if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous for loop."));
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 41a6b1757935d..f04cefa622acd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -44,36 +44,40 @@ public SExpression(Location location, AExpression expression) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+
         Class<?> rtnType = scope.getReturnType();
         boolean isVoid = rtnType == void.class;
 
         AExpression.Input expressionInput = new AExpression.Input();
-        expressionInput.read = lastSource && !isVoid;
+        expressionInput.read = input.lastSource && !isVoid;
         AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
 
-
-        if ((lastSource == false || isVoid) && expressionOutput.statement == false) {
+        if ((input.lastSource == false || isVoid) && expressionOutput.statement == false) {
             throw createError(new IllegalArgumentException("Not a statement."));
         }
 
-        boolean rtn = lastSource && isVoid == false && expressionOutput.actual != void.class;
+        boolean rtn = input.lastSource && isVoid == false && expressionOutput.actual != void.class;
 
         expression.input.expected = rtn ? rtnType : expressionOutput.actual;
         expression.input.internal = rtn;
         expression.cast();
 
-        methodEscape = rtn;
-        loopEscape = rtn;
-        allEscape = rtn;
-        statementCount = 1;
+        output = new Output();
+        output.methodEscape = rtn;
+        output.loopEscape = rtn;
+        output.allEscape = rtn;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
     StatementNode write(ClassNode classNode) {
         ExpressionNode expressionNode = expression.cast(expression.write(classNode));
 
-        if (methodEscape) {
+        if (output.methodEscape) {
             ReturnNode returnNode = new ReturnNode();
 
             returnNode.setExpressionNode(expressionNode);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 803e9f493ac69..241c50b8fe2b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -52,12 +52,14 @@ public SFor(Location location, ANode initializer, AExpression condition, AExpres
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+
         scope = scope.newLocalScope();
 
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(scriptRoot, scope);
+                ((SDeclBlock)initializer).analyze(scriptRoot, scope, new Input());
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
@@ -110,25 +112,30 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             afterthought.cast();
         }
 
+        output = new Output();
+
         if (block != null) {
-            block.beginLoop = true;
-            block.inLoop = true;
+            Input blockInput = new Input();
+            blockInput.beginLoop = true;
+            blockInput.inLoop = true;
 
-            block.analyze(scriptRoot, scope);
+            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
 
-            if (block.loopEscape && !block.anyContinue) {
+            if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
             }
 
-            if (continuous && !block.anyBreak) {
-                methodEscape = true;
-                allEscape = true;
+            if (continuous && blockOutput.anyBreak == false) {
+                output.methodEscape = true;
+                output.allEscape = true;
             }
 
-            block.statementCount = Math.max(1, block.statementCount);
+            blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 2ae9ef8d5d78f..96bf040099a17 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.painless.ir.ReturnNode;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.node.AStatement.Input;
+import org.elasticsearch.painless.node.AStatement.Output;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.lang.invoke.MethodType;
@@ -134,9 +136,10 @@ void analyze(ScriptRoot scriptRoot) {
             throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
         }
 
-        block.lastSource = true;
-        block.analyze(scriptRoot, functionScope.newLocalScope());
-        methodEscape = block.methodEscape;
+        Input blockInput = new Input();
+        blockInput.lastSource = true;
+        Output blockOutput = block.analyze(scriptRoot, functionScope.newLocalScope(), blockInput);
+        methodEscape = blockOutput.methodEscape;
 
         if (methodEscape == false && isAutoReturnEnabled == false && returnType != void.class) {
             throw createError(new IllegalArgumentException("not all paths provide a return value " +
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 518c5dbac9e09..a7d738670ec6f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -43,7 +43,10 @@ public SIf(Location location, AExpression condition, SBlock ifblock) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
         condition.analyze(scriptRoot, scope, conditionInput);
@@ -57,15 +60,18 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
-        ifblock.lastSource = lastSource;
-        ifblock.inLoop = inLoop;
-        ifblock.lastLoop = lastLoop;
+        Input ifblockInput = new Input();
+        ifblockInput.lastSource = input.lastSource;
+        ifblockInput.inLoop = input.inLoop;
+        ifblockInput.lastLoop = input.lastLoop;
+
+        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
 
-        ifblock.analyze(scriptRoot, scope.newLocalScope());
+        output.anyContinue = ifblockOutput.anyContinue;
+        output.anyBreak = ifblockOutput.anyBreak;
+        output.statementCount = ifblockOutput.statementCount;
 
-        anyContinue = ifblock.anyContinue;
-        anyBreak = ifblock.anyBreak;
-        statementCount = ifblock.statementCount;
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index d45a18f8d0eac..f433db860ccfb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -48,7 +48,10 @@ public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
         condition.analyze(scriptRoot, scope, conditionInput);
@@ -62,32 +65,36 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
-        ifblock.lastSource = lastSource;
-        ifblock.inLoop = inLoop;
-        ifblock.lastLoop = lastLoop;
+        Input ifblockInput = new Input();
+        ifblockInput.lastSource = input.lastSource;
+        ifblockInput.inLoop = input.inLoop;
+        ifblockInput.lastLoop = input.lastLoop;
 
-        ifblock.analyze(scriptRoot, scope.newLocalScope());
+        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
 
-        anyContinue = ifblock.anyContinue;
-        anyBreak = ifblock.anyBreak;
-        statementCount = ifblock.statementCount;
+        output.anyContinue = ifblockOutput.anyContinue;
+        output.anyBreak = ifblockOutput.anyBreak;
+        output.statementCount = ifblockOutput.statementCount;
 
         if (elseblock == null) {
             throw createError(new IllegalArgumentException("Extraneous else statement."));
         }
 
-        elseblock.lastSource = lastSource;
-        elseblock.inLoop = inLoop;
-        elseblock.lastLoop = lastLoop;
+        Input elseblockInput = new Input();
+        elseblockInput.lastSource = input.lastSource;
+        elseblockInput.inLoop = input.inLoop;
+        elseblockInput.lastLoop = input.lastLoop;
+
+        Output elseblockOutput = elseblock.analyze(scriptRoot, scope.newLocalScope(), elseblockInput);
 
-        elseblock.analyze(scriptRoot, scope.newLocalScope());
+        output.methodEscape = ifblockOutput.methodEscape && elseblockOutput.methodEscape;
+        output.loopEscape = ifblockOutput.loopEscape && elseblockOutput.loopEscape;
+        output.allEscape = ifblockOutput.allEscape && elseblockOutput.allEscape;
+        output.anyContinue |= elseblockOutput.anyContinue;
+        output.anyBreak |= elseblockOutput.anyBreak;
+        output.statementCount = Math.max(ifblockOutput.statementCount, elseblockOutput.statementCount);
 
-        methodEscape = ifblock.methodEscape && elseblock.methodEscape;
-        loopEscape = ifblock.loopEscape && elseblock.loopEscape;
-        allEscape = ifblock.allEscape && elseblock.allEscape;
-        anyContinue |= elseblock.anyContinue;
-        anyBreak |= elseblock.anyBreak;
-        statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index aa4a198d8efdf..23a5d4183edbc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -40,7 +40,10 @@ public SReturn(Location location, AExpression expression) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (expression == null) {
             if (scope.getReturnType() != void.class) {
                 throw location.createError(new ClassCastException("Cannot cast from " +
@@ -55,11 +58,13 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
             expression.cast();
         }
 
-        methodEscape = true;
-        loopEscape = true;
-        allEscape = true;
+        output.methodEscape = true;
+        output.loopEscape = true;
+        output.allEscape = true;
+
+        output.statementCount = 1;
 
-        statementCount = 1;
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index 32aca4587c311..5e1846457d515 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -52,13 +52,18 @@ final class SSubEachArray extends AStatement {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
         array = scope.defineInternalVariable(location, expression.output.actual, "array" + location.getOffset(), true);
         index = scope.defineInternalVariable(location, int.class, "index" + location.getOffset(), true);
         indexed = expression.output.actual.getComponentType();
         cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index cb18b898dc357..4198452ff8e79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -57,7 +57,10 @@ final class SSubEachIterable extends AStatement {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
         iterator = scope.defineInternalVariable(location, Iterator.class, "itr" + location.getOffset(), true);
@@ -74,6 +77,8 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index d897ba8653d1f..86b40fafa378b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -41,16 +41,21 @@ public SThrow(Location location, AExpression expression) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         AExpression.Input expressionInput = new AExpression.Input();
         expressionInput.expected = Exception.class;
         expression.analyze(scriptRoot, scope, expressionInput);
         expression.cast();
 
-        methodEscape = true;
-        loopEscape = true;
-        allEscape = true;
-        statementCount = 1;
+        output.methodEscape = true;
+        output.loopEscape = true;
+        output.allEscape = true;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 6c73b625a64ac..7798fd13b13a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -46,42 +46,49 @@ public STry(Location location, SBlock block, List<SCatch> catches) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
         }
 
-        block.lastSource = lastSource;
-        block.inLoop = inLoop;
-        block.lastLoop = lastLoop;
+        Input blockInput = new Input();
+        blockInput.lastSource = input.lastSource;
+        blockInput.inLoop = input.inLoop;
+        blockInput.lastLoop = input.lastLoop;
 
-        block.analyze(scriptRoot, scope.newLocalScope());
+        Output blockOutput = block.analyze(scriptRoot, scope.newLocalScope(), blockInput);
 
-        methodEscape = block.methodEscape;
-        loopEscape = block.loopEscape;
-        allEscape = block.allEscape;
-        anyContinue = block.anyContinue;
-        anyBreak = block.anyBreak;
+        output.methodEscape = blockOutput.methodEscape;
+        output.loopEscape = blockOutput.loopEscape;
+        output.allEscape = blockOutput.allEscape;
+        output.anyContinue = blockOutput.anyContinue;
+        output.anyBreak = blockOutput.anyBreak;
 
         int statementCount = 0;
 
         for (SCatch catc : catches) {
-            catc.lastSource = lastSource;
-            catc.inLoop = inLoop;
-            catc.lastLoop = lastLoop;
+            Input catchInput = new Input();
+            catchInput.lastSource = input.lastSource;
+            catchInput.inLoop = input.inLoop;
+            catchInput.lastLoop = input.lastLoop;
 
-            catc.analyze(scriptRoot, scope.newLocalScope());
+            Output catchOutput = catc.analyze(scriptRoot, scope.newLocalScope(), catchInput);
 
-            methodEscape &= catc.methodEscape;
-            loopEscape &= catc.loopEscape;
-            allEscape &= catc.allEscape;
-            anyContinue |= catc.anyContinue;
-            anyBreak |= catc.anyBreak;
+            output.methodEscape &= catchOutput.methodEscape;
+            output.loopEscape &= catchOutput.loopEscape;
+            output.allEscape &= catchOutput.allEscape;
+            output.anyContinue |= catchOutput.anyContinue;
+            output.anyBreak |= catchOutput.anyBreak;
 
-            statementCount = Math.max(statementCount, catc.statementCount);
+            statementCount = Math.max(statementCount, catchOutput.statementCount);
         }
 
-        this.statementCount = block.statementCount + statementCount;
+        output.statementCount = blockOutput.statementCount + statementCount;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 0ba07f1d5a5a2..eead51786e2d3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -45,7 +45,10 @@ public SWhile(Location location, AExpression condition, SBlock block) {
     }
 
     @Override
-    void analyze(ScriptRoot scriptRoot, Scope scope) {
+    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+        this.input = input;
+        output = new Output();
+
         scope = scope.newLocalScope();
 
         AExpression.Input conditionInput = new AExpression.Input();
@@ -66,24 +69,27 @@ void analyze(ScriptRoot scriptRoot, Scope scope) {
         }
 
         if (block != null) {
-            block.beginLoop = true;
-            block.inLoop = true;
+            Input blockInput = new Input();
+            blockInput.beginLoop = true;
+            blockInput.inLoop = true;
 
-            block.analyze(scriptRoot, scope);
+            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
 
-            if (block.loopEscape && !block.anyContinue) {
+            if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
             }
 
-            if (continuous && !block.anyBreak) {
-                methodEscape = true;
-                allEscape = true;
+            if (continuous && blockOutput.anyBreak == false) {
+                output.methodEscape = true;
+                output.allEscape = true;
             }
 
-            block.statementCount = Math.max(1, block.statementCount);
+            blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
         }
 
-        statementCount = 1;
+        output.statementCount = 1;
+
+        return output;
     }
 
     @Override

From 89d68736613ee7feeed10181c16216a764f86b4e Mon Sep 17 00:00:00 2001
From: debadair <debadair@elastic.co>
Date: Fri, 13 Mar 2020 16:44:18 -0700
Subject: [PATCH 038/285] [DOCS] Added missing definitions and edited for
 consistency. (#53519)

* [DOCS] Added missing defintions and edited for consistency with the Kibana glossary.

* Apply suggestions from code review

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>

* Added API key
---
 docs/reference/glossary.asciidoc | 192 ++++++++++++++++++++++++++-----
 1 file changed, 162 insertions(+), 30 deletions(-)

diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc
index 40ff0828afe2d..fac44f8bcdc53 100644
--- a/docs/reference/glossary.asciidoc
+++ b/docs/reference/glossary.asciidoc
@@ -24,28 +24,55 @@ Also see <<glossary-text,text>> and <<glossary-term,term>>.
 // end::analysis-def[]
 --
 
+[[glossary-api-key]] API key ::
+// tag::api-key-def[]
+A unique identifier that you can use for authentication when submitting {es} requests. 
+When TLS is enabled, all requests must be authenticated using either basic authentication 
+(user name and password) or an API key.
+// end::api-key-def[]
+
+
+[[glossary-auto-follow-pattern]] auto-follow pattern ::
+// tag::auto-follow-pattern-def[]
+An <<glossary-index-pattern,index pattern>> that automatically configures new indices as 
+<<glossary-follower-index,follower indices>> for <<glossary-ccr,{ccr}>>.
+For more information, see {ref}/ccr-auto-follow.html#_managing_auto_follow_patterns[Managing auto follow patterns].
+// end::auto-follow-pattern-def[]
+
 [[glossary-cluster]] cluster ::
 // tag::cluster-def[]
-A cluster consists of one or more <<glossary-node,nodes>> which share the
-same cluster name. Each cluster has a single master node which is
-chosen automatically by the cluster and which can be replaced if the
-current master node fails.
+One or more <<glossary-node,nodes>> that share the
+same cluster name. Each cluster has a single master node, which is
+chosen automatically by the cluster and can be replaced if it fails.
 // end::cluster-def[]
 
+[[glossary-cold-phase]] cold phase ::
+// tag::cold-phase-def[]
+The third possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the cold phase, an index is no longer updated and seldom queried. 
+The information still needs to be searchable, but it’s okay if those queries are slower.
+// end::cold-phase-def[]
+
 [[glossary-ccr]] {ccr} (CCR)::
 // tag::ccr-def[]
-The {ccr} feature enables you to replicate indices in remote clusters to your
+A feature that enables you to replicate indices in remote clusters to your
 local cluster. For more information, see 
 {ref}/xpack-ccr.html[{ccr-cap}].  
 // end::ccr-def[]
   
 [[glossary-ccs]] {ccs} (CCS)::
 // tag::ccs-def[]
-The {ccs} feature enables any node to act as a federated client across
+A feature that enables any node to act as a federated client across
 multiple clusters.
 See {ref}/modules-cross-cluster-search.html[Search across clusters].
 // end::ccs-def[]
 
+[[glossary-delete-phase]] delete phase ::
+// tag::delete-phase-def[]
+The last possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the delete phase, an index is no longer needed and can safely be deleted.
+// end::delete-phase-def[]
+
 [[glossary-document]] document ::
 +
 --
@@ -93,10 +120,22 @@ hence it is called a filter. Filters are simple checks for set inclusion or excl
 In most cases, the goal of filtering is to reduce the number of documents that have to be examined.
 // end::filter-def[]
 
+[[glossary-flush]] flush ::
+// tag::flush-def[]
+Peform a Lucene commit to write index updates in the transaction log (translog) to disk. 
+Because a Lucene commit is a relatively expensive operation, 
+{es} records index and delete operations in the translog and 
+automatically flushes changes to disk in batches.
+To recover from a crash, operations that have been acknowledged but not yet committed 
+can be replayed from the translog.
+Before upgrading, you can explicitly call the {ref}/indices-flush.html[Flush] API
+to ensure that all changes are committed to disk.
+// end::flush-def[]
+
 [[glossary-follower-index]] follower index ::  
 // tag::follower-index-def[]
-Follower indices are the target indices for <<glossary-ccr,{ccr}>>. They exist
-in your local cluster and replicate <<glossary-leader-index,leader indices>>.
+The target index for <<glossary-ccr,{ccr}>>. A follower index exists
+in a local cluster and replicates a <<glossary-leader-index,leader index>>.
 // end::follower-index-def[]
 
 [[glossary-force-merge]] force merge ::  
@@ -107,7 +146,7 @@ and free up the space used by deleted documents.
 // end::force-merge-def-short[]
 You should not force merge indices that are actively being written to.
 Merging is normally performed automatically, but you can use force merge after 
-<<glossary-rollover, rollover>> to reduce the shards in the old index to a single segment. 
+<<glossary-rollover,rollover>> to reduce the shards in the old index to a single segment. 
 See the {ref}/indices-forcemerge.html[force merge API].
 // end::force-merge-def[]
 
@@ -123,6 +162,19 @@ before you are ready to archive or delete them.
 See the {ref}/freeze-index-api.html[freeze API].
 // end::freeze-def[]
 
+[[glossary-frozen-index]] frozen index ::  
+// tag::frozen-index-def[]
+An index reduced to a low overhead state that still enables occasional searches.
+Frozen indices use a memory-efficient shard implementation and throttle searches to conserve resources.
+Searching a frozen index is lower overhead than re-opening a closed index to enable searching.
+// end::frozen-index-def[]
+
+[[glossary-hot-phase]] hot phase ::
+// tag::hot-phase-def[]
+The first possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the hot phase, an index is actively updated and queried. 
+// end::hot-phase-def[]
+
 [[glossary-id]] id ::
 // tag::id-def[]
 The ID of a <<glossary-document,document>> identifies a document. The
@@ -134,11 +186,12 @@ then it will be auto-generated. (also see <<glossary-routing,routing>>)
 +
 --
 // tag::index-def[]
-An index is like a _table_ in a relational database. It has a
-<<glossary-mapping,mapping>> which contains a <<glossary-type,type>>,
-which contains the <<glossary-field,field>> in the index.
+// tag::index-def-short[]
+An optimized collection of JSON documents. Each document is a collection of fields, 
+the key-value pairs that contain your data. 
+// end::index-def-short[]
 
-An index is a logical namespace which maps to one or more
+An index is a logical namespace that maps to one or more
 <<glossary-primary-shard,primary shards>> and can have zero or more
 <<glossary-replica-shard,replica shards>>.
 // end::index-def[]
@@ -149,17 +202,38 @@ An index is a logical namespace which maps to one or more
 --
 // tag::index-alias-def[]
 // tag::index-alias-desc[]
-An index alias is a secondary name
-used to refer to one or more existing indices.
+An index alias is a logical name used to reference one or more indices.
 
-Most {es} APIs accept an index alias
-in place of an index name.
+Most {es} APIs accept an index alias in place of an index name.
 // end::index-alias-desc[]
 
 See {ref}/indices-add-alias.html[Add index alias].
 // end::index-alias-def[]
 --
 
+[[glossary-index-lifecycle]] index lifecycle ::
+// tag::index-lifecycle-def[]
+The four phases an index can transition through: 
+<<glossary-hot-phase,hot>>, <<glossary-warm-phase,warm>>,
+<<glossary-cold-phase,cold>>, and <<glossary-delete-phase,delete>>.
+For more information, see {ref}/ilm-policy-definition.html[Index lifecycle].
+// end::index-lifecycle-def[]
+
+[[glossary-index-lifecycle-policy]] index lifecycle policy ::
+// tag::index-lifecycle-policy-def[]
+Specifies how an index moves between phases in the index lifecycle and 
+what actions to perform during each phase. 
+// end::index-lifecycle-policy-def[]
+
+[[glossary-index-pattern]] index pattern ::
+// tag::index-pattern-def[]
+A string that can contain the `*` wildcard to match multiple index names.
+In most cases, the index parameter in an {es} request can be the name of a specific index, 
+a list of index names, or an index pattern.
+For example, if you have the indices `datastream-000001`, `datastream-000002`, and `datastream-000003`,
+to search across all three you could use the `datastream-*` index pattern.  
+// end::index-pattern-def[]
+
 [[glossary-index-template]] index template ::
 +
 --
@@ -167,18 +241,24 @@ See {ref}/indices-add-alias.html[Add index alias].
 // tag::index-template-def-short[]
 Defines settings and mappings to apply to new indexes that match a simple naming pattern, such as _logs-*_. 
 // end::index-template-def-short[]
+
 An index template can also attach a lifecycle policy to the new index.
-Index templates are used to automatically configure indices created during <<glossary-rollover, rollover>>. 
+Index templates are used to automatically configure indices created during <<glossary-rollover,rollover>>. 
 // end::index-template-def[]
 --
 
 [[glossary-leader-index]] leader index ::  
 // tag::leader-index-def[]
-Leader indices are the source indices for <<glossary-ccr,{ccr}>>. They exist
-on remote clusters and are replicated to 
+The source index for <<glossary-ccr,{ccr}>>. A leader index exists
+on a remote cluster and is replicated to 
 <<glossary-follower-index,follower indices>>.
 // end::leader-index-def[]
 
+[[glossary-local-cluster]] local cluster ::
+// tag::local-cluster-def[]
+The cluster that pulls data from a <<glossary-remote-cluster,remote cluster>> in {ccs} or {ccr}.
+// end::local-cluster-def[]
+
 [[glossary-mapping]] mapping ::
 +
 --
@@ -194,18 +274,12 @@ automatically when a document is indexed.
 --
 
 [[glossary-node]] node ::
-+
---
 // tag::node-def[]
-A node is a running instance of Elasticsearch which belongs to a
+A running instance of {es} that belongs to a
 <<glossary-cluster,cluster>>. Multiple nodes can be started on a single
 server for testing purposes, but usually you should have one node per
 server.
-
-At startup, a node will use unicast to discover an existing cluster with
-the same cluster name and will try to join that cluster.
 // end::node-def[]
---
 
 [[glossary-primary-shard]] primary shard ::
 +
@@ -267,9 +341,18 @@ during the following processes:
 [[glossary-reindex]] reindex ::
 
 // tag::reindex-def[]
-To cycle through some or all documents in one or more indices, re-writing them into the same or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade Elasticsearch between two incompatible index versions.
+To cycle through some or all documents in one or more indices, re-writing them into the same 
+or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade {es} between two incompatible index versions.
 // end::reindex-def[]
 
+[[glossary-remote-cluster]] remote cluster ::
+
+// tag::remote-cluster-def[]
+A separate cluster, often in a different data center or locale, that contains indices that 
+can be replicated or searched by the <<glossary-local-cluster,local cluster>>. 
+The connection to a remote cluster is unidirectional.
+// end::remote-cluster-def[]
+
 [[glossary-replica-shard]] replica shard ::
 +
 --
@@ -294,14 +377,35 @@ shard will never be started on the same node as its primary shard.
 --
 // tag::rollover-def[]
 // tag::rollover-def-short[]
-Redirect an alias to begin writing to a new index when the existing index reaches a certain age, number of docs, or size. 
+Redirect an index alias to begin writing to a new index when the existing index reaches 
+a certain size, number of docs, or age. 
 // end::rollover-def-short[]
-The new index is automatically configured according to any matching <<glossary-index-template, index templates>>. 
+
+The new index is automatically configured according to any matching <<glossary-index-template,index templates>>. 
 For example, if you're indexing log data, you might use rollover to create daily or weekly indices. 
 See the {ref}/indices-rollover-index.html[rollover index API].
 // end::rollover-def[]
 --
 
+[[glossary-rollup]] rollup ::
+// tag::rollup-def[]
+Summarize high-granularity data into a more compressed format to 
+maintain access to historical data in a cost-effective way.
+// end::rollup-def[]
+
+[[glossary-rollup-index]] rollup index ::
+// tag::rollup-index-def[]
+A special type of index for storing historical data at reduced granularity. 
+Documents are summarized and indexed into a rollup index by a <<glossary-rollup-job,rollup job>>.
+// end::rollup-index-def[]
+
+[[glossary-rollup-job]] rollup job ::
+// tag::rollup-job-def[]
+A background task that runs continuously to summarize documents in an index and 
+index the summaries into a separate rollup index. 
+The job configuration controls what information is rolled up and how often. 
+// end::rollup-job-def[]
+
 [[glossary-routing]] routing ::
 +
 --
@@ -340,14 +444,36 @@ nodes.
 --
 
 [[glossary-shrink]] shrink ::
++
+--
 // tag::shrink-def[]
 // tag::shrink-def-short[]
 Reduce the number of primary shards in an index.
 // end::shrink-def-short[]
+
 You can shrink an index to reduce its overhead when the request volume drops.
 For example, you might opt to shrink an index once it is no longer the write index. 
 See the {ref}/indices-shrink-index.html[shrink index API].
 // end::shrink-def[]
+--
+
+[[glossary-snapshot]] snapshot ::
+// tag::snapshot-def[]
+A backup taken from a running {es} cluster. 
+You can take snapshots of individual indices or of the entire cluster. 
+// end::snapshot-def[]
+
+[[glossary-snapshot-lifecycle-policy]] snapshot lifecycle policy ::
+// tag::snapshot-lifecycle-policy-def[]
+Specifies how frequently to perform automatic backups of a cluster and 
+how long to retain the resulting snapshots. 
+// end::snapshot-lifecycle-policy-def[]
+
+[[glossary-snapshot-repository]] snapshot repository ::
+// tag::snapshot-repository-def[]
+Specifies where snapshots are to be stored. 
+Snapshots can be written to a shared filesystem or to a remote repository.
+// end::snapshot-repository-def[]
 
 [[glossary-source_field]] source field ::
 // tag::source-field-def[]
@@ -399,3 +525,9 @@ A type used to represent the _type_ of document, e.g. an `email`, a `user`, or a
 Types are deprecated and are in the process of being removed.
 See {ref}/removal-of-types.html[Removal of mapping types].
 // end::type-def[]
+
+[[glossary-warm-phase]] warm phase ::
+// tag::warm-phase-def[]
+The second possible phase in the <<glossary-index-lifecycle,index lifecycle>>. 
+In the warm phase, an index is generally optimized for search and no longer updated.
+// end::warm-phase-def[]

From 7bc75f48d84149bd81ac6779360417c755d3b878 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 14 Mar 2020 10:22:29 -0400
Subject: [PATCH 039/285] Upgrade to Jackson 2.10.3 and GeoIP2 to 2.13.1
 (#53523)

This commit upgrades our Jackson dependency to 2.10.3 and our GeoIP2
dependency to 2.13.1.
---
 buildSrc/version.properties                   |  8 ++------
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 libs/x-content/build.gradle                   |  1 +
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../jackson-dataformat-cbor-2.10.3.jar.sha1   |  1 +
 .../jackson-dataformat-cbor-2.8.11.jar.sha1   |  1 -
 .../jackson-dataformat-smile-2.10.3.jar.sha1  |  1 +
 .../jackson-dataformat-smile-2.8.11.jar.sha1  |  1 -
 .../jackson-dataformat-yaml-2.10.3.jar.sha1   |  1 +
 .../jackson-dataformat-yaml-2.8.11.jar.sha1   |  1 -
 .../licenses/snakeyaml-1.17.jar.sha1          |  1 -
 .../licenses/snakeyaml-1.24.jar.sha1          |  1 +
 .../common/xcontent/XContentParserTests.java  | 19 +++++++++++--------
 .../ingest/common/JsonProcessorTests.java     |  9 +++++++--
 modules/ingest-geoip/build.gradle             | 10 ++++------
 .../licenses/geoip2-2.13.1.jar.sha1           |  1 +
 .../licenses/geoip2-2.9.0.jar.sha1            |  1 -
 .../jackson-annotations-2.10.3.jar.sha1       |  1 +
 .../jackson-annotations-2.8.11.jar.sha1       |  1 -
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../licenses/maxmind-db-1.2.2.jar.sha1        |  1 -
 .../licenses/maxmind-db-1.3.1.jar.sha1        |  1 +
 plugins/discovery-ec2/build.gradle            |  2 +-
 .../jackson-annotations-2.10.3.jar.sha1       |  1 +
 .../jackson-annotations-2.8.11.jar.sha1       |  1 -
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 plugins/repository-s3/build.gradle            |  2 +-
 .../jackson-annotations-2.10.3.jar.sha1       |  1 +
 .../jackson-annotations-2.8.11.jar.sha1       |  1 -
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../xcontent/XContentParserUtilsTests.java    | 11 +----------
 .../index/IndexingSlowLogTests.java           |  8 ++++----
 x-pack/plugin/sql/jdbc/build.gradle           |  5 +++++
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../jackson-dataformat-cbor-2.10.3.jar.sha1   |  1 +
 .../jackson-dataformat-cbor-2.8.11.jar.sha1   |  1 -
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../xpack/sql/client/RemoteFailureTests.java  |  5 +++--
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 49 files changed, 59 insertions(+), 59 deletions(-)
 create mode 100644 client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
 delete mode 100644 libs/x-content/licenses/snakeyaml-1.17.jar.sha1
 create mode 100644 libs/x-content/licenses/snakeyaml-1.24.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index dea67de1657d1..9ef5da5062255 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -7,12 +7,8 @@ bundled_jdk = 13.0.2+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-# note that ingest-geoip has a hard-coded version; if you modify this version,
-# you should also inspect that version to see if it can be advanced along with
-# the com.maxmind.geoip2:geoip2 dependency
-jackson           = 2.8.11
-jacksondatabind   = 2.8.11.6
-snakeyaml         = 1.17
+jackson           = 2.10.3
+snakeyaml         = 1.24
 icu4j             = 62.1
 supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
diff --git a/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1 b/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
index 14adf7d102d78..be056980ef50d 100644
--- a/libs/x-content/build.gradle
+++ b/libs/x-content/build.gradle
@@ -49,6 +49,7 @@ forbiddenApisMain {
 thirdPartyAudit.ignoreMissingClasses(
   // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
   'com.fasterxml.jackson.databind.ObjectMapper',
+  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
 )
 
 dependencyLicenses {
diff --git a/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..6c96c20f48232
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
@@ -0,0 +1 @@
+1ba01fef9c3b7ed388d91e71dc733b315c7374cd
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
deleted file mode 100644
index 378ba524422bc..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..24ac4b32095e4
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
@@ -0,0 +1 @@
+ff397547ff168e77279a1cd549e2ca4923c991aa
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
deleted file mode 100644
index 510afb3df53e6..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..8f1ecab9ecb38
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
@@ -0,0 +1 @@
+4dc1a172812d9da27c1afd6a08f4f12aad7b14dd
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
deleted file mode 100644
index 78a68d715ec3d..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a
\ No newline at end of file
diff --git a/libs/x-content/licenses/snakeyaml-1.17.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
deleted file mode 100644
index 9ac6e87f2244a..0000000000000
--- a/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7a27ea250c5130b2922b86dea63cbb1cc10a660c
\ No newline at end of file
diff --git a/libs/x-content/licenses/snakeyaml-1.24.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.24.jar.sha1
new file mode 100644
index 0000000000000..a6e7bd5a1e7c1
--- /dev/null
+++ b/libs/x-content/licenses/snakeyaml-1.24.jar.sha1
@@ -0,0 +1 @@
+13a9c0d6776483c3876e3ff9384f9bb55b17001b
\ No newline at end of file
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
index 0cfa01876c590..66c0041723e8c 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -78,14 +78,17 @@ public void testFloat() throws IOException {
 
             assertEquals(value, number.floatValue(), 0.0f);
 
-            if (xContentType == XContentType.CBOR) {
-                // CBOR parses back a float
-                assertTrue(number instanceof Float);
-            } else {
-                // JSON, YAML and SMILE parses back the float value as a double
-                // This will change for SMILE in Jackson 2.9 where all binary based
-                // formats will return a float
-                assertTrue(number instanceof Double);
+            switch (xContentType) {
+                case CBOR:
+                case SMILE:
+                    assertThat(number, instanceOf(Float.class));
+                    break;
+                case JSON:
+                case YAML:
+                    assertThat(number, instanceOf(Double.class));
+                    break;
+                default:
+                    throw new AssertionError("unexpected x-content type [" + xContentType + "]");
             }
         }
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index 099e8e1866b8e..c9e81ccf4c811 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -65,7 +65,7 @@ public void testInvalidValue() {
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
         assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
-            "was expecting ('true', 'false' or 'null')"));
+            "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"));
     }
 
     public void testByteArray() {
@@ -75,7 +75,12 @@ public void testByteArray() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
-        assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
+        assertThat(
+            exception.getCause().getMessage(),
+            containsString(
+                "Unrecognized token 'B': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
+            )
+        );
     }
 
     public void testNull() throws Exception {
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index 6789f96d883b8..dfa165e566170 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -25,13 +25,11 @@ esplugin {
 }
 
 dependencies {
-  // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
-  compile('com.maxmind.geoip2:geoip2:2.9.0')
+  compile('com.maxmind.geoip2:geoip2:2.13.1')
   // geoip2 dependencies:
-  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
-  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
-  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
-  compile('com.maxmind.db:maxmind-db:1.2.2')
+  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
+  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
+  compile('com.maxmind.db:maxmind-db:1.3.1')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
 }
diff --git a/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
new file mode 100644
index 0000000000000..253d9f12e7a3a
--- /dev/null
+++ b/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
@@ -0,0 +1 @@
+f27d1a49d5a29dd4a7ac5006ce2eb16b8b9bb888
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
deleted file mode 100644
index 8cb79bbb9207a..0000000000000
--- a/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c12b463a2c10824225c0b27952c49b464cb0e4c6
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..9c725f2d90e69
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
deleted file mode 100644
index 30e7d1a7b1a74..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..688ae92d10792
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
deleted file mode 100644
index 3dd3ad36f4cc3..0000000000000
--- a/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-78c22a03de1e222b0751855aff7bb6e6db5569e5
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
new file mode 100644
index 0000000000000..aebff2c3a849c
--- /dev/null
+++ b/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
@@ -0,0 +1 @@
+211bca628225bc0f719051b16deb03a747d7a14f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index a9ae35393de16..bc507dc5d9742 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -36,7 +36,7 @@ dependencies {
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..9c725f2d90e69
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
deleted file mode 100644
index 30e7d1a7b1a74..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..688ae92d10792
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index a92f07b71ee01..04624f8512976 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -43,7 +43,7 @@ dependencies {
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   compile "joda-time:joda-time:${versions.joda}"
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..9c725f2d90e69
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
deleted file mode 100644
index 30e7d1a7b1a74..0000000000000
--- a/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..688ae92d10792
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
index db3dabdcafe49..6b99f28f3a8c1 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
@@ -31,7 +31,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 
@@ -90,15 +89,7 @@ public void testStoredFieldsValueBoolean() throws IOException {
 
     public void testStoredFieldsValueBinary() throws IOException {
         final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8");
-        assertParseFieldsSimpleValue(value, (xcontentType, result) -> {
-            if (xcontentType == XContentType.JSON || xcontentType == XContentType.YAML) {
-                //binary values will be parsed back and returned as base64 strings when reading from json and yaml
-                assertArrayEquals(value, Base64.getDecoder().decode((String) result));
-            } else {
-                //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
-                assertArrayEquals(value, ((BytesArray) result).array());
-            }
-        });
+        assertParseFieldsSimpleValue(value, (xcontentType, result) -> assertArrayEquals(value, ((BytesArray) result).array()));
     }
 
     public void testStoredFieldsValueNull() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index dbff49460015b..78d25e996ab2a 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -103,13 +103,13 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         final UncheckedIOException e = expectThrows(UncheckedIOException.class,
             () -> IndexingSlowLogMessage.of(index, doc, 10, true, 3));
         assertThat(e, hasToString(containsString("_failed_to_convert_[Unrecognized token 'invalid':"
-            + " was expecting ('true', 'false' or 'null')\\n"
-            + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
+            + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\\n"
+            + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
         assertNotNull(e.getCause());
         assertThat(e.getCause(), instanceOf(JsonParseException.class));
         assertThat(e.getCause(), hasToString(containsString("Unrecognized token 'invalid':"
-                + " was expecting ('true', 'false' or 'null')\n"
-                + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
+                + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n"
+                + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
     }
 
     public void testReformatSetting() {
diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle
index 7cdec567ef8f4..dc42f90476571 100644
--- a/x-pack/plugin/sql/jdbc/build.gradle
+++ b/x-pack/plugin/sql/jdbc/build.gradle
@@ -44,3 +44,8 @@ dependencyLicenses {
 shadowJar {
   relocate 'com.fasterxml', 'org.elasticsearch.fasterxml'
 }
+
+thirdPartyAudit.ignoreMissingClasses(
+  'com.fasterxml.jackson.databind.ObjectMapper',
+  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
+)
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..6c96c20f48232
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
@@ -0,0 +1 @@
+1ba01fef9c3b7ed388d91e71dc733b315c7374cd
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
deleted file mode 100644
index 378ba524422bc..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
index 2029493bcbec4..1409e80837768 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
@@ -92,7 +92,8 @@ public void testNoType() {
     public void testInvalidJson() {
         IOException e = expectThrows(IOException.class, () -> parse("invalid_json.txt"));
         assertEquals(
-            "Can't parse error from Elasticsearch [Unrecognized token 'I': was expecting 'null', 'true', 'false' or NaN] "
+            "Can't parse error from Elasticsearch [Unrecognized token 'I': "
+                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] "
                 + "at [line 1 col 1]. Response:\n"
                 + "I'm not json at all",
             e.getMessage());
@@ -118,7 +119,7 @@ public void testTotalGarbage() {
             }).streamInput()));
         assertThat(e.getMessage(),
             startsWith("Can't parse error from Elasticsearch [Unrecognized token 'ÿ': "
-                + "was expecting ('true', 'false' or 'null')] at [line 1 col 1]. Response:\n"));
+                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] at [line 1 col 4]. Response:\n"));
     }
 
     public void testTooBig() {
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file

From fa6d5158930b7e0ce9836b8f9fc0adc78416cb99 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 14 Mar 2020 18:11:31 -0400
Subject: [PATCH 040/285] Remove extra code in allocation commands parsing
 (#53579)

This commit removes some code that is duplicated in the parsing of
allocation commands in the cluster reroute API.
---
 .../cluster/routing/allocation/command/AllocationCommands.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
index 67122cb3ff1b1..4f7da131977fb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
@@ -137,9 +137,6 @@ public static AllocationCommands fromXContent(XContentParser parser) throws IOEx
             throw new ElasticsearchParseException("No commands");
         }
         if (token == XContentParser.Token.FIELD_NAME) {
-            if (!parser.currentName().equals("commands")) {
-                throw new ElasticsearchParseException("expected field name to be named [commands], got [{}] instead", parser.currentName());
-            }
             if (!parser.currentName().equals("commands")) {
                 throw new ElasticsearchParseException("expected field name to be named [commands], got [{}] instead", parser.currentName());
             }

From 060b4eed593a53026c15d965ea664fe675bd0845 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Sun, 15 Mar 2020 18:10:14 -0700
Subject: [PATCH 041/285] Revert "Upgrade to Jackson 2.10.3 and GeoIP2 to
 2.13.1 (#53523)"

This reverts commit 7bc75f48

Signed-off-by: Mark Vieira <portugee@gmail.com>
---
 buildSrc/version.properties                   |  8 ++++++--
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 -
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 +
 libs/x-content/build.gradle                   |  1 -
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 -
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 +
 .../jackson-dataformat-cbor-2.10.3.jar.sha1   |  1 -
 .../jackson-dataformat-cbor-2.8.11.jar.sha1   |  1 +
 .../jackson-dataformat-smile-2.10.3.jar.sha1  |  1 -
 .../jackson-dataformat-smile-2.8.11.jar.sha1  |  1 +
 .../jackson-dataformat-yaml-2.10.3.jar.sha1   |  1 -
 .../jackson-dataformat-yaml-2.8.11.jar.sha1   |  1 +
 .../licenses/snakeyaml-1.17.jar.sha1          |  1 +
 .../licenses/snakeyaml-1.24.jar.sha1          |  1 -
 .../common/xcontent/XContentParserTests.java  | 19 ++++++++-----------
 .../ingest/common/JsonProcessorTests.java     |  9 ++-------
 modules/ingest-geoip/build.gradle             | 10 ++++++----
 .../licenses/geoip2-2.13.1.jar.sha1           |  1 -
 .../licenses/geoip2-2.9.0.jar.sha1            |  1 +
 .../jackson-annotations-2.10.3.jar.sha1       |  1 -
 .../jackson-annotations-2.8.11.jar.sha1       |  1 +
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 +
 .../licenses/maxmind-db-1.2.2.jar.sha1        |  1 +
 .../licenses/maxmind-db-1.3.1.jar.sha1        |  1 -
 plugins/discovery-ec2/build.gradle            |  2 +-
 .../jackson-annotations-2.10.3.jar.sha1       |  1 -
 .../jackson-annotations-2.8.11.jar.sha1       |  1 +
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 +
 plugins/repository-s3/build.gradle            |  2 +-
 .../jackson-annotations-2.10.3.jar.sha1       |  1 -
 .../jackson-annotations-2.8.11.jar.sha1       |  1 +
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 +
 .../xcontent/XContentParserUtilsTests.java    | 11 ++++++++++-
 .../index/IndexingSlowLogTests.java           |  8 ++++----
 x-pack/plugin/sql/jdbc/build.gradle           |  5 -----
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 -
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 +
 .../jackson-dataformat-cbor-2.10.3.jar.sha1   |  1 -
 .../jackson-dataformat-cbor-2.8.11.jar.sha1   |  1 +
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 -
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 +
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 -
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 +
 .../xpack/sql/client/RemoteFailureTests.java  |  5 ++---
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 -
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 +
 49 files changed, 59 insertions(+), 59 deletions(-)
 delete mode 100644 client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
 create mode 100644 client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/snakeyaml-1.17.jar.sha1
 delete mode 100644 libs/x-content/licenses/snakeyaml-1.24.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
 create mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
 delete mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
 create mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 9ef5da5062255..dea67de1657d1 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -7,8 +7,12 @@ bundled_jdk = 13.0.2+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.10.3
-snakeyaml         = 1.24
+# note that ingest-geoip has a hard-coded version; if you modify this version,
+# you should also inspect that version to see if it can be advanced along with
+# the com.maxmind.geoip2:geoip2 dependency
+jackson           = 2.8.11
+jacksondatabind   = 2.8.11.6
+snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
diff --git a/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1 b/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
deleted file mode 100644
index f23937b0d82a4..0000000000000
--- a/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
index be056980ef50d..14adf7d102d78 100644
--- a/libs/x-content/build.gradle
+++ b/libs/x-content/build.gradle
@@ -49,7 +49,6 @@ forbiddenApisMain {
 thirdPartyAudit.ignoreMissingClasses(
   // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
   'com.fasterxml.jackson.databind.ObjectMapper',
-  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
 )
 
 dependencyLicenses {
diff --git a/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
deleted file mode 100644
index f23937b0d82a4..0000000000000
--- a/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
deleted file mode 100644
index 6c96c20f48232..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1ba01fef9c3b7ed388d91e71dc733b315c7374cd
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..378ba524422bc
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
@@ -0,0 +1 @@
+8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
deleted file mode 100644
index 24ac4b32095e4..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ff397547ff168e77279a1cd549e2ca4923c991aa
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..510afb3df53e6
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
@@ -0,0 +1 @@
+d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
deleted file mode 100644
index 8f1ecab9ecb38..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4dc1a172812d9da27c1afd6a08f4f12aad7b14dd
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..78a68d715ec3d
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
@@ -0,0 +1 @@
+2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a
\ No newline at end of file
diff --git a/libs/x-content/licenses/snakeyaml-1.17.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
new file mode 100644
index 0000000000000..9ac6e87f2244a
--- /dev/null
+++ b/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
@@ -0,0 +1 @@
+7a27ea250c5130b2922b86dea63cbb1cc10a660c
\ No newline at end of file
diff --git a/libs/x-content/licenses/snakeyaml-1.24.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.24.jar.sha1
deleted file mode 100644
index a6e7bd5a1e7c1..0000000000000
--- a/libs/x-content/licenses/snakeyaml-1.24.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-13a9c0d6776483c3876e3ff9384f9bb55b17001b
\ No newline at end of file
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
index 66c0041723e8c..0cfa01876c590 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -78,17 +78,14 @@ public void testFloat() throws IOException {
 
             assertEquals(value, number.floatValue(), 0.0f);
 
-            switch (xContentType) {
-                case CBOR:
-                case SMILE:
-                    assertThat(number, instanceOf(Float.class));
-                    break;
-                case JSON:
-                case YAML:
-                    assertThat(number, instanceOf(Double.class));
-                    break;
-                default:
-                    throw new AssertionError("unexpected x-content type [" + xContentType + "]");
+            if (xContentType == XContentType.CBOR) {
+                // CBOR parses back a float
+                assertTrue(number instanceof Float);
+            } else {
+                // JSON, YAML and SMILE parses back the float value as a double
+                // This will change for SMILE in Jackson 2.9 where all binary based
+                // formats will return a float
+                assertTrue(number instanceof Double);
             }
         }
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index c9e81ccf4c811..099e8e1866b8e 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -65,7 +65,7 @@ public void testInvalidValue() {
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
         assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
-            "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"));
+            "was expecting ('true', 'false' or 'null')"));
     }
 
     public void testByteArray() {
@@ -75,12 +75,7 @@ public void testByteArray() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
-        assertThat(
-            exception.getCause().getMessage(),
-            containsString(
-                "Unrecognized token 'B': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
-            )
-        );
+        assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
     }
 
     public void testNull() throws Exception {
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index dfa165e566170..6789f96d883b8 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -25,11 +25,13 @@ esplugin {
 }
 
 dependencies {
-  compile('com.maxmind.geoip2:geoip2:2.13.1')
+  // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
+  compile('com.maxmind.geoip2:geoip2:2.9.0')
   // geoip2 dependencies:
-  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
-  compile('com.maxmind.db:maxmind-db:1.3.1')
+  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
+  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
+  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
+  compile('com.maxmind.db:maxmind-db:1.2.2')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
 }
diff --git a/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
deleted file mode 100644
index 253d9f12e7a3a..0000000000000
--- a/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f27d1a49d5a29dd4a7ac5006ce2eb16b8b9bb888
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
new file mode 100644
index 0000000000000..8cb79bbb9207a
--- /dev/null
+++ b/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
@@ -0,0 +1 @@
+c12b463a2c10824225c0b27952c49b464cb0e4c6
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
deleted file mode 100644
index 9c725f2d90e69..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..30e7d1a7b1a74
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
@@ -0,0 +1 @@
+391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
deleted file mode 100644
index 688ae92d10792..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
new file mode 100644
index 0000000000000..3dd3ad36f4cc3
--- /dev/null
+++ b/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
@@ -0,0 +1 @@
+78c22a03de1e222b0751855aff7bb6e6db5569e5
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
deleted file mode 100644
index aebff2c3a849c..0000000000000
--- a/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-211bca628225bc0f719051b16deb03a747d7a14f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index bc507dc5d9742..a9ae35393de16 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -36,7 +36,7 @@ dependencies {
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
deleted file mode 100644
index 9c725f2d90e69..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..30e7d1a7b1a74
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
@@ -0,0 +1 @@
+391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
deleted file mode 100644
index 688ae92d10792..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 04624f8512976..a92f07b71ee01 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -43,7 +43,7 @@ dependencies {
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   compile "joda-time:joda-time:${versions.joda}"
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
deleted file mode 100644
index 9c725f2d90e69..0000000000000
--- a/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..30e7d1a7b1a74
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
@@ -0,0 +1 @@
+391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
deleted file mode 100644
index 688ae92d10792..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
new file mode 100644
index 0000000000000..f491259db56bc
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
@@ -0,0 +1 @@
+35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
index 6b99f28f3a8c1..db3dabdcafe49 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
@@ -31,6 +31,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 
@@ -89,7 +90,15 @@ public void testStoredFieldsValueBoolean() throws IOException {
 
     public void testStoredFieldsValueBinary() throws IOException {
         final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8");
-        assertParseFieldsSimpleValue(value, (xcontentType, result) -> assertArrayEquals(value, ((BytesArray) result).array()));
+        assertParseFieldsSimpleValue(value, (xcontentType, result) -> {
+            if (xcontentType == XContentType.JSON || xcontentType == XContentType.YAML) {
+                //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+                assertArrayEquals(value, Base64.getDecoder().decode((String) result));
+            } else {
+                //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+                assertArrayEquals(value, ((BytesArray) result).array());
+            }
+        });
     }
 
     public void testStoredFieldsValueNull() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index 78d25e996ab2a..dbff49460015b 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -103,13 +103,13 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         final UncheckedIOException e = expectThrows(UncheckedIOException.class,
             () -> IndexingSlowLogMessage.of(index, doc, 10, true, 3));
         assertThat(e, hasToString(containsString("_failed_to_convert_[Unrecognized token 'invalid':"
-            + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\\n"
-            + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
+            + " was expecting ('true', 'false' or 'null')\\n"
+            + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
         assertNotNull(e.getCause());
         assertThat(e.getCause(), instanceOf(JsonParseException.class));
         assertThat(e.getCause(), hasToString(containsString("Unrecognized token 'invalid':"
-                + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n"
-                + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
+                + " was expecting ('true', 'false' or 'null')\n"
+                + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
     }
 
     public void testReformatSetting() {
diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle
index dc42f90476571..7cdec567ef8f4 100644
--- a/x-pack/plugin/sql/jdbc/build.gradle
+++ b/x-pack/plugin/sql/jdbc/build.gradle
@@ -44,8 +44,3 @@ dependencyLicenses {
 shadowJar {
   relocate 'com.fasterxml', 'org.elasticsearch.fasterxml'
 }
-
-thirdPartyAudit.ignoreMissingClasses(
-  'com.fasterxml.jackson.databind.ObjectMapper',
-  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
-)
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
deleted file mode 100644
index f23937b0d82a4..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
deleted file mode 100644
index 6c96c20f48232..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1ba01fef9c3b7ed388d91e71dc733b315c7374cd
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..378ba524422bc
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
@@ -0,0 +1 @@
+8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
deleted file mode 100644
index f23937b0d82a4..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
deleted file mode 100644
index f23937b0d82a4..0000000000000
--- a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
index 1409e80837768..2029493bcbec4 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
@@ -92,8 +92,7 @@ public void testNoType() {
     public void testInvalidJson() {
         IOException e = expectThrows(IOException.class, () -> parse("invalid_json.txt"));
         assertEquals(
-            "Can't parse error from Elasticsearch [Unrecognized token 'I': "
-                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] "
+            "Can't parse error from Elasticsearch [Unrecognized token 'I': was expecting 'null', 'true', 'false' or NaN] "
                 + "at [line 1 col 1]. Response:\n"
                 + "I'm not json at all",
             e.getMessage());
@@ -119,7 +118,7 @@ public void testTotalGarbage() {
             }).streamInput()));
         assertThat(e.getMessage(),
             startsWith("Can't parse error from Elasticsearch [Unrecognized token 'ÿ': "
-                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] at [line 1 col 4]. Response:\n"));
+                + "was expecting ('true', 'false' or 'null')] at [line 1 col 1]. Response:\n"));
     }
 
     public void testTooBig() {
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
deleted file mode 100644
index f23937b0d82a4..0000000000000
--- a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
new file mode 100644
index 0000000000000..e7ad1e74ed6b8
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
@@ -0,0 +1 @@
+876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file

From bd580527b22a1c2eda21b417bad50c0a35281f5b Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Mon, 16 Mar 2020 08:54:51 +0000
Subject: [PATCH 042/285] Do not log no-op reconnections at DEBUG (#53469)

Today the NodeConnectionsService emits a DEBUG-level log message each time it
calls TransportService#connectToNode, which happens for every node in the
cluster every ten seconds, and also at every cluster state update. That's a lot
of log messages. Most of these calls are no-ops and can be ignored, but if the
call was not a no-op then it may be worth investigating further. Since the logs
do not distinguish the interesting and uninteresting cases, they are not
useful.

This commit distinguishes the two cases and pushes the noisy logging for the
common no-op case down to TRACE level, leaving only useful and actionable
information in the DEBUG-level logs.
---
 .../cluster/NodeConnectionsService.java       |  38 ++++--
 .../cluster/NodeConnectionsServiceTests.java  | 119 +++++++++++++++++-
 2 files changed, 141 insertions(+), 16 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index b004af9d38ab6..acd3294f38641 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -304,20 +304,32 @@ private class ConnectionTarget {
             @Override
             protected void doRun() {
                 assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
-                transportService.connectToNode(discoveryNode, new ActionListener<Void>() {
-                    @Override
-                    public void onResponse(Void aVoid) {
-                        assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
-                        consecutiveFailureCount.set(0);
-                        logger.debug("connected to {}", discoveryNode);
-                        onCompletion(ActivityType.CONNECTING, null, disconnectActivity);
-                    }
+                if (transportService.nodeConnected(discoveryNode)) {
+                    // transportService.connectToNode is a no-op if already connected, but we don't want any DEBUG logging in this case
+                    // since we run this for every node on every cluster state update.
+                    logger.trace("still connected to {}", discoveryNode);
+                    onConnected();
+                } else {
+                    logger.debug("connecting to {}", discoveryNode);
+                    transportService.connectToNode(discoveryNode, new ActionListener<Void>() {
+                        @Override
+                        public void onResponse(Void aVoid) {
+                            assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held";
+                            logger.debug("connected to {}", discoveryNode);
+                            onConnected();
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            abstractRunnable.onFailure(e);
+                        }
+                    });
+                }
+            }
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        abstractRunnable.onFailure(e);
-                    }
-                });
+            private void onConnected() {
+                consecutiveFailureCount.set(0);
+                onCompletion(ActivityType.CONNECTING, null, disconnectActivity);
             }
 
             @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index eb2f1c9d845a1..afb63edfd15e1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cluster;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -31,10 +33,13 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -72,7 +77,6 @@
 public class NodeConnectionsServiceTests extends ESTestCase {
 
     private ThreadPool threadPool;
-    private MockTransport transport;
     private TransportService transportService;
     private Map<DiscoveryNode, CheckedRunnable<Exception>> nodeConnectionBlocks;
 
@@ -301,6 +305,116 @@ public void testOnlyBlocksOnConnectionsToNewNodes() throws Exception {
         }
     }
 
+    @TestLogging(reason="testing that DEBUG-level logging is reasonable", value="org.elasticsearch.cluster.NodeConnectionsService:DEBUG")
+    public void testDebugLogging() throws IllegalAccessException {
+        final DeterministicTaskQueue deterministicTaskQueue
+            = new DeterministicTaskQueue(builder().put(NODE_NAME_SETTING.getKey(), "node").build(), random());
+
+        MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
+        TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool());
+        transportService.start();
+        transportService.acceptIncomingRequests();
+
+        final NodeConnectionsService service
+            = new NodeConnectionsService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), transportService);
+        service.start();
+
+        final List<DiscoveryNode> allNodes = generateNodes();
+        final DiscoveryNodes targetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
+        service.connectToNodes(targetNodes, () -> {});
+        deterministicTaskQueue.runAllRunnableTasks();
+
+        // periodic reconnections to unexpectedly-disconnected nodes are logged
+        final Set<DiscoveryNode> disconnectedNodes = new HashSet<>(randomSubsetOf(allNodes));
+        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
+            transportService.disconnectFromNode(disconnectedNode);
+        }
+        MockLogAppender appender = new MockLogAppender();
+        try {
+            appender.start();
+            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            for (DiscoveryNode targetNode : targetNodes) {
+                if (disconnectedNodes.contains(targetNode)) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                } else {
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                }
+            }
+
+            runTasksUntil(deterministicTaskQueue, CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(Settings.EMPTY).millis());
+            appender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            appender.stop();
+        }        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
+            transportService.disconnectFromNode(disconnectedNode);
+        }
+
+        // changes to the expected set of nodes are logged, including reconnections to any unexpectedly-disconnected nodes
+        final DiscoveryNodes newTargetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
+        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
+            transportService.disconnectFromNode(disconnectedNode);
+        }
+        appender = new MockLogAppender();
+        try {
+            appender.start();
+            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            for (DiscoveryNode targetNode : targetNodes) {
+                if (disconnectedNodes.contains(targetNode) && newTargetNodes.get(targetNode.getId()) != null) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                } else {
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                }
+                if (newTargetNodes.get(targetNode.getId()) == null) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("disconnected from " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "disconnected from " + targetNode));
+                }
+            }
+            for (DiscoveryNode targetNode : newTargetNodes) {
+                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("disconnected from " + targetNode,
+                    "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                    "disconnected from " + targetNode));
+                if (targetNodes.get(targetNode.getId()) == null) {
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connecting to " + targetNode));
+                    appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode,
+                        "org.elasticsearch.cluster.NodeConnectionsService", Level.DEBUG,
+                        "connected to " + targetNode));
+                }
+            }
+
+            service.disconnectFromNodesExcept(newTargetNodes);
+            service.connectToNodes(newTargetNodes, () -> {});
+            deterministicTaskQueue.runAllRunnableTasks();
+            appender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+            appender.stop();
+        }
+    }
+
     private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long endTimeMillis) {
         while (deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) {
             if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) {
@@ -339,9 +453,8 @@ public void setUp() throws Exception {
         super.setUp();
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
         this.threadPool = threadPool;
-        this.transport = new MockTransport(threadPool);
         nodeConnectionBlocks = newConcurrentMap();
-        transportService = new TestTransportService(transport, threadPool);
+        transportService = new TestTransportService(new MockTransport(threadPool), threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
     }

From a2a47567369ff9e321bf19f05f08daddc02a54b9 Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Mon, 16 Mar 2020 09:54:10 +0000
Subject: [PATCH 043/285] New wildcard field optimised for wildcard queries
 (#49993)

Indexes values using size 3 ngrams and also stores the full original as a binary doc value.
Wildcard queries operate by using a cheap approximation query on the ngram field followed up by a more expensive verification query using an automaton on the binary doc values.  Also supports aggregations and sorting.
---
 docs/reference/mapping/types.asciidoc         |   4 +-
 .../reference/mapping/types/wildcard.asciidoc |  53 ++
 .../plain/BinaryDVIndexFieldData.java         |  17 +-
 .../license/XPackLicenseState.java            |  10 +
 .../test/wildcard/10_wildcard_basic.yml       | 218 +++++++
 x-pack/plugin/wildcard/build.gradle           |  18 +
 .../xpack/wildcard/Wildcard.java              |  31 +
 .../mapper/AutomatonQueryOnBinaryDv.java      | 104 ++++
 .../wildcard/mapper/WildcardFieldMapper.java  | 575 ++++++++++++++++++
 .../mapper/WildcardFieldMapperTests.java      | 331 ++++++++++
 .../mapper/WildcardFieldTypeTests.java        |  19 +
 11 files changed, 1363 insertions(+), 17 deletions(-)
 create mode 100644 docs/reference/mapping/types/wildcard.asciidoc
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
 create mode 100644 x-pack/plugin/wildcard/build.gradle
 create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
 create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
 create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
 create mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
 create mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java

diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index 1b5afad31f0c6..d26d897474deb 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -7,7 +7,7 @@ document:
 [float]
 === Core datatypes
 
-string::         <<text,`text`>> and <<keyword,`keyword`>>
+string::         <<text,`text`>>, <<keyword,`keyword`>> and <<wildcard,`wildcard`>>
 <<number>>::     `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
 <<date>>::       `date`
 <<date_nanos>>:: `date_nanos`
@@ -131,3 +131,5 @@ include::types/token-count.asciidoc[]
 include::types/shape.asciidoc[]
 
 include::types/constant-keyword.asciidoc[]
+
+include::types/wildcard.asciidoc[]
diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
new file mode 100644
index 0000000000000..51d10ff53ca92
--- /dev/null
+++ b/docs/reference/mapping/types/wildcard.asciidoc
@@ -0,0 +1,53 @@
+[role="xpack"]
+[testenv="basic"]
+[[wildcard]]
+=== Wildcard datatype
+++++
+<titleabbrev>Wildcard</titleabbrev>
+++++
+
+A `wildcard` field stores values optimised for wildcard grep-like queries.
+Wildcard queries are possible on other field types but suffer from constraints:
+* `text` fields limit matching of any wildcard expressions to individual tokens rather than the original whole value held in a field
+* `keyword` fields are untokenized but slow at performing wildcard queries (especially patterns with leading wildcards).
+
+Internally the `wildcard` field indexes the whole field value using ngrams and stores the full string.
+The index is used as a rough filter to cut down the number of values that are then checked by retrieving and checking the full values.
+This field is especially well suited to run grep-like queries on log lines. Storage costs are typically lower than those of `keyword`
+fields but search speeds for exact matches on full terms are slower.
+
+You index and search a wildcard field as follows
+
+[source,console]
+--------------------------------------------------
+PUT my_index
+{
+  "mappings": {
+    "properties": {
+      "my_wildcard": {
+        "type": "wildcard"
+      }
+    }
+  }
+}
+
+PUT my_index/_doc/1
+{
+  "my_wildcard" : "This string can be quite lengthy"
+}
+
+POST my_index/_doc/_search
+{
+  "query": {
+      "wildcard" : "*quite*lengthy"
+  }
+}
+
+
+--------------------------------------------------
+
+
+==== Limitations
+
+* `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.
+
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
index 27088382f2025..06352640dc162 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
@@ -21,8 +21,6 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedSetSortField;
-import org.apache.lucene.search.SortedSetSelector;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
@@ -54,20 +52,7 @@ public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exce
     public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested,
             boolean reverse) {
         XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
-        /**
-         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
-         * returns a custom sort field otherwise.
-         */
-        if (nested != null ||
-                (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) ||
-                (source.sortMissingFirst(missingValue) == false && source.sortMissingLast(missingValue) == false)) {
-            return new SortField(getFieldName(), source, reverse);
-        }
-        SortField sortField = new SortedSetSortField(fieldName, reverse,
-            sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN);
-        sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ?
-            SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST);
-        return sortField;
+        return new SortField(getFieldName(), source, reverse);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index 1d90ef9488789..4e58023eb260b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -613,6 +613,16 @@ public boolean isFlattenedAllowed() {
     public boolean isVectorsAllowed() {
         return allowForAllLicenses();
     }
+    
+
+    /**
+     * Determine if Wildcard support should be enabled.
+     * <p>
+     *  Wildcard is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isWildcardAllowed() {
+        return status.active;
+    }    
 
     public boolean isOdbcAllowed() {
         return isAllowedByLicense(OperationMode.PLATINUM);
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
new file mode 100644
index 0000000000000..c67a79c8218da
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
@@ -0,0 +1,218 @@
+setup:
+  - skip:
+      features: headers
+      version: " - 7.9.99"
+      reason: "wildcard fields were added from 8.0"
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            number_of_replicas: 0
+          mappings:
+            properties:
+              my_wildcard:
+                 type: wildcard
+  - do:
+      index:
+        index: test-index
+        id: 1
+        body:
+          my_wildcard: hello world
+  - do:
+      index:
+        index: test-index
+        id: 2
+        body:
+          my_wildcard: goodbye world
+
+  - do:
+      indices.refresh: {}
+
+---
+"Short prefix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hel*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Long prefix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hello wor*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Short unrooted query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Long unrooted query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello worl*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Short suffix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ld" }
+
+
+  - match: {hits.total.value: 2}
+
+---
+"Long suffix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello world" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"No wildcard wildcard query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hello world" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Term query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            term:
+              my_wildcard: "hello world"
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Terms query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            terms:
+              my_wildcard: ["hello world", "does not exist"]
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Prefix query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            prefix:
+              my_wildcard:
+                value: "hell*"
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Sequence fail":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*world*hello*" }
+
+
+  - match: {hits.total.value: 0}
+
+---
+"Aggs work":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*world*" }
+          aggs:
+            top_vals:
+              terms: {field: "my_wildcard" }
+
+
+  - match: {hits.total.value: 2}
+  - length: { aggregations.top_vals.buckets: 2 }
+
+---
+"Sort works":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          sort: [ { "my_wildcard": "desc" } ]
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.1._id: "2" }
+
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          sort: [ { "my_wildcard": "asc" } ]
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.1._id: "1" }
+
+
diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle
new file mode 100644
index 0000000000000..22e29aa0832f2
--- /dev/null
+++ b/x-pack/plugin/wildcard/build.gradle
@@ -0,0 +1,18 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'wildcard'
+  description 'A plugin for a keyword field type with efficient wildcard search'
+  classname 'org.elasticsearch.xpack.wildcard.Wildcard'
+  extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-wildcard'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+integTest.enabled = false
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
new file mode 100644
index 0000000000000..3749dc2622c8b
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class Wildcard extends Plugin implements MapperPlugin {
+
+
+    public Wildcard(Settings settings) {
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+        mappers.put(WildcardFieldMapper.CONTENT_TYPE, new WildcardFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(mappers);
+    }
+}
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
new file mode 100644
index 0000000000000..648fbc7e0cdc3
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/AutomatonQueryOnBinaryDv.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.ByteRunAutomaton;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Query that runs an Automaton across all binary doc values. 
+ * Expensive to run so normally used in conjunction with more selective query clauses.
+ */
+public class AutomatonQueryOnBinaryDv extends Query {
+
+    private final String field;
+    private final String matchPattern;
+    private final Automaton automaton;
+
+    public AutomatonQueryOnBinaryDv(String field, String matchPattern, Automaton automaton) {
+        this.field = field;
+        this.matchPattern = matchPattern;
+        this.automaton = automaton;
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+                
+        ByteRunAutomaton bytesMatcher = new ByteRunAutomaton(automaton);
+        
+        return new ConstantScoreWeight(this, boost) {
+
+            @Override
+            public Scorer scorer(LeafReaderContext context) throws IOException {
+                ByteArrayDataInput badi = new ByteArrayDataInput();
+                final BinaryDocValues values = DocValues.getBinary(context.reader(), field);               
+                TwoPhaseIterator twoPhase = new TwoPhaseIterator(values) {
+                    @Override
+                    public boolean matches() throws IOException {
+                        BytesRef arrayOfValues = values.binaryValue();
+                        badi.reset(arrayOfValues.bytes);
+                        badi.setPosition(arrayOfValues.offset);
+                        
+                        int size = badi.readVInt();
+                        for (int i=0; i< size; i++) {
+                            int valLength = badi.readVInt();
+                            if (bytesMatcher.run(arrayOfValues.bytes, badi.getPosition(), valLength)) {
+                                return true;
+                            }
+                            badi.skipBytes(valLength);
+                        }
+                        return false;
+                    }
+
+                    @Override
+                    public float matchCost() {
+                        // TODO: how can we compute this?
+                        return 1000f;
+                    }
+                };
+                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
+            }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                return true;
+            }
+        };
+    }
+    @Override
+    public String toString(String field) {
+        return field+":"+matchPattern;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        AutomatonQueryOnBinaryDv other = (AutomatonQueryOnBinaryDv) obj;
+        return Objects.equals(field, other.field)  && Objects.equals(matchPattern, other.matchPattern);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, matchPattern);
+    }
+
+}
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
new file mode 100644
index 0000000000000..e489d8a35bb9f
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -0,0 +1,575 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ngram.NGramTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.automaton.Automaton;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
+import org.elasticsearch.index.mapper.BinaryFieldMapper.CustomBinaryDocValuesField;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.similarity.SimilarityProvider;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.TypeParsers.parseField;
+
+/**
+ * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching
+ */
+public class WildcardFieldMapper extends FieldMapper {
+
+    public static final String CONTENT_TYPE = "wildcard";
+    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; 
+    public static final int NGRAM_SIZE = 3;        
+    static final NamedAnalyzer WILDCARD_ANALYZER = new NamedAnalyzer("_wildcard", AnalyzerScope.GLOBAL, new Analyzer() {
+        @Override
+        public TokenStreamComponents createComponents(String fieldName) {
+            Tokenizer tokenizer = new NGramTokenizer(NGRAM_SIZE, NGRAM_SIZE);
+            return new TokenStreamComponents(tokenizer);
+        }
+    });    
+
+    public static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new WildcardFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setIndexAnalyzer(WILDCARD_ANALYZER);
+            FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.setStoreTermVectorOffsets(false);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.freeze();
+        }
+        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;        
+    }
+
+    public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
+        protected int ignoreAbove = Defaults.IGNORE_ABOVE;
+
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public Builder docValues(boolean docValues) {
+            if (docValues == false) {
+                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");                
+            }
+            return this;
+        }
+
+        @Override
+        public Builder indexOptions(IndexOptions indexOptions) {
+            if (indexOptions != IndexOptions.DOCS) {
+                throw new MapperParsingException("The field [" + name + "] cannot have indexOptions = " + indexOptions);
+            }
+            return this;
+        }
+
+        @Override
+        public Builder store(boolean store) {
+            if (store) {
+                throw new MapperParsingException("The field [" + name + "] cannot have store = true");                
+            }
+            return this;
+        }
+
+        @Override
+        public Builder similarity(SimilarityProvider similarity) {
+            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");                
+        }
+
+        @Override
+        public Builder index(boolean index) {
+            if (index == false) {
+                throw new MapperParsingException("The field [" + name + "] cannot have index = false");                
+            }
+            return this;
+        }
+        
+        public Builder ignoreAbove(int ignoreAbove) {
+            if (ignoreAbove < 0) {
+                throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
+            }
+            this.ignoreAbove = ignoreAbove;
+            return this;
+        }        
+          
+        
+        @Override
+        protected void setupFieldType(BuilderContext context) {
+            super.setupFieldType(context);
+            fieldType().setHasDocValues(true);
+            fieldType().setTokenized(false);
+            fieldType().setIndexOptions(IndexOptions.DOCS);                
+        }
+
+        @Override
+        public WildcardFieldType fieldType() {
+            return (WildcardFieldType) super.fieldType();
+        }
+
+        @Override
+        public WildcardFieldMapper build(BuilderContext context) {
+            setupFieldType(context);            
+            return new WildcardFieldMapper(
+                    name, fieldType, defaultFieldType, ignoreAbove, 
+                    context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+                throws MapperParsingException {
+            WildcardFieldMapper.Builder builder = new WildcardFieldMapper.Builder(name);
+            parseField(builder, name, node, parserContext);
+            
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String propName = entry.getKey();
+                Object propNode = entry.getValue();
+                if (propName.equals("ignore_above")) {
+                    builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
+                    iterator.remove();
+                }
+            }            
+            
+            return builder;
+        }
+    }
+    
+     public static final char TOKEN_START_OR_END_CHAR = 0;    
+    
+     public static final class WildcardFieldType extends MappedFieldType {
+
+        public WildcardFieldType() {            
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);            
+        }
+
+        protected WildcardFieldType(WildcardFieldType ref) {
+            super(ref);
+        }
+
+        public WildcardFieldType clone() {
+            WildcardFieldType result = new WildcardFieldType(this);
+            return result;
+        }
+        
+                
+        // Holds parsed information about the wildcard pattern
+        static class PatternStructure {
+            boolean openStart, openEnd, hasSymbols;            
+            int lastGap =0;
+            int wildcardCharCount, wildcardStringCount;
+            String[] fragments;
+            Integer []  precedingGapSizes;
+            final String pattern;
+            
+            @SuppressWarnings("fallthrough") // Intentionally uses fallthrough mirroring implementation in Lucene's WildcardQuery
+            PatternStructure (String wildcardText) {
+                this.pattern = wildcardText;
+                ArrayList<String> fragmentList = new ArrayList<>();
+                ArrayList<Integer> precedingGapSizeList = new ArrayList<>();
+                StringBuilder sb = new StringBuilder();               
+                for (int i = 0; i < wildcardText.length();) {
+                    final int c = wildcardText.codePointAt(i);
+                    int length = Character.charCount(c);
+                    switch (c) {
+                    case WildcardQuery.WILDCARD_STRING:
+                        if (i == 0) {
+                            openStart = true;
+                        }
+                        openEnd = true;                        
+                        hasSymbols = true;
+                        wildcardStringCount++;
+                        
+                        if (sb.length() > 0) {
+                            precedingGapSizeList.add(lastGap);
+                            fragmentList.add(sb.toString());
+                            sb = new StringBuilder();
+                        }
+                        lastGap = Integer.MAX_VALUE;
+                        break;
+                    case WildcardQuery.WILDCARD_CHAR:
+                        if (i == 0) {
+                            openStart = true;
+                        }
+                        hasSymbols = true;
+                        wildcardCharCount++;
+                        openEnd = true;
+                        if (sb.length() > 0) {
+                            precedingGapSizeList.add(lastGap);
+                            fragmentList.add(sb.toString());
+                            sb = new StringBuilder();       
+                            lastGap = 0;
+                        }
+                        
+                        if (lastGap != Integer.MAX_VALUE) {
+                            lastGap++;
+                        }
+                        break;
+                    case WildcardQuery.WILDCARD_ESCAPE:
+                        // add the next codepoint instead, if it exists
+                        if (i + length < wildcardText.length()) {
+                            final int nextChar = wildcardText.codePointAt(i + length);
+                            length += Character.charCount(nextChar);
+                            sb.append(Character.toChars(nextChar));
+                            openEnd = false;
+                            break;
+                        } // else fallthru, lenient parsing with a trailing \
+                    default:
+                        openEnd = false;
+                        sb.append(Character.toChars(c));
+                    }
+                    i += length;
+                }
+                if (sb.length() > 0) {
+                    precedingGapSizeList.add(lastGap);
+                    fragmentList.add(sb.toString());
+                    lastGap = 0;
+                }
+                fragments = fragmentList.toArray(new String[0]);
+                precedingGapSizes = precedingGapSizeList.toArray(new Integer[0]);
+                
+            }
+            
+            public boolean needsVerification() {
+                // Return true if term queries are not enough evidence
+                if (fragments.length == 1 && wildcardCharCount == 0) {
+                    // The one case where we don't need verification is when 
+                    // we have a single fragment and no ? characters
+                    return false;
+                }
+                return true;
+            }
+            
+            // Returns number of positions for last gap (Integer.MAX means unlimited gap)
+            public int getPrecedingGapSize(int fragmentNum) {
+                return precedingGapSizes[fragmentNum];
+            }
+
+            public boolean isMatchAll() {
+                return fragments.length == 0 && wildcardStringCount >0 && wildcardCharCount ==0;
+            }
+
+            @Override
+            public int hashCode() {
+                return pattern.hashCode();
+            }
+
+            @Override
+            public boolean equals(Object obj) {
+                PatternStructure other = (PatternStructure) obj;
+                return pattern.equals(other.pattern);
+            }
+            
+            
+        }
+        
+
+        @Override
+        public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
+            PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
+            ArrayList<String> tokens = new ArrayList<>();
+
+            for (int i = 0; i < patternStructure.fragments.length; i++) {
+                String fragment = patternStructure.fragments[i];
+                int fLength = fragment.length();
+                if (fLength == 0) {
+                    continue;
+                }
+                
+                // Add any start/end of string character
+                if (i == 0 && patternStructure.openStart == false) {
+                    // Start-of-string anchored (is not a leading wildcard)
+                    fragment = TOKEN_START_OR_END_CHAR + fragment;
+                }
+                if (patternStructure.openEnd == false && i == patternStructure.fragments.length - 1) {
+                    // End-of-string anchored (is not a trailing wildcard)
+                    fragment = fragment + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
+                }
+                if (fragment.codePointCount(0, fragment.length()) <= NGRAM_SIZE) {
+                    tokens.add(fragment);
+                } else {
+                    // Break fragment into multiple Ngrams                
+                    TokenStream tokenizer = WILDCARD_ANALYZER.tokenStream(name(), fragment);
+                    CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
+                    String lastUnusedToken = null;
+                    try {
+                        tokenizer.reset();
+                        boolean takeThis = true;
+                        // minimise number of terms searched - eg for "12345" and 3grams we only need terms
+                        // `123` and `345` - no need to search for 234. We take every other ngram.
+                        while (tokenizer.incrementToken()) {
+                            String tokenValue = termAtt.toString();
+                            if (takeThis) {
+                                tokens.add(tokenValue);
+                            } else {
+                                lastUnusedToken = tokenValue;                                
+                            }
+                            // alternate
+                            takeThis = !takeThis;
+                        }
+                        if (lastUnusedToken != null) {
+                            // given `cake` and 3 grams the loop above would output only `cak` and we need to add trailing
+                            // `ake` to complete the logic.
+                            tokens.add(lastUnusedToken);
+                        }
+                        tokenizer.end();
+                        tokenizer.close();
+                    } catch (IOException ioe) {
+                        throw new ElasticsearchParseException("Error parsing wildcard query pattern fragment [" + fragment + "]");
+                    }
+                }
+            }
+
+            if (patternStructure.isMatchAll()) {
+                return new MatchAllDocsQuery();
+            } 
+            BooleanQuery approximation = createApproximationQuery(tokens);
+            if (approximation.clauses().size() > 1 || patternStructure.needsVerification()) {
+                BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder();
+                verifyingBuilder.add(new BooleanClause(approximation, Occur.MUST));
+                Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), wildcardPattern));
+                verifyingBuilder.add(new BooleanClause(new AutomatonQueryOnBinaryDv(name(), wildcardPattern, automaton), Occur.MUST));
+                return verifyingBuilder.build();
+            }
+            return approximation;
+        }                
+
+        private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
+            BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
+            if (tokens.size() <= MAX_CLAUSES_IN_APPROXIMATION_QUERY) {
+                for (String token : tokens) {
+                    addClause(token, bqBuilder);
+                }
+                return bqBuilder.build();
+            }
+            // Thin out the number of clauses using a selection spread evenly across the range
+            float step = (float) (tokens.size() - 1) / (float) (MAX_CLAUSES_IN_APPROXIMATION_QUERY - 1); // set step size
+            for (int i = 0; i < MAX_CLAUSES_IN_APPROXIMATION_QUERY; i++) {
+                addClause(tokens.get(Math.round(step * i)), bqBuilder); // add each element of a position which is a multiple of step
+            }
+            // TODO we can be smarter about pruning here. e.g.
+            // * Avoid wildcard queries if there are sufficient numbers of other terms that are full 3grams that are cheaper term queries
+            // * We can select terms on their scarcity rather than even spreads across the search string.
+            
+            return bqBuilder.build();
+        }
+
+        private void addClause(String token, BooleanQuery.Builder bqBuilder) {
+            assert token.codePointCount(0, token.length()) <= NGRAM_SIZE;
+            if (token.codePointCount(0, token.length()) == NGRAM_SIZE) {
+                TermQuery tq = new TermQuery(new Term(name(), token));
+                bqBuilder.add(new BooleanClause(tq, Occur.MUST));
+            } else {
+                WildcardQuery wq = new WildcardQuery(new Term(name(), token + "*"));
+                wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
+                bqBuilder.add(new BooleanClause(wq, Occur.MUST));
+            }
+
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            return new DocValuesFieldExistsQuery(name());
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, context);
+        }
+        
+        @Override
+        public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+            return wildcardQuery(value + "*", method, context);
+        }        
+
+        @Override
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            BooleanQuery.Builder bq = new BooleanQuery.Builder();
+            for (Object value : values) {
+                bq.add(termQuery(value, context), Occur.SHOULD);
+            }
+            return new ConstantScoreQuery(bq.build());
+        }     
+                
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+            failIfNoDocValues();
+            return new IndexFieldData.Builder() {
+
+                @Override
+                public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                        CircuitBreakerService breakerService, MapperService mapperService) {
+                    return new WildcardBytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
+                }};
+        }        
+    }
+     
+    static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
+
+        WildcardBytesBinaryDVIndexFieldData(Index index, String fieldName) {
+            super(index, fieldName);
+        }
+
+        @Override
+        public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
+                    sortMode, nested);
+            return new SortField(getFieldName(), source, reverse);
+        }
+    
+    }
+
+    private int ignoreAbove;
+
+    private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+                int ignoreAbove, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
+        this.ignoreAbove = ignoreAbove;
+        assert fieldType.indexOptions() == IndexOptions.DOCS;
+        
+        ngramFieldType = fieldType.clone();
+        ngramFieldType.setTokenized(true);                    
+        ngramFieldType.freeze();
+    }
+
+    /** Values that have more chars than the return value of this method will
+     *  be skipped at parsing time. */
+    // pkg-private for testing
+    int ignoreAbove() {
+        return ignoreAbove;
+    }    
+    
+    @Override
+    protected WildcardFieldMapper clone() {
+        return (WildcardFieldMapper) super.clone();
+    }
+
+    @Override
+    public WildcardFieldType fieldType() {
+        return (WildcardFieldType) super.fieldType();
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+        if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
+            builder.field("ignore_above", ignoreAbove);
+        }
+    }
+    
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        final String value;
+        if (context.externalValueSet()) {
+            value = context.externalValue().toString();
+        } else {
+            XContentParser parser = context.parser();
+            if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+                value = fieldType().nullValueAsString();
+            } else {
+                value =  parser.textOrNull();
+            }
+        }
+        ParseContext.Document parseDoc = context.doc();
+        
+        createFields(value, parseDoc, fields);        
+    }   
+    
+    // For internal use by Lucene only - used to define ngram index
+    final MappedFieldType ngramFieldType;
+    
+    void createFields(String value, Document parseDoc, List<IndexableField>fields) {
+        if (value == null || value.length() > ignoreAbove) {
+            return;
+        }
+        String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
+        Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
+        fields.add(ngramField);
+        
+        CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name());
+        if (dvField == null) {
+            dvField = new CustomBinaryDocValuesField(fieldType().name(), value.getBytes(StandardCharsets.UTF_8));
+            parseDoc.addWithKey(fieldType().name(), dvField);
+        } else {
+            dvField.add(value.getBytes(StandardCharsets.UTF_8));
+        }        
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+    
+
+    @Override
+    protected void doMerge(Mapper mergeWith) {
+        super.doMerge(mergeWith);
+        this.ignoreAbove = ((WildcardFieldMapper) mergeWith).ignoreAbove;
+    }    
+}
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
new file mode 100644
index 0000000000000..f8c8ddc7f5eef
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
@@ -0,0 +1,331 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper.Builder;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.function.BiFunction;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class WildcardFieldMapperTests extends ESTestCase {
+
+    private static final String KEYWORD_FIELD_NAME = "keyword_field";
+    private static final String WILDCARD_FIELD_NAME = "wildcard_field";
+    static final int MAX_FIELD_LENGTH = 100;
+    static WildcardFieldMapper wildcardFieldType;
+    static KeywordFieldMapper keywordFieldType;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        Builder builder = new WildcardFieldMapper.Builder(WILDCARD_FIELD_NAME);
+        builder.ignoreAbove(MAX_FIELD_LENGTH);        
+        wildcardFieldType = builder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));
+        
+        
+        org.elasticsearch.index.mapper.KeywordFieldMapper.Builder kwBuilder = new KeywordFieldMapper.Builder(KEYWORD_FIELD_NAME);
+        keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));        
+        super.setUp();
+    }
+    
+    public void testIllegalDocValuesArgument() {
+        Builder ft = new WildcardFieldMapper.Builder("test");
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> ft.docValues(false));
+        assertEquals("The field [test] cannot have doc values = false", e.getMessage());
+    }        
+
+    public void testIllegalIndexedArgument() {
+        Builder ft = new WildcardFieldMapper.Builder("test");
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> ft.index(false));
+        assertEquals("The field [test] cannot have index = false", e.getMessage());
+    }
+    
+    public void testTooBigKeywordField() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+        
+        // Create a string that is too large and will not be indexed
+        String docContent = randomABString(MAX_FIELD_LENGTH + 1);
+        Document doc = new Document();
+        ParseContext.Document parseDoc = new ParseContext.Document();                        
+        addFields(parseDoc, doc, docContent);
+        indexDoc(parseDoc, doc, iw);        
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("*a*", null, null);
+        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
+        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
+                
+        reader.close();
+        dir.close();        
+    }
+    
+    //Test long query strings don't cause exceptions
+    public void testTooBigQueryField() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+        
+        // Create a string that is too large and will not be indexed
+        String docContent = randomABString(10);
+        Document doc = new Document();
+        ParseContext.Document parseDoc = new ParseContext.Document();                        
+        addFields(parseDoc, doc, docContent);
+        indexDoc(parseDoc, doc, iw);        
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        String queryString = randomABString((BooleanQuery.getMaxClauseCount() * 2) + 1);
+        Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(queryString, null, null);
+        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
+        assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
+                
+        reader.close();
+        dir.close();        
+    }    
+    
+    
+    public void testSearchResultsVersusKeywordField() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+        int numDocs = 100;
+        HashSet<String> values = new HashSet<>();
+        for (int i = 0; i < numDocs; i++) {
+            Document doc = new Document();
+            ParseContext.Document parseDoc = new ParseContext.Document();                        
+            String docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
+            if (values.contains(docContent) == false) {                
+                addFields(parseDoc, doc, docContent);
+                values.add(docContent);
+            }
+            // Occasionally add a multi-value field
+            if (randomBoolean()) {
+                docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
+                if (values.contains(docContent) == false) {                
+                    addFields(parseDoc, doc, docContent);
+                    values.add(docContent);
+                }                
+            }
+            indexDoc(parseDoc, doc, iw);
+            
+        }
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        int numSearches = 100;
+        for (int i = 0; i < numSearches; i++) {
+            String randomWildcardPattern = getRandomWildcardPattern();
+
+            Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(randomWildcardPattern, null, null);
+            TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, values.size() + 1, Sort.INDEXORDER);
+
+            Query keywordFieldQuery = new WildcardQuery(new Term(KEYWORD_FIELD_NAME, randomWildcardPattern));
+            TopDocs kwTopDocs = searcher.search(keywordFieldQuery, values.size() + 1, Sort.INDEXORDER);
+
+            assertThat(kwTopDocs.totalHits.value, equalTo(wildcardFieldTopDocs.totalHits.value));
+
+            HashSet<Integer> expectedDocs = new HashSet<>();
+            for (ScoreDoc topDoc : kwTopDocs.scoreDocs) {
+                expectedDocs.add(topDoc.doc);
+            }
+            for (ScoreDoc wcTopDoc : wildcardFieldTopDocs.scoreDocs) {
+                assertTrue(expectedDocs.remove(wcTopDoc.doc));
+            }
+            assertThat(expectedDocs.size(), equalTo(0));
+        }
+        
+        
+        //Test keyword and wildcard sort operations are also equivalent
+        QueryShardContext shardContextMock = createMockShardContext();
+        
+        FieldSortBuilder wildcardSortBuilder = new FieldSortBuilder(WILDCARD_FIELD_NAME);
+        SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field;        
+        ScoreDoc[] wildcardHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(wildcardSortField)).scoreDocs;
+
+        FieldSortBuilder keywordSortBuilder = new FieldSortBuilder(KEYWORD_FIELD_NAME);
+        SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field;        
+        ScoreDoc[] keywordHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(keywordSortField)).scoreDocs;
+        
+        assertThat(wildcardHits.length, equalTo(keywordHits.length));
+        for (int i = 0; i < wildcardHits.length; i++) {
+            assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc));            
+        }
+        
+        reader.close();
+        dir.close();
+    }
+
+    
+    
+    protected MappedFieldType provideMappedFieldType(String name) {
+        if (name.equals(WILDCARD_FIELD_NAME)) {
+            return wildcardFieldType.fieldType();            
+        } else {
+            return keywordFieldType.fieldType();
+        }
+    }    
+    
+    protected final QueryShardContext createMockShardContext() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index,
+            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class));
+        BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup = (fieldType, fieldIndexName) -> {
+            IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName);
+            return builder.build(idxSettings, fieldType, new IndexFieldDataCache.None(), null, null);
+        };
+        return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup,
+                null, null, null, xContentRegistry(), null, null, null,
+                () -> randomNonNegativeLong(), null, null, () -> true) {
+
+            @Override
+            public MappedFieldType fieldMapper(String name) {
+                return provideMappedFieldType(name);
+            }
+        };
+    }       
+    
+    private void addFields(ParseContext.Document parseDoc, Document doc, String docContent) throws IOException {
+        ArrayList<IndexableField> fields = new ArrayList<>();
+        wildcardFieldType.createFields(docContent, parseDoc, fields);
+
+        for (IndexableField indexableField : fields) {
+            doc.add(indexableField);
+        }
+        // Add keyword fields too
+        doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent)));        
+        doc.add(new StringField(KEYWORD_FIELD_NAME, docContent, Field.Store.YES));
+    }
+
+    private void indexDoc(ParseContext.Document parseDoc, Document doc, RandomIndexWriter iw) throws IOException {
+        IndexableField field = parseDoc.getByKey(wildcardFieldType.name());
+        if (field != null) {
+            doc.add(field);
+        }
+        iw.addDocument(doc);
+    }
+
+    protected IndexSettings createIndexSettings() {
+        return new IndexSettings(
+                IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                        .numberOfShards(1).numberOfReplicas(0).creationDate(System.currentTimeMillis()).build(),
+                Settings.EMPTY);
+    }
+
+
+    static String randomABString(int minLength) {
+        StringBuilder sb = new StringBuilder();
+        while (sb.length() < minLength) {
+            if (randomBoolean()) {
+                sb.append("a");
+            } else {
+                sb.append("b");
+            }
+        }
+        return sb.toString();
+    }
+
+    private void randomSyntaxChar(StringBuilder sb) {
+        switch (randomInt(3)) {
+        case 0:
+            sb.append(WildcardQuery.WILDCARD_CHAR);
+            break;
+        case 1:
+            sb.append(WildcardQuery.WILDCARD_STRING);
+            break;
+        case 2:
+            sb.append(WildcardQuery.WILDCARD_ESCAPE);
+            sb.append(WildcardQuery.WILDCARD_STRING);
+            break;
+        case 3:
+            sb.append(WildcardQuery.WILDCARD_ESCAPE);
+            sb.append(WildcardQuery.WILDCARD_CHAR);
+            break;
+        }
+    }
+
+    private String getRandomWildcardPattern() {
+        StringBuilder sb = new StringBuilder();
+        int numFragments = 1 + randomInt(4);
+        if (randomInt(10) == 1) {
+            randomSyntaxChar(sb);
+        }
+        for (int i = 0; i < numFragments; i++) {
+            if (i > 0) {
+                randomSyntaxChar(sb);
+            }
+            sb.append(randomABString(1 + randomInt(6)));
+        }
+        if (randomInt(10) == 1) {
+            randomSyntaxChar(sb);
+        }
+        return sb.toString();
+    }
+}
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java
new file mode 100644
index 0000000000000..0d090e4a25058
--- /dev/null
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.wildcard.mapper;
+
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+public class WildcardFieldTypeTests extends FieldTypeTestCase {
+
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new WildcardFieldMapper.WildcardFieldType();
+    }  
+}

From 340fcd1c5aa54c01a85ecd01945408f504ebdf59 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Mon, 16 Mar 2020 10:18:12 +0000
Subject: [PATCH 044/285] Mute SnapshotLifecycleRestIT.testFullPolicySnapshot
 (#53588)

https://github.com/elastic/elasticsearch/issues/50358
---
 .../org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index de77dbed9f7f4..f2c9e9c609502 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -91,6 +91,7 @@ public void testMissingRepo() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50358")
     public void testFullPolicySnapshot() throws Exception {
         final String indexName = "test";
         final String policyName = "full-policy";

From 01eee1a97f9541f9a3df62231588cb81976d4e84 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova <mayya.sharipova@elastic.co>
Date: Mon, 16 Mar 2020 06:49:37 -0400
Subject: [PATCH 045/285] Highlighters skip ignored keyword values (#53408)

Keyword field values with length more than ignore_above are not
indexed. But highlighters still were retrieving these values
from _source and were trying to highlight them. This sometimes lead to
errors if a field length exceeded  max_analyzed_offset. But also this
is a wrong behaviour to attempt to highlight something that was not
ignored during indexing.

This PR checks if a keyword value was ignored because of its length,
and if yes, skips highlighting it.

Closes #43800
---
 .../search.highlight/40_keyword_ignore.yml    | 64 +++++++++++++++++++
 .../index/mapper/KeywordFieldMapper.java      |  3 +-
 .../subphase/highlight/PlainHighlighter.java  | 13 +++-
 .../highlight/UnifiedHighlighter.java         | 14 +++-
 4 files changed, 89 insertions(+), 5 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
new file mode 100644
index 0000000000000..16442d2b3d823
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/40_keyword_ignore.yml
@@ -0,0 +1,64 @@
+---
+setup:
+  - do:
+      indices.create:
+          index: test-index
+          body:
+            mappings:
+              "properties":
+                "k1":
+                  "type": "keyword"
+                "k2":
+                  "type": "keyword"
+                  "ignore_above": 3
+  - do:
+      bulk:
+        index: test-index
+        refresh: true
+        body:
+          - '{"index": {"_id": "1"}}'
+          - '{"k1": "123", "k2" : "123"}'
+          - '{"index": {"_id": "2"}}'
+          - '{"k1": "1234", "k2" : "1234"}'
+
+---
+"Plain Highligher should skip highlighting ignored keyword values":
+  - skip:
+      version: " - 7.9.99"
+      reason: "skip highlighting of ignored values was introduced in 7.7"
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            prefix:
+              k1: "12"
+          highlight:
+            require_field_match: false
+            fields:
+              k2:
+                type: plain
+
+  - match: {hits.hits.0.highlight.k2.0: "<em>123</em>"}
+  - is_false: hits.hits.1.highlight # no highlight for a value that was ignored
+
+---
+"Unified Highligher should skip highlighting ignored keyword values":
+  - skip:
+      version: " - 7.9.99"
+      reason: "skip highlighting of ignored values was introduced in 7.7"
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            prefix:
+              k1: "12"
+          highlight:
+            require_field_match: false
+            fields:
+              k2:
+                type: unified
+
+  - match: {hits.hits.0.highlight.k2.0: "<em>123</em>"}
+  - is_false: hits.hits.1.highlight # no highlight for a value that was ignored
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 524a1eb79f133..b83427a53a3ed 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -322,8 +322,7 @@ protected KeywordFieldMapper(String simpleName, MappedFieldType fieldType, Mappe
 
     /** Values that have more chars than the return value of this method will
      *  be skipped at parsing time. */
-    // pkg-private for testing
-    int ignoreAbove() {
+    public int ignoreAbove() {
         return ignoreAbove;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
index b792be60f78fc..d1743e6182e2f 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@@ -102,6 +103,12 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         ArrayList<TextFragment> fragsList = new ArrayList<>();
         List<Object> textsToHighlight;
         Analyzer analyzer = context.getMapperService().documentMapper().mappers().indexAnalyzer();
+        Integer keywordIgnoreAbove = null;
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+            KeywordFieldMapper mapper = (KeywordFieldMapper) context.getMapperService().documentMapper()
+                .mappers().getMapper(highlighterContext.fieldName);
+            keywordIgnoreAbove = mapper.ignoreAbove();
+        }
         final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset();
 
         try {
@@ -110,7 +117,11 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
 
             for (Object textToHighlight : textsToHighlight) {
                 String text = convertFieldValue(fieldType, textToHighlight);
-                if (text.length() > maxAnalyzedOffset) {
+                int textLength = text.length();
+                if (keywordIgnoreAbove != null  && textLength > keywordIgnoreAbove) {
+                    continue; // skip highlighting keyword terms that were ignored during indexing
+                }
+                if (textLength > maxAnalyzedOffset) {
                     throw new IllegalArgumentException(
                         "The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
                             "] doc of [" + context.index().getName() + "] index " +
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index 81f700920f827..28ff7220d9bd7 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@@ -65,11 +66,16 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
         FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
         Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
         final int maxAnalyzedOffset = context.getIndexSettings().getHighlightMaxAnalyzedOffset();
+        Integer keywordIgnoreAbove = null;
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+            KeywordFieldMapper mapper = (KeywordFieldMapper) context.getMapperService().documentMapper()
+                .mappers().getMapper(highlighterContext.fieldName);
+            keywordIgnoreAbove = mapper.ignoreAbove();
+        }
 
         List<Snippet> snippets = new ArrayList<>();
         int numberOfFragments = field.fieldOptions().numberOfFragments();
         try {
-
             final Analyzer analyzer = getAnalyzer(context.getMapperService().documentMapper(), hitContext);
             List<Object> fieldValues = loadFieldValues(fieldType, field, context, hitContext,
                 highlighterContext.highlight.forceSource(field));
@@ -81,7 +87,11 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
             final CustomUnifiedHighlighter highlighter;
             final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
             final OffsetSource offsetSource = getOffsetSource(fieldType);
-            if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValue.length() > maxAnalyzedOffset)) {
+            int fieldValueLength = fieldValue.length();
+            if (keywordIgnoreAbove != null  && fieldValueLength > keywordIgnoreAbove) {
+                return null; // skip highlighting keyword terms that were ignored during indexing
+            }
+            if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValueLength > maxAnalyzedOffset)) {
                 throw new IllegalArgumentException(
                     "The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
                         "] doc of [" + context.index().getName() + "] index " + "has exceeded [" +

From eaa8ead6742a8e7dcf343bcbaff8de031550fd77 Mon Sep 17 00:00:00 2001
From: Marios Trivyzas <matriv@gmail.com>
Date: Mon, 16 Mar 2020 11:55:08 +0100
Subject: [PATCH 046/285] SQL: Fix issue with LIKE/RLIKE as painless script
 (#53495)

Add missing asScript() implementation for LIKE/RLIKE expressions.

When LIKE/RLIKE are used for example in GROUP BY or are wrapped with
scalar functions in a WHERE clause, the translation must produce a
painless script which will be executed to implement the correct
behaviour and previously this was completely missing, and as a
consquence wrong results were silently (no error) returned.

Fixes: #53486
---
 .../ql/expression/predicate/regex/Like.java   | 13 --------
 .../predicate/regex/LikePattern.java          |  8 ++---
 .../ql/expression/predicate/regex/RLike.java  | 17 ++---------
 .../predicate/regex/RLikePattern.java         | 20 +++++++++++++
 .../predicate/regex/RegexMatch.java           | 30 +++++++++++++++++--
 .../predicate/regex/StringPattern.java        | 13 ++++++++
 .../ql/planner/ExpressionTranslators.java     |  4 +--
 .../ql/optimizer/OptimizerRulesTests.java     |  5 ++--
 .../xpack/sql/parser/ExpressionBuilder.java   |  3 +-
 .../xpack/sql/optimizer/OptimizerTests.java   |  3 +-
 .../sql/planner/QueryTranslatorTests.java     | 17 +++++++++++
 11 files changed, 92 insertions(+), 41 deletions(-)
 create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
 create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java

diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
index dbe751276be0d..dfde94b5df76e 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java
@@ -6,8 +6,6 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
-import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 
@@ -26,15 +24,4 @@ protected NodeInfo<Like> info() {
     protected Like replaceChild(Expression newLeft) {
         return new Like(source(), newLeft, pattern());
     }
-
-    @Override
-    public Boolean fold() {
-        Object val = field().fold();
-        return RegexOperation.match(val, pattern().asJavaRegex());
-    }
-
-    @Override
-    protected Processor makeProcessor() {
-        return new RegexProcessor(pattern().asJavaRegex());
-    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
index fd6663b4b5ac3..096f7d8ae27e1 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java
@@ -17,7 +17,7 @@
  *
  * To prevent conflicts with ES, the string and char must be validated to not contain '*'.
  */
-public class LikePattern {
+public class LikePattern implements StringPattern {
 
     private final String pattern;
     private final char escape;
@@ -43,9 +43,7 @@ public char escape() {
         return escape;
     }
 
-    /**
-     * Returns the pattern in (Java) regex format.
-     */
+    @Override
     public String asJavaRegex() {
         return regex;
     }
@@ -83,4 +81,4 @@ public boolean equals(Object obj) {
         return Objects.equals(pattern, other.pattern)
                 && escape == other.escape;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
index 9ec70bc79d1d8..af40f4ffb1750 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLike.java
@@ -6,14 +6,12 @@
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
-import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 
-public class RLike extends RegexMatch<String> {
+public class RLike extends RegexMatch<RLikePattern> {
 
-    public RLike(Source source, Expression value, String pattern) {
+    public RLike(Source source, Expression value, RLikePattern pattern) {
         super(source, value, pattern);
     }
 
@@ -26,15 +24,4 @@ protected NodeInfo<RLike> info() {
     protected RLike replaceChild(Expression newChild) {
         return new RLike(source(), newChild, pattern());
     }
-    
-    @Override
-    public Boolean fold() {
-        Object val = field().fold();
-        return RegexOperation.match(val, pattern());
-    }
-
-    @Override
-    protected Processor makeProcessor() {
-        return new RegexProcessor(pattern());
-    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
new file mode 100644
index 0000000000000..69747f03e2150
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ql.expression.predicate.regex;
+
+public class RLikePattern implements StringPattern {
+
+    private final String regexpPattern;
+
+    public RLikePattern(String regexpPattern) {
+        this.regexpPattern = regexpPattern;
+    }
+
+    @Override
+    public String asJavaRegex() {
+        return regexpPattern;
+    }
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
index 517c243992c24..565a75d489765 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java
@@ -10,15 +10,19 @@
 import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.Nullability;
 import org.elasticsearch.xpack.ql.expression.function.scalar.UnaryScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.Objects;
 
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
 
-public abstract class RegexMatch<T> extends UnaryScalarFunction {
+public abstract class RegexMatch<T extends StringPattern> extends UnaryScalarFunction {
 
     private final T pattern;
     
@@ -54,8 +58,30 @@ public boolean foldable() {
         // right() is not directly foldable in any context but Like can fold it.
         return field().foldable();
     }
-    
+
+    @Override
+    public Boolean fold() {
+        Object val = field().fold();
+        return RegexProcessor.RegexOperation.match(val, pattern().asJavaRegex());
+    }
+
     @Override
+    protected Processor makeProcessor() {
+        return new RegexProcessor(pattern().asJavaRegex());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate fieldAsScript = asScript(field());
+        return new ScriptTemplate(
+                formatTemplate(format("{sql}.", "regex({},{})", fieldAsScript.template())),
+                paramsBuilder()
+                        .script(fieldAsScript.params())
+                        .variable(pattern.asJavaRegex())
+                        .build(),
+                dataType());
+    }
+
     public boolean equals(Object obj) {
         return super.equals(obj) && Objects.equals(((RegexMatch<?>) obj).pattern(), pattern());
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
new file mode 100644
index 0000000000000..6dac865b4ac5b
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ql.expression.predicate.regex;
+
+interface StringPattern {
+    /**
+     * Returns the pattern in (Java) regex format.
+     */
+    String asJavaRegex();
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
index ebbd84ef07c7c..042e886f5d0fb 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
@@ -111,7 +111,7 @@ public static Query doTranslate(RegexMatch e, TranslatorHandler handler) {
             }
 
             if (e instanceof RLike) {
-                String pattern = ((RLike) e).pattern();
+                String pattern = ((RLike) e).pattern().asJavaRegex();
                 q = new RegexQuery(e.source(), targetFieldName, pattern);
             }
 
@@ -351,4 +351,4 @@ private static Query boolQuery(Source source, Query left, Query right, boolean i
         }
         return new BoolQuery(source, isAnd, left, right);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
index b853443a0ea7c..9121b42e7c9c8 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
@@ -48,13 +49,13 @@
 import java.util.List;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.TestUtils.of;
 import static org.elasticsearch.xpack.ql.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.ql.expression.Literal.NULL;
 import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.ql.TestUtils.of;
 
 public class OptimizerRulesTests extends ESTestCase {
 
@@ -189,7 +190,7 @@ public void testConstantFoldingLikes() {
                 new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0)))
                         .canonical());
         assertEquals(TRUE,
-                new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), "test.emp")).canonical());
+                new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
     }
 
     public void testArithmeticFolding() {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
index 0455039a34e25..bf8b22d8a39d5 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
@@ -235,7 +236,7 @@ public Expression visitPredicated(PredicatedContext ctx) {
                 e = new Like(source, exp, visitPattern(pCtx.pattern()));
                 break;
             case SqlBaseParser.RLIKE:
-                e = new RLike(source, exp, string(pCtx.regex));
+                e = new RLike(source, exp, new RLikePattern(string(pCtx.regex)));
                 break;
             case SqlBaseParser.NULL:
                 // shortcut to avoid double negation later on (since there's no IsNull (missing in ES is a negated exists))
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
index d4ef29f667ee0..922b32d199f7f 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
@@ -369,7 +370,7 @@ public void testGenericNullableExpression() {
         // comparison
         assertNullLiteral(rule.rule(new GreaterThan(EMPTY, getFieldAttribute(), NULL)));
         // regex
-        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, "123")));
+        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123"))));
     }
 
     public void testNullFoldingOnCast() {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
index a70a0d610c36b..2225ead0dfd31 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
@@ -547,6 +547,23 @@ private void assertDifferentRLikeAndNotRLikePatterns(String firstPattern, String
         assertEquals("keyword", rqsq.field());
     }
 
+    public void testLikeRLikeAsPainlessScripts() {
+        LogicalPlan p = plan("SELECT count(*), CASE WHEN keyword LIKE '%foo%' THEN 1 WHEN keyword RLIKE '.*bar.*' THEN 2 " +
+                             "ELSE 3 END AS t FROM test GROUP BY t");
+        assertTrue(p instanceof Aggregate);
+        Expression condition = ((Aggregate) p).groupings().get(0);
+        assertFalse(condition.foldable());
+        GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
+        assertNotNull(groupingContext);
+        ScriptTemplate scriptTemplate = groupingContext.tail.script();
+        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
+                     "doc,params.v0),params.v1),params.v2,InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
+                     "doc,params.v3),params.v4),params.v5,params.v6])",
+                scriptTemplate.toString());
+        assertEquals("[{v=keyword}, {v=^.*foo.*$}, {v=1}, {v=keyword}, {v=.*bar.*}, {v=2}, {v=3}]",
+                scriptTemplate.params().toString());
+    }
+
     public void testTranslateNotExpression_WhereClause_Painless() {
         LogicalPlan p = plan("SELECT * FROM test WHERE NOT(POSITION('x', keyword) = 0)");
         assertTrue(p instanceof Project);

From 7e0a3c4007376fec594436c703b35ef56e728da7 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 16 Mar 2020 13:13:31 +0200
Subject: [PATCH 047/285] [ML] Extend classification to support multiple
 classes (#53539)

* [ML] Extend classification to support multiple classes

Prepares classification analysis to support more than just
two classes. It introduces a new parameter to the process config
which dictates the `num_classes` to the process. It also
changes the max classes limit to `30` provisionally.

* We can't test cardinality is too high in the YML tests anymore

* Extract max number of classes in a constant
---
 .../ml/dataframe/analyses/Classification.java |  14 +-
 .../dataframe/analyses/DataFrameAnalysis.java |  28 ++-
 .../dataframe/analyses/OutlierDetection.java  |   2 +-
 .../ml/dataframe/analyses/Regression.java     |   2 +-
 .../analyses/ClassificationTests.java         |  47 ++++-
 .../ml/integration/ClassificationIT.java      |   2 +
 .../scroll/TimeBasedExtractedFields.java      |   3 +-
 .../extractor/ExtractedFieldsDetector.java    |  17 +-
 .../process/AnalyticsProcessConfig.java       |  30 +++-
 .../xpack/ml/extractor/ExtractedFields.java   |  14 +-
 .../DataFrameDataExtractorTests.java          |   6 +-
 .../ExtractedFieldsDetectorTests.java         |   4 +-
 .../process/AnalyticsProcessConfigTests.java  | 170 ++++++++++++++++++
 .../ml/extractor/ExtractedFieldsTests.java    |   8 +-
 .../test/ml/start_data_frame_analytics.yml    |  21 +--
 15 files changed, 305 insertions(+), 63 deletions(-)
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index 0f365ad671b76..c0965f8d7ebb5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -46,9 +46,16 @@ public class Classification implements DataFrameAnalysis {
 
     private static final String STATE_DOC_ID_SUFFIX = "_classification_state#1";
 
+    private static final String NUM_CLASSES = "num_classes";
+
     private static final ConstructingObjectParser<Classification, Void> LENIENT_PARSER = createParser(true);
     private static final ConstructingObjectParser<Classification, Void> STRICT_PARSER = createParser(false);
 
+    /**
+     * The max number of classes classification supports
+     */
+    private static final int MAX_DEPENDENT_VARIABLE_CARDINALITY = 30;
+
     private static ConstructingObjectParser<Classification, Void> createParser(boolean lenient) {
         ConstructingObjectParser<Classification, Void> parser = new ConstructingObjectParser<>(
             NAME.getPreferredName(),
@@ -218,7 +225,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
+    public Map<String, Object> getParams(FieldInfo fieldInfo) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
@@ -227,10 +234,11 @@ public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
         if (predictionFieldName != null) {
             params.put(PREDICTION_FIELD_NAME.getPreferredName(), predictionFieldName);
         }
-        String predictionFieldType = getPredictionFieldType(extractedFields.get(dependentVariable));
+        String predictionFieldType = getPredictionFieldType(fieldInfo.getTypes(dependentVariable));
         if (predictionFieldType != null) {
             params.put(PREDICTION_FIELD_TYPE, predictionFieldType);
         }
+        params.put(NUM_CLASSES, fieldInfo.getCardinality(dependentVariable));
         return params;
     }
 
@@ -272,7 +280,7 @@ public List<RequiredField> getRequiredFields() {
     @Override
     public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
         // This restriction is due to the fact that currently the C++ backend only supports binomial classification.
-        return Collections.singletonList(FieldCardinalityConstraint.between(dependentVariable, 2, 2));
+        return Collections.singletonList(FieldCardinalityConstraint.between(dependentVariable, 2, MAX_DEPENDENT_VARIABLE_CARDINALITY));
     }
 
     @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
index 664b38e4fc05d..941224dc30a67 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.dataframe.analyses;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 
@@ -16,9 +17,9 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
 
     /**
      * @return The analysis parameters as a map
-     * @param extractedFields map of (name, types) for all the extracted fields
+     * @param fieldInfo Information about the fields like types and cardinalities
      */
-    Map<String, Object> getParams(Map<String, Set<String>> extractedFields);
+    Map<String, Object> getParams(FieldInfo fieldInfo);
 
     /**
      * @return {@code true} if this analysis supports fields with categorical values (i.e. text, keyword, ip)
@@ -64,4 +65,27 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
      * Returns the document id for the analysis state
      */
     String getStateDocId(String jobId);
+
+    /**
+     * Summarizes information about the fields that is necessary for analysis to generate
+     * the parameters needed for the process configuration.
+     */
+    interface FieldInfo {
+
+        /**
+         * Returns the types for the given field or {@code null} if the field is unknown
+         * @param field the field whose types to return
+         * @return the types for the given field or {@code null} if the field is unknown
+         */
+        @Nullable
+        Set<String> getTypes(String field);
+
+        /**
+         * Returns the cardinality of the given field or {@code null} if there is no cardinality for that field
+         * @param field the field whose cardinality to get
+         * @return the cardinality of the given field or {@code null} if there is no cardinality for that field
+         */
+        @Nullable
+        Long getCardinality(String field);
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
index 654d5ba4d1a29..2c83afa87808d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
@@ -192,7 +192,7 @@ public int hashCode() {
     }
 
     @Override
-    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
+    public Map<String, Object> getParams(FieldInfo fieldInfo) {
         Map<String, Object> params = new HashMap<>();
         if (nNeighbors != null) {
             params.put(N_NEIGHBORS.getPreferredName(), nNeighbors);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
index 86f8039090c74..d8c490ddcb804 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
@@ -155,7 +155,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     @Override
-    public Map<String, Object> getParams(Map<String, Set<String>> extractedFields) {
+    public Map<String, Object> getParams(FieldInfo fieldInfo) {
         Map<String, Object> params = new HashMap<>();
         params.put(DEPENDENT_VARIABLE.getPreferredName(), dependentVariable);
         params.putAll(boostedTreeParams.getParams());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index e7b2dbbc09f95..7429fe08d180d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -187,38 +187,46 @@ public void testGetTrainingPercent() {
     }
 
     public void testGetParams() {
-        Map<String, Set<String>> extractedFields =
+        DataFrameAnalysis.FieldInfo fieldInfo = new TestFieldInfo(
             Map.of(
                 "foo", Set.of(BooleanFieldMapper.CONTENT_TYPE),
                 "bar", Set.of(NumberFieldMapper.NumberType.LONG.typeName()),
-                "baz", Set.of(KeywordFieldMapper.CONTENT_TYPE));
+                "baz", Set.of(KeywordFieldMapper.CONTENT_TYPE)),
+            Map.of(
+                "foo", 10L,
+                "bar", 20L,
+                "baz", 30L)
+        );
         assertThat(
-            new Classification("foo").getParams(extractedFields),
+            new Classification("foo").getParams(fieldInfo),
             equalTo(
                 Map.of(
                     "dependent_variable", "foo",
                     "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "foo_prediction",
-                    "prediction_field_type", "bool")));
+                    "prediction_field_type", "bool",
+                    "num_classes", 10L)));
         assertThat(
-            new Classification("bar").getParams(extractedFields),
+            new Classification("bar").getParams(fieldInfo),
             equalTo(
                 Map.of(
                     "dependent_variable", "bar",
                     "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "bar_prediction",
-                    "prediction_field_type", "int")));
+                    "prediction_field_type", "int",
+                    "num_classes", 20L)));
         assertThat(
-            new Classification("baz").getParams(extractedFields),
+            new Classification("baz").getParams(fieldInfo),
             equalTo(
                 Map.of(
                     "dependent_variable", "baz",
                     "class_assignment_objective", Classification.ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL,
                     "num_top_classes", 2,
                     "prediction_field_name", "baz_prediction",
-                    "prediction_field_type", "string")));
+                    "prediction_field_type", "string",
+                    "num_classes", 30L)));
     }
 
     public void testRequiredFieldsIsNonEmpty() {
@@ -232,7 +240,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() {
         assertThat(constraints.size(), equalTo(1));
         assertThat(constraints.get(0).getField(), equalTo(classification.getDependentVariable()));
         assertThat(constraints.get(0).getLowerBound(), equalTo(2L));
-        assertThat(constraints.get(0).getUpperBound(), equalTo(2L));
+        assertThat(constraints.get(0).getUpperBound(), equalTo(30L));
     }
 
     public void testGetExplicitlyMappedFields() {
@@ -331,4 +339,25 @@ public void testExtractJobIdFromStateDoc() {
     protected Classification mutateInstanceForVersion(Classification instance, Version version) {
         return mutateForVersion(instance, version);
     }
+
+    private static class TestFieldInfo implements DataFrameAnalysis.FieldInfo {
+
+        private final Map<String, Set<String>> fieldTypes;
+        private final Map<String, Long> fieldCardinalities;
+
+        private TestFieldInfo(Map<String, Set<String>> fieldTypes, Map<String, Long> fieldCardinalities) {
+            this.fieldTypes = fieldTypes;
+            this.fieldCardinalities = fieldCardinalities;
+        }
+
+        @Override
+        public Set<String> getTypes(String field) {
+            return fieldTypes.get(field);
+        }
+
+        @Override
+        public Long getCardinality(String field) {
+            return fieldCardinalities.get(field);
+        }
+    }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index d2f90eed635ea..a58418bbd35ef 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -322,9 +322,11 @@ public void testStopAndRestart() throws Exception {
         assertEvaluation(KEYWORD_FIELD, KEYWORD_FIELD_VALUES, "ml." + predictedClassField);
     }
 
+    @AwaitsFix(bugUrl = "Muted until ml-cpp supports multiple classes")
     public void testDependentVariableCardinalityTooHighError() throws Exception {
         initialize("cardinality_too_high");
         indexData(sourceIndex, 6, 5, KEYWORD_FIELD);
+
         // Index one more document with a class different than the two already used.
         client().execute(IndexAction.INSTANCE, new IndexRequest(sourceIndex)
             .source(KEYWORD_FIELD, "fox")
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
index 8202c0ef3d28e..eb13f2395ef8d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java
@@ -14,6 +14,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -27,7 +28,7 @@ public class TimeBasedExtractedFields extends ExtractedFields {
     private final ExtractedField timeField;
 
     public TimeBasedExtractedFields(ExtractedField timeField, List<ExtractedField> allFields) {
-        super(allFields);
+        super(allFields, Collections.emptyMap());
         if (!allFields.contains(timeField)) {
             throw new IllegalArgumentException("timeField should also be contained in allFields");
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
index 0c885806d730c..2254a3de0a60c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
@@ -58,15 +58,15 @@ public class ExtractedFieldsDetector {
     private final DataFrameAnalyticsConfig config;
     private final int docValueFieldsLimit;
     private final FieldCapabilitiesResponse fieldCapabilitiesResponse;
-    private final Map<String, Long> fieldCardinalities;
+    private final Map<String, Long> cardinalitiesForFieldsWithConstraints;
 
     ExtractedFieldsDetector(String[] index, DataFrameAnalyticsConfig config, int docValueFieldsLimit,
-                            FieldCapabilitiesResponse fieldCapabilitiesResponse, Map<String, Long> fieldCardinalities) {
+                            FieldCapabilitiesResponse fieldCapabilitiesResponse, Map<String, Long> cardinalitiesForFieldsWithConstraints) {
         this.index = Objects.requireNonNull(index);
         this.config = Objects.requireNonNull(config);
         this.docValueFieldsLimit = docValueFieldsLimit;
         this.fieldCapabilitiesResponse = Objects.requireNonNull(fieldCapabilitiesResponse);
-        this.fieldCardinalities = Objects.requireNonNull(fieldCardinalities);
+        this.cardinalitiesForFieldsWithConstraints = Objects.requireNonNull(cardinalitiesForFieldsWithConstraints);
     }
 
     public Tuple<ExtractedFields, List<FieldSelection>> detect() {
@@ -286,12 +286,13 @@ private void checkRequiredFields(Set<String> fields) {
 
     private void checkFieldsWithCardinalityLimit() {
         for (FieldCardinalityConstraint constraint : config.getAnalysis().getFieldCardinalityConstraints()) {
-            constraint.check(fieldCardinalities.get(constraint.getField()));
+            constraint.check(cardinalitiesForFieldsWithConstraints.get(constraint.getField()));
         }
     }
 
     private ExtractedFields detectExtractedFields(Set<String> fields, Set<FieldSelection> fieldSelection) {
-        ExtractedFields extractedFields = ExtractedFields.build(fields, Collections.emptySet(), fieldCapabilitiesResponse);
+        ExtractedFields extractedFields = ExtractedFields.build(fields, Collections.emptySet(), fieldCapabilitiesResponse,
+            cardinalitiesForFieldsWithConstraints);
         boolean preferSource = extractedFields.getDocValueFields().size() > docValueFieldsLimit;
         extractedFields = deduplicateMultiFields(extractedFields, preferSource, fieldSelection);
         if (preferSource) {
@@ -321,7 +322,7 @@ private ExtractedFields deduplicateMultiFields(ExtractedFields extractedFields,
                     chooseMultiFieldOrParent(preferSource, requiredFields, parent, multiField, fieldSelection));
             }
         }
-        return new ExtractedFields(new ArrayList<>(nameOrParentToField.values()));
+        return new ExtractedFields(new ArrayList<>(nameOrParentToField.values()), cardinalitiesForFieldsWithConstraints);
     }
 
     private ExtractedField chooseMultiFieldOrParent(boolean preferSource, Set<String> requiredFields, ExtractedField parent,
@@ -372,7 +373,7 @@ private ExtractedFields fetchFromSourceIfSupported(ExtractedFields extractedFiel
         for (ExtractedField field : extractedFields.getAllFields()) {
             adjusted.add(field.supportsFromSource() ? field.newFromSource() : field);
         }
-        return new ExtractedFields(adjusted);
+        return new ExtractedFields(adjusted, cardinalitiesForFieldsWithConstraints);
     }
 
     private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFields) {
@@ -389,7 +390,7 @@ private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFi
                 adjusted.add(field);
             }
         }
-        return new ExtractedFields(adjusted);
+        return new ExtractedFields(adjusted, cardinalitiesForFieldsWithConstraints);
     }
 
     private void addIncludedFields(ExtractedFields extractedFields, Set<FieldSelection> fieldSelection) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
index 714f63091801f..0daec5365ff48 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
@@ -14,10 +14,9 @@
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
-import static java.util.stream.Collectors.toMap;
-
 public class AnalyticsProcessConfig implements ToXContentObject {
 
     private static final String JOB_ID = "job_id";
@@ -93,12 +92,31 @@ private DataFrameAnalysisWrapper(DataFrameAnalysis analysis, ExtractedFields ext
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("name", analysis.getWriteableName());
-            builder.field(
-                "parameters",
-                analysis.getParams(
-                    extractedFields.getAllFields().stream().collect(toMap(ExtractedField::getName, ExtractedField::getTypes))));
+            builder.field("parameters", analysis.getParams(new AnalysisFieldInfo(extractedFields)));
             builder.endObject();
             return builder;
         }
     }
+
+    private static class AnalysisFieldInfo implements DataFrameAnalysis.FieldInfo {
+
+        private final ExtractedFields extractedFields;
+
+        AnalysisFieldInfo(ExtractedFields extractedFields) {
+            this.extractedFields = Objects.requireNonNull(extractedFields);
+        }
+
+        @Override
+        public Set<String> getTypes(String field) {
+            Optional<ExtractedField> extractedField = extractedFields.getAllFields().stream()
+                .filter(f -> f.getName().equals(field))
+                .findAny();
+            return extractedField.isPresent() ? extractedField.get().getTypes() : null;
+        }
+
+        @Override
+        public Long getCardinality(String field) {
+            return extractedFields.getCardinalitiesForFieldsWithConstraints().get(field);
+        }
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
index 3a36bb7ff76d0..ab314a5d21851 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/ExtractedFields.java
@@ -28,12 +28,14 @@ public class ExtractedFields {
     private final List<ExtractedField> allFields;
     private final List<ExtractedField> docValueFields;
     private final String[] sourceFields;
+    private final Map<String, Long> cardinalitiesForFieldsWithConstraints;
 
-    public ExtractedFields(List<ExtractedField> allFields) {
+    public ExtractedFields(List<ExtractedField> allFields, Map<String, Long> cardinalitiesForFieldsWithConstraints) {
         this.allFields = Collections.unmodifiableList(allFields);
         this.docValueFields = filterFields(ExtractedField.Method.DOC_VALUE, allFields);
         this.sourceFields = filterFields(ExtractedField.Method.SOURCE, allFields).stream().map(ExtractedField::getSearchField)
             .toArray(String[]::new);
+        this.cardinalitiesForFieldsWithConstraints = Collections.unmodifiableMap(cardinalitiesForFieldsWithConstraints);
     }
 
     public List<ExtractedField> getAllFields() {
@@ -48,14 +50,20 @@ public List<ExtractedField> getDocValueFields() {
         return docValueFields;
     }
 
+    public Map<String, Long> getCardinalitiesForFieldsWithConstraints() {
+        return cardinalitiesForFieldsWithConstraints;
+    }
+
     private static List<ExtractedField> filterFields(ExtractedField.Method method, List<ExtractedField> fields) {
         return fields.stream().filter(field -> field.getMethod() == method).collect(Collectors.toList());
     }
 
     public static ExtractedFields build(Collection<String> allFields, Set<String> scriptFields,
-                                        FieldCapabilitiesResponse fieldsCapabilities) {
+                                        FieldCapabilitiesResponse fieldsCapabilities,
+                                        Map<String, Long> cardinalitiesForFieldsWithConstraints) {
         ExtractionMethodDetector extractionMethodDetector = new ExtractionMethodDetector(scriptFields, fieldsCapabilities);
-        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()));
+        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()),
+            cardinalitiesForFieldsWithConstraints);
     }
 
     public static TimeField newTimeField(String name, ExtractedField.Method method) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
index d54997f1dbdc3..b75392e03c2bb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
@@ -81,7 +81,7 @@ public void setUpTests() {
         query = QueryBuilders.matchAllQuery();
         extractedFields = new ExtractedFields(Arrays.asList(
             new DocValueField("field_1", Collections.singleton("keyword")),
-            new DocValueField("field_2", Collections.singleton("keyword"))));
+            new DocValueField("field_2", Collections.singleton("keyword"))), Collections.emptyMap());
         scrollSize = 1000;
         headers = Collections.emptyMap();
 
@@ -299,7 +299,7 @@ public void testIncludeSourceIsFalseAndAtLeastOneSourceField() throws IOExceptio
         // Explicit cast of ExtractedField args necessary for Eclipse due to https://bugs.eclipse.org/bugs/show_bug.cgi?id=530915
         extractedFields = new ExtractedFields(Arrays.asList(
             (ExtractedField) new DocValueField("field_1", Collections.singleton("keyword")),
-            (ExtractedField) new SourceField("field_2", Collections.singleton("text"))));
+            (ExtractedField) new SourceField("field_2", Collections.singleton("text"))), Collections.emptyMap());
 
         TestExtractor dataExtractor = createExtractor(false, false);
 
@@ -404,7 +404,7 @@ public void testGetCategoricalFields() {
             (ExtractedField) new DocValueField("field_integer", Collections.singleton("integer")),
             (ExtractedField) new DocValueField("field_long", Collections.singleton("long")),
             (ExtractedField) new DocValueField("field_keyword", Collections.singleton("keyword")),
-            (ExtractedField) new SourceField("field_text", Collections.singleton("text"))));
+            (ExtractedField) new SourceField("field_text", Collections.singleton("text"))), Collections.emptyMap());
         TestExtractor dataExtractor = createExtractor(true, true);
 
         assertThat(dataExtractor.getCategoricalFields(OutlierDetectionTests.createRandom()), empty());
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
index 49a302a498b82..a7a5784c452f5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
@@ -294,10 +294,10 @@ public void testDetect_GivenClassificationAndDependentVariableHasInvalidCardinal
             .build();
 
         ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(SOURCE_INDEX,
-            buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 3L));
+            buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 31L));
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect);
 
-        assertThat(e.getMessage(), equalTo("Field [some_keyword] must have at most [2] distinct values but there were at least [3]"));
+        assertThat(e.getMessage(), equalTo("Field [some_keyword] must have at most [30] distinct values but there were at least [31]"));
     }
 
     public void testDetect_GivenIgnoredField() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
new file mode 100644
index 0000000000000..a4db8de032af5
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfigTests.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
+import org.elasticsearch.xpack.ml.extractor.DocValueField;
+import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+
+public class AnalyticsProcessConfigTests extends ESTestCase {
+
+    private String jobId;
+    private long rows;
+    private int cols;
+    private ByteSizeValue memoryLimit;
+    private int threads;
+    private String resultsField;
+    private Set<String> categoricalFields;
+
+    @Before
+    public void setUpConfigParams() {
+        jobId = randomAlphaOfLength(10);
+        rows = randomNonNegativeLong();
+        cols = randomIntBetween(1, 42000);
+        memoryLimit = new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
+        threads = randomIntBetween(1, 8);
+        resultsField = randomAlphaOfLength(10);
+
+        int categoricalFieldsSize = randomIntBetween(0, 5);
+        categoricalFields = new HashSet<>();
+        for (int i = 0; i < categoricalFieldsSize; i++) {
+            categoricalFields.add(randomAlphaOfLength(10));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenOutlierDetection() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float"))), Collections.emptyMap());
+        DataFrameAnalysis analysis = new OutlierDetection.Builder().build();
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "outlier_detection"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenRegression() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float")),
+            new DocValueField("test_dep_var", Collections.singleton("keyword"))), Collections.emptyMap());
+        DataFrameAnalysis analysis = new Regression("test_dep_var");
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "regression"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
+        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenClassificationAndDepVarIsKeyword() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float")),
+            new DocValueField("test_dep_var", Collections.singleton("keyword"))), Collections.singletonMap("test_dep_var", 5L));
+        DataFrameAnalysis analysis = new Classification("test_dep_var");
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "classification"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
+        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
+        assertThat(paramsAsMap, hasEntry("prediction_field_type", "string"));
+        assertThat(paramsAsMap, hasEntry("num_classes", 5));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testToXContent_GivenClassificationAndDepVarIsInteger() throws IOException {
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+            new DocValueField("field_1", Collections.singleton("double")),
+            new DocValueField("field_2", Collections.singleton("float")),
+            new DocValueField("test_dep_var", Collections.singleton("integer"))), Collections.singletonMap("test_dep_var", 8L));
+        DataFrameAnalysis analysis = new Classification("test_dep_var");
+
+        AnalyticsProcessConfig processConfig = createProcessConfig(analysis, extractedFields);
+        Map<String, Object> asMap = toMap(processConfig);
+
+        assertRandomizedFields(asMap);
+
+        assertThat(asMap, hasKey("analysis"));
+        Map<String, Object> analysisAsMap = (Map<String, Object>) asMap.get("analysis");
+        assertThat(analysisAsMap, hasEntry("name", "classification"));
+        assertThat(analysisAsMap, hasKey("parameters"));
+        Map<String, Object> paramsAsMap = (Map<String, Object>) analysisAsMap.get("parameters");
+        assertThat(paramsAsMap, hasEntry("dependent_variable", "test_dep_var"));
+        assertThat(paramsAsMap, hasEntry("prediction_field_type", "int"));
+        assertThat(paramsAsMap, hasEntry("num_classes", 8));
+    }
+
+    private AnalyticsProcessConfig createProcessConfig(DataFrameAnalysis analysis, ExtractedFields extractedFields) {
+        return new AnalyticsProcessConfig(jobId, rows, cols, memoryLimit, threads, resultsField, categoricalFields, analysis,
+            extractedFields);
+    }
+
+    private static Map<String, Object> toMap(AnalyticsProcessConfig config) throws IOException {
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            config.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            return XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.toString(builder), false);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertRandomizedFields(Map<String, Object> configAsMap) {
+        assertThat(configAsMap, hasEntry("job_id", jobId));
+        assertThat(configAsMap, hasEntry("rows", rows));
+        assertThat(configAsMap, hasEntry("cols", cols));
+        assertThat(configAsMap, hasEntry("memory_limit", memoryLimit.getBytes()));
+        assertThat(configAsMap, hasEntry("threads", threads));
+        assertThat(configAsMap, hasEntry("results_field", resultsField));
+        assertThat(configAsMap, hasKey("categorical_fields"));
+        assertThat((List<String>) configAsMap.get("categorical_fields"), containsInAnyOrder(categoricalFields.toArray()));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
index 5ac983e7d505b..a51eafd1d8b3d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/extractor/ExtractedFieldsTests.java
@@ -32,7 +32,7 @@ public void testAllTypesOfFields() {
         ExtractedField sourceField1 = new SourceField("src1", Collections.singleton("text"));
         ExtractedField sourceField2 = new SourceField("src2", Collections.singleton("text"));
         ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
-                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2));
+                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2), Collections.emptyMap());
 
         assertThat(extractedFields.getAllFields().size(), equalTo(6));
         assertThat(extractedFields.getDocValueFields().stream().map(ExtractedField::getName).toArray(String[]::new),
@@ -54,7 +54,7 @@ public void testBuildGivenMixtureOfTypes() {
         when(fieldCapabilitiesResponse.getField("airline")).thenReturn(airlineCaps);
 
         ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("time", "value", "airline", "airport"),
-            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse);
+            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse, Collections.emptyMap());
 
         assertThat(extractedFields.getDocValueFields().size(), equalTo(2));
         assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("time"));
@@ -77,7 +77,7 @@ public void testBuildGivenMultiFields() {
         when(fieldCapabilitiesResponse.getField("airport.keyword")).thenReturn(keyword);
 
         ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("airline.text", "airport.keyword"),
-                Collections.emptySet(), fieldCapabilitiesResponse);
+                Collections.emptySet(), fieldCapabilitiesResponse, Collections.emptyMap());
 
         assertThat(extractedFields.getDocValueFields().size(), equalTo(1));
         assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("airport.keyword"));
@@ -119,7 +119,7 @@ public void testBuildGivenFieldWithoutMappings() {
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ExtractedFields.build(
-                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse));
+                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse, Collections.emptyMap()));
         assertThat(e.getMessage(), equalTo("cannot retrieve field [value] because it has no mappings"));
     }
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
index 81077b1e69a8c..ab6cfac9515e5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
@@ -142,7 +142,7 @@
         id: "start_given_empty_dest_index"
 
 ---
-"Test start classification analysis when the dependent variable cardinality is too low or too high":
+"Test start classification analysis when the dependent variable cardinality is too low":
   - do:
       indices.create:
         index: index-with-dep-var-with-too-high-card
@@ -179,22 +179,3 @@
       catch: /Field \[keyword_field\] must have at least \[2\] distinct values but there were \[1\]/
       ml.start_data_frame_analytics:
         id: "classification-cardinality-limits"
-
-  - do:
-      index:
-        index: index-with-dep-var-with-too-high-card
-        body: { numeric_field: 2.0, keyword_field: "class_b" }
-
-  - do:
-      index:
-        index: index-with-dep-var-with-too-high-card
-        body: { numeric_field: 3.0, keyword_field: "class_c" }
-
-  - do:
-      indices.refresh:
-        index: index-with-dep-var-with-too-high-card
-
-  - do:
-      catch: /Field \[keyword_field\] must have at most \[2\] distinct values but there were at least \[3\]/
-      ml.start_data_frame_analytics:
-        id: "classification-cardinality-limits"

From ec3481ed13254ecdec32acf7a0fafd536ec77aff Mon Sep 17 00:00:00 2001
From: Marios Trivyzas <matriv@gmail.com>
Date: Mon, 16 Mar 2020 13:49:48 +0100
Subject: [PATCH 048/285] SQL: Fix NPE for parameterized LIKE/RLIKE (#53573)

Fix NPE when `null` is passed as a parameter for a parameterized
pattern of LIKE/RLIKE. e.g.: `field LIKE ?` params=[null]`
Check for null pattern in LIKE/RLIKE as for RLIKE (RegexpQuery) we
get an IllegalArgumentExpression from Lucence but for LIKE
(WildcardQuery) we get an NPE.

Fixes: #53557
---
 .../xpack/sql/parser/ExpressionBuilder.java            |  3 +++
 .../xpack/sql/parser/ExpressionTests.java              | 10 ++++++++++
 2 files changed, 13 insertions(+)

diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
index bf8b22d8a39d5..671d90fbd54f6 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
@@ -264,6 +264,9 @@ public LikePattern visitPattern(PatternContext ctx) {
         }
 
         String pattern = string(ctx.value);
+        if (pattern == null) {
+            throw new ParsingException(source(ctx.value), "Pattern must not be [null]");
+        }
         int pos = pattern.indexOf('*');
         if (pos >= 0) {
             throw new ParsingException(source(ctx.value),
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
index ad2d3066e121f..6211f40e0f0e5 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
@@ -21,16 +21,19 @@
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
 
 import java.time.Duration;
 import java.time.Period;
 import java.time.temporal.TemporalAmount;
+import java.util.Collections;
 import java.util.Locale;
 
 import static java.lang.String.format;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.LONG;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -542,4 +545,11 @@ public void testCaseWithOperand() {
         assertEquals("WHEN 1 THEN 'one'", ifc.sourceText());
         assertEquals("many", c.elseResult().toString());
     }
+
+    public void testLikePatternWithNullParameterNotAllowed() {
+        ParsingException e = expectThrows(ParsingException.class,
+                () -> parser.createExpression("a LIKE ?",
+                    Collections.singletonList(new SqlTypedParamValue(KEYWORD.typeName(), null))));
+        assertEquals("line 1:9: Pattern must not be [null]", e.getMessage());
+    }
 }

From 4bfe288d2f20c97c2a195669cee12ca0c73423eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= <przemyslaw.witek@elastic.co>
Date: Mon, 16 Mar 2020 14:11:36 +0100
Subject: [PATCH 049/285] Make classification evaluation metrics work when
 there is field mapping type mismatch (#53458)

---
 .../evaluation/classification/Accuracy.java   |  11 +-
 .../classification/PainlessScripts.java       |  35 ++++
 .../evaluation/classification/Precision.java  |   9 +-
 .../evaluation/classification/Recall.java     |  11 +-
 .../classification/PainlessScriptsTests.java  |  19 ++
 .../ClassificationEvaluationIT.java           | 195 +++++++++++++++---
 6 files changed, 227 insertions(+), 53 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
index 7aa95e14b576e..440a46540cecb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java
@@ -28,11 +28,9 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -66,12 +64,6 @@ public class Accuracy implements EvaluationMetric {
 
     static final String OVERALL_ACCURACY_AGG_NAME = "classification_overall_accuracy";
 
-    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
-
-    private static Script buildScript(Object...args) {
-        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
-    }
-
     private static final ObjectParser<Accuracy, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Accuracy::new);
 
     public static Accuracy fromXContent(XContentParser parser) {
@@ -112,7 +104,8 @@ public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> a
         List<AggregationBuilder> aggs = new ArrayList<>();
         List<PipelineAggregationBuilder> pipelineAggs = new ArrayList<>();
         if (overallAccuracy.get() == null) {
-            aggs.add(AggregationBuilders.avg(OVERALL_ACCURACY_AGG_NAME).script(buildScript(actualField, predictedField)));
+            Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
+            aggs.add(AggregationBuilders.avg(OVERALL_ACCURACY_AGG_NAME).script(script));
         }
         if (result.get() == null) {
             Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> matrixAggs =
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
new file mode 100644
index 0000000000000..152bef11c09a2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScripts.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification;
+
+import org.elasticsearch.script.Script;
+
+import java.text.MessageFormat;
+import java.util.Locale;
+
+/**
+ * Painless scripts used by classification metrics in this package.
+ */
+final class PainlessScripts {
+
+    /**
+     * Template for the comparison script.
+     * It uses "String.valueOf" method in case the mapping types of the two fields are different.
+     */
+    private static final MessageFormat COMPARISON_SCRIPT_TEMPLATE =
+        new MessageFormat("String.valueOf(doc[''{0}''].value).equals(String.valueOf(doc[''{1}''].value))", Locale.ROOT);
+
+    /**
+     * Builds script that tests field values equality for the given actual and predicted field names.
+     *
+     * @param actualField name of the actual field
+     * @param predictedField name of the predicted field
+     * @return script that tests whether the values of actualField and predictedField are equal
+     */
+    static Script buildIsEqualScript(String actualField, String predictedField) {
+        return new Script(COMPARISON_SCRIPT_TEMPLATE.format(new Object[]{ actualField, predictedField }));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
index 899c25f1c64e0..c2fe3e4069fcc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java
@@ -34,11 +34,9 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -59,17 +57,12 @@ public class Precision implements EvaluationMetric {
 
     public static final ParseField NAME = new ParseField("precision");
 
-    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
     private static final String AGG_NAME_PREFIX = "classification_precision_";
     static final String ACTUAL_CLASSES_NAMES_AGG_NAME = AGG_NAME_PREFIX + "by_actual_class";
     static final String BY_PREDICTED_CLASS_AGG_NAME = AGG_NAME_PREFIX + "by_predicted_class";
     static final String PER_PREDICTED_CLASS_PRECISION_AGG_NAME = AGG_NAME_PREFIX + "per_predicted_class_precision";
     static final String AVG_PRECISION_AGG_NAME = AGG_NAME_PREFIX + "avg_precision";
 
-    private static Script buildScript(Object...args) {
-        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
-    }
-
     private static final ObjectParser<Precision, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Precision::new);
 
     public static Precision fromXContent(XContentParser parser) {
@@ -116,7 +109,7 @@ public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> a
                 topActualClassNames.get().stream()
                     .map(className -> new KeyedFilter(className, QueryBuilders.termQuery(predictedField, className)))
                     .toArray(KeyedFilter[]::new);
-            Script script = buildScript(actualField, predictedField);
+            Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
             return Tuple.tuple(
                 List.of(
                     AggregationBuilders.filters(BY_PREDICTED_CLASS_AGG_NAME, keyedFiltersPredicted)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
index b71cbecd54d4c..2fdd979fcb825 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
@@ -30,11 +31,9 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -54,16 +53,11 @@ public class Recall implements EvaluationMetric {
 
     public static final ParseField NAME = new ParseField("recall");
 
-    private static final String PAINLESS_TEMPLATE = "doc[''{0}''].value == doc[''{1}''].value";
     private static final String AGG_NAME_PREFIX = "classification_recall_";
     static final String BY_ACTUAL_CLASS_AGG_NAME = AGG_NAME_PREFIX + "by_actual_class";
     static final String PER_ACTUAL_CLASS_RECALL_AGG_NAME = AGG_NAME_PREFIX + "per_actual_class_recall";
     static final String AVG_RECALL_AGG_NAME = AGG_NAME_PREFIX + "avg_recall";
 
-    private static Script buildScript(Object...args) {
-        return new Script(new MessageFormat(PAINLESS_TEMPLATE, Locale.ROOT).format(args));
-    }
-
     private static final ObjectParser<Recall, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Recall::new);
 
     public static Recall fromXContent(XContentParser parser) {
@@ -98,11 +92,12 @@ public final Tuple<List<AggregationBuilder>, List<PipelineAggregationBuilder>> a
         if (result.get() != null) {
             return Tuple.tuple(List.of(), List.of());
         }
-        Script script = buildScript(actualField, predictedField);
+        Script script = PainlessScripts.buildIsEqualScript(actualField, predictedField);
         return Tuple.tuple(
             List.of(
                 AggregationBuilders.terms(BY_ACTUAL_CLASS_AGG_NAME)
                     .field(actualField)
+                    .order(List.of(BucketOrder.count(false), BucketOrder.key(true)))
                     .size(MAX_CLASSES_CARDINALITY)
                     .subAggregation(AggregationBuilders.avg(PER_ACTUAL_CLASS_RECALL_AGG_NAME).script(script))),
             List.of(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
new file mode 100644
index 0000000000000..5a397bae366d7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PainlessScriptsTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification;
+
+import org.elasticsearch.script.Script;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PainlessScriptsTests extends ESTestCase {
+
+    public void testBuildIsEqualScript() {
+        Script script = PainlessScripts.buildIsEqualScript("act", "pred");
+        assertThat(script.getIdOrCode(), equalTo("String.valueOf(doc['act'].value).equals(String.valueOf(doc['pred'].value))"));
+    }
+}
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
index bdcfac276eaaf..773f91b5027d2 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java
@@ -37,11 +37,13 @@ public class ClassificationEvaluationIT extends MlNativeDataFrameAnalyticsIntegT
 
     private static final String ANIMALS_DATA_INDEX = "test-evaluate-animals-index";
 
-    private static final String ANIMAL_NAME_FIELD = "animal_name";
+    private static final String ANIMAL_NAME_KEYWORD_FIELD = "animal_name_keyword";
     private static final String ANIMAL_NAME_PREDICTION_FIELD = "animal_name_prediction";
-    private static final String NO_LEGS_FIELD = "no_legs";
+    private static final String NO_LEGS_KEYWORD_FIELD = "no_legs_keyword";
+    private static final String NO_LEGS_INTEGER_FIELD = "no_legs_integer";
     private static final String NO_LEGS_PREDICTION_FIELD = "no_legs_prediction";
-    private static final String IS_PREDATOR_FIELD = "predator";
+    private static final String IS_PREDATOR_KEYWORD_FIELD = "predator_keyword";
+    private static final String IS_PREDATOR_BOOLEAN_FIELD = "predator_boolean";
     private static final String IS_PREDATOR_PREDICTION_FIELD = "predator_prediction";
 
     @Before
@@ -61,7 +63,7 @@ public void cleanup() {
 
     public void testEvaluate_DefaultMetrics() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, null));
+            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, null));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(
@@ -74,7 +76,7 @@ public void testEvaluate_AllMetrics() {
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
                 new Classification(
-                    ANIMAL_NAME_FIELD,
+                    ANIMAL_NAME_KEYWORD_FIELD,
                     ANIMAL_NAME_PREDICTION_FIELD,
                     List.of(new Accuracy(), new MulticlassConfusionMatrix(), new Precision(), new Recall())));
 
@@ -91,7 +93,7 @@ public void testEvaluate_AllMetrics() {
     public void testEvaluate_Accuracy_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Accuracy())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -110,10 +112,9 @@ public void testEvaluate_Accuracy_KeywordField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(5.0 / 75));
     }
 
-    public void testEvaluate_Accuracy_IntegerField() {
+    private void evaluateAccuracy_IntegerField(String actualField) {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
-            evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(NO_LEGS_FIELD, NO_LEGS_PREDICTION_FIELD, List.of(new Accuracy())));
+            evaluateDataFrame(ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -132,10 +133,18 @@ public void testEvaluate_Accuracy_IntegerField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(15.0 / 75));
     }
 
-    public void testEvaluate_Accuracy_BooleanField() {
+    public void testEvaluate_Accuracy_IntegerField() {
+        evaluateAccuracy_IntegerField(NO_LEGS_INTEGER_FIELD);
+    }
+
+    public void testEvaluate_Accuracy_IntegerField_MappingTypeMismatch() {
+        evaluateAccuracy_IntegerField(NO_LEGS_KEYWORD_FIELD);
+    }
+
+    private void evaluateAccuracy_BooleanField(String actualField) {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(IS_PREDATOR_FIELD, IS_PREDATOR_PREDICTION_FIELD, List.of(new Accuracy())));
+                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Accuracy())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -151,10 +160,18 @@ public void testEvaluate_Accuracy_BooleanField() {
         assertThat(accuracyResult.getOverallAccuracy(), equalTo(45.0 / 75));
     }
 
-    public void testEvaluate_Precision() {
+    public void testEvaluate_Accuracy_BooleanField() {
+        evaluateAccuracy_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
+    }
+
+    public void testEvaluate_Accuracy_BooleanField_MappingTypeMismatch() {
+        evaluateAccuracy_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
+    }
+
+    public void testEvaluate_Precision_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -173,20 +190,78 @@ public void testEvaluate_Precision() {
         assertThat(precisionResult.getAvgPrecision(), equalTo(5.0 / 75));
     }
 
+    private void evaluatePrecision_IntegerField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_PREDICTION_FIELD, List.of(new Precision())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName()));
+        assertThat(
+            precisionResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Precision.PerClassResult("1", 0.2),
+                    new Precision.PerClassResult("2", 0.2),
+                    new Precision.PerClassResult("3", 0.2),
+                    new Precision.PerClassResult("4", 0.2),
+                    new Precision.PerClassResult("5", 0.2))));
+        assertThat(precisionResult.getAvgPrecision(), equalTo(0.2));
+    }
+
+    public void testEvaluate_Precision_IntegerField() {
+        evaluatePrecision_IntegerField(NO_LEGS_INTEGER_FIELD);
+    }
+
+    public void testEvaluate_Precision_IntegerField_MappingTypeMismatch() {
+        evaluatePrecision_IntegerField(NO_LEGS_KEYWORD_FIELD);
+    }
+
+    private void evaluatePrecision_BooleanField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Precision())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName()));
+        assertThat(
+            precisionResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Precision.PerClassResult("false", 0.5),
+                    new Precision.PerClassResult("true", 9.0 / 13))));
+        assertThat(precisionResult.getAvgPrecision(), equalTo(31.0 / 52));
+    }
+
+    public void testEvaluate_Precision_BooleanField() {
+        evaluatePrecision_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
+    }
+
+    public void testEvaluate_Precision_BooleanField_MappingTypeMismatch() {
+        evaluatePrecision_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
+    }
+
     public void testEvaluate_Precision_CardinalityTooHigh() {
         indexDistinctAnimals(ANIMALS_DATA_INDEX, 1001);
         ElasticsearchStatusException e =
             expectThrows(
                 ElasticsearchStatusException.class,
                 () -> evaluateDataFrame(
-                    ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision()))));
-        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name] is too high"));
+                    ANIMALS_DATA_INDEX,
+                    new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Precision()))));
+        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name_keyword] is too high"));
     }
 
-    public void testEvaluate_Recall() {
+    public void testEvaluate_Recall_KeywordField() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
-                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall())));
+                ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -205,21 +280,79 @@ public void testEvaluate_Recall() {
         assertThat(recallResult.getAvgRecall(), equalTo(5.0 / 75));
     }
 
+    private void evaluateRecall_IntegerField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, NO_LEGS_INTEGER_FIELD, List.of(new Recall())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName()));
+        assertThat(
+            recallResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Recall.PerClassResult("1", 1.0),
+                    new Recall.PerClassResult("2", 1.0),
+                    new Recall.PerClassResult("3", 1.0),
+                    new Recall.PerClassResult("4", 1.0),
+                    new Recall.PerClassResult("5", 1.0))));
+        assertThat(recallResult.getAvgRecall(), equalTo(1.0));
+    }
+
+    public void testEvaluate_Recall_IntegerField() {
+        evaluateRecall_IntegerField(NO_LEGS_INTEGER_FIELD);
+    }
+
+    public void testEvaluate_Recall_IntegerField_MappingTypeMismatch() {
+        evaluateRecall_IntegerField(NO_LEGS_KEYWORD_FIELD);
+    }
+
+    private void evaluateRecall_BooleanField(String actualField) {
+        EvaluateDataFrameAction.Response evaluateDataFrameResponse =
+            evaluateDataFrame(
+                ANIMALS_DATA_INDEX, new Classification(actualField, IS_PREDATOR_PREDICTION_FIELD, List.of(new Recall())));
+
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
+        assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
+
+        Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(0);
+        assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName()));
+        assertThat(
+            recallResult.getClasses(),
+            equalTo(
+                List.of(
+                    new Recall.PerClassResult("true", 0.6),
+                    new Recall.PerClassResult("false", 0.6))));
+        assertThat(recallResult.getAvgRecall(), equalTo(0.6));
+    }
+
+    public void testEvaluate_Recall_BooleanField() {
+        evaluateRecall_BooleanField(IS_PREDATOR_BOOLEAN_FIELD);
+    }
+
+    public void testEvaluate_Recall_BooleanField_MappingTypeMismatch() {
+        evaluateRecall_BooleanField(IS_PREDATOR_KEYWORD_FIELD);
+    }
+
     public void testEvaluate_Recall_CardinalityTooHigh() {
         indexDistinctAnimals(ANIMALS_DATA_INDEX, 1001);
         ElasticsearchStatusException e =
             expectThrows(
                 ElasticsearchStatusException.class,
                 () -> evaluateDataFrame(
-                    ANIMALS_DATA_INDEX, new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall()))));
-        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name] is too high"));
+                    ANIMALS_DATA_INDEX,
+                    new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new Recall()))));
+        assertThat(e.getMessage(), containsString("Cardinality of field [animal_name_keyword] is too high"));
     }
 
     private void evaluateWithMulticlassConfusionMatrix() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
-                new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix())));
+                new Classification(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix())));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -299,7 +432,8 @@ public void testEvaluate_ConfusionMatrixMetricWithUserProvidedSize() {
         EvaluateDataFrameAction.Response evaluateDataFrameResponse =
             evaluateDataFrame(
                 ANIMALS_DATA_INDEX,
-                new Classification(ANIMAL_NAME_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix(3, null))));
+                new Classification(
+                    ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_FIELD, List.of(new MulticlassConfusionMatrix(3, null))));
 
         assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName()));
         assertThat(evaluateDataFrameResponse.getMetrics(), hasSize(1));
@@ -336,11 +470,13 @@ public void testEvaluate_ConfusionMatrixMetricWithUserProvidedSize() {
     private static void createAnimalsIndex(String indexName) {
         client().admin().indices().prepareCreate(indexName)
             .setMapping(
-                ANIMAL_NAME_FIELD, "type=keyword",
+                ANIMAL_NAME_KEYWORD_FIELD, "type=keyword",
                 ANIMAL_NAME_PREDICTION_FIELD, "type=keyword",
-                NO_LEGS_FIELD, "type=integer",
+                NO_LEGS_KEYWORD_FIELD, "type=keyword",
+                NO_LEGS_INTEGER_FIELD, "type=integer",
                 NO_LEGS_PREDICTION_FIELD, "type=integer",
-                IS_PREDATOR_FIELD, "type=boolean",
+                IS_PREDATOR_KEYWORD_FIELD, "type=keyword",
+                IS_PREDATOR_BOOLEAN_FIELD, "type=boolean",
                 IS_PREDATOR_PREDICTION_FIELD, "type=boolean")
             .get();
     }
@@ -355,11 +491,13 @@ private static void indexAnimalsData(String indexName) {
                     bulkRequestBuilder.add(
                         new IndexRequest(indexName)
                             .source(
-                                ANIMAL_NAME_FIELD, animalNames.get(i),
+                                ANIMAL_NAME_KEYWORD_FIELD, animalNames.get(i),
                                 ANIMAL_NAME_PREDICTION_FIELD, animalNames.get((i + j) % animalNames.size()),
-                                NO_LEGS_FIELD, i + 1,
+                                NO_LEGS_KEYWORD_FIELD, String.valueOf(i + 1),
+                                NO_LEGS_INTEGER_FIELD, i + 1,
                                 NO_LEGS_PREDICTION_FIELD, j + 1,
-                                IS_PREDATOR_FIELD, i % 2 == 0,
+                                IS_PREDATOR_KEYWORD_FIELD, String.valueOf(i % 2 == 0),
+                                IS_PREDATOR_BOOLEAN_FIELD, i % 2 == 0,
                                 IS_PREDATOR_PREDICTION_FIELD, (i + j) % 2 == 0));
                 }
             }
@@ -375,7 +513,8 @@ private static void indexDistinctAnimals(String indexName, int distinctAnimalCou
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int i = 0; i < distinctAnimalCount; i++) {
             bulkRequestBuilder.add(
-                new IndexRequest(indexName).source(ANIMAL_NAME_FIELD, "animal_" + i, ANIMAL_NAME_PREDICTION_FIELD, randomAlphaOfLength(5)));
+                new IndexRequest(indexName)
+                    .source(ANIMAL_NAME_KEYWORD_FIELD, "animal_" + i, ANIMAL_NAME_PREDICTION_FIELD, randomAlphaOfLength(5)));
         }
         BulkResponse bulkResponse = bulkRequestBuilder.get();
         if (bulkResponse.hasFailures()) {

From d6cd4ce7849ba215407c8c5fa815c9b373fb8480 Mon Sep 17 00:00:00 2001
From: Andrei Stefan <astefan@users.noreply.github.com>
Date: Mon, 16 Mar 2020 15:22:42 +0200
Subject: [PATCH 050/285] QL: constant_keyword support (#53241)

---
 .../sql/language/data-types.asciidoc          |  31 ++--
 .../extractor/AbstractFieldHitExtractor.java  |   2 +
 .../xpack/ql/index/IndexResolver.java         |  21 ++-
 .../xpack/ql/type/ConstantKeywordEsField.java |  22 +++
 .../xpack/ql/type/DataTypeConverter.java      |   8 +-
 .../xpack/ql/type/DataTypes.java              |  38 ++--
 .../xpack/ql/type/KeywordEsField.java         |   7 +-
 .../xpack/ql/type/TextEsField.java            |   3 +-
 .../elasticsearch/xpack/ql/type/Types.java    |   3 +
 .../xpack/ql/expression/LiteralTests.java     |   3 +-
 .../ql/type/DataTypeConversionTests.java      |   7 +
 .../xpack/ql/type/TypesTests.java             |  14 +-
 .../resources/mapping-constant-keyword.json   |   8 +
 .../test/resources/mapping-multi-field.json   |   4 +
 .../elasticsearch/xpack/sql/jdbc/EsType.java  |   1 +
 .../xpack/sql/jdbc/TypeConverter.java         |   8 +
 .../xpack/sql/jdbc/TypeUtils.java             |   3 +-
 .../xpack/sql/qa/FieldExtractorTestCase.java  |  34 ++++
 .../xpack/sql/qa/jdbc/DataLoader.java         |   9 +-
 .../sql/qa/src/main/resources/alias.csv-spec  |   6 +-
 .../qa/src/main/resources/command.csv-spec    |   6 +-
 .../main/resources/constant-keyword.csv-spec  | 163 ++++++++++++++++++
 .../single-node-only/command-sys.csv-spec     | 148 ++++++++--------
 .../xpack/sql/type/SqlDataTypeConverter.java  |   3 +-
 .../xpack/sql/type/SqlDataTypes.java          |  11 ++
 .../analysis/index/IndexResolverTests.java    |  51 +++++-
 .../logical/command/sys/SysTypesTests.java    |   6 +-
 .../sql/type/SqlDataTypeConverterTests.java   |   7 +
 28 files changed, 499 insertions(+), 128 deletions(-)
 create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
 create mode 100644 x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
 create mode 100644 x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec

diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc
index 811bb1ac6a47f..2c799dc7276eb 100644
--- a/docs/reference/sql/language/data-types.asciidoc
+++ b/docs/reference/sql/language/data-types.asciidoc
@@ -13,21 +13,22 @@ s|SQL precision
 
 4+h| Core types
 
-| <<null-value, `null`>>      | null          | NULL        | 0
-| <<boolean, `boolean`>>      | boolean       | BOOLEAN     | 1
-| <<number, `byte`>>          | byte          | TINYINT     | 3
-| <<number, `short`>>         | short         | SMALLINT    | 5
-| <<number, `integer`>>       | integer       | INTEGER     | 10
-| <<number, `long`>>          | long          | BIGINT      | 19
-| <<number, `double`>>        | double        | DOUBLE      | 15
-| <<number, `float`>>         | float         | REAL        | 7
-| <<number, `half_float`>>    | half_float    | FLOAT       | 3
-| <<number, `scaled_float`>>  | scaled_float  | DOUBLE      | 15
-| <<keyword, `keyword`>>      | keyword       | VARCHAR     | 32,766
-| <<text, `text`>>            | text          | VARCHAR     | 2,147,483,647
-| <<binary, `binary`>>        | binary        | VARBINARY   | 2,147,483,647
-| <<date, `date`>>            | datetime      | TIMESTAMP   | 29
-| <<ip, `ip`>>                | ip            | VARCHAR     | 39
+| <<null-value, `null`>>                   | null            | NULL        | 0
+| <<boolean, `boolean`>>                   | boolean         | BOOLEAN     | 1
+| <<number, `byte`>>                       | byte            | TINYINT     | 3
+| <<number, `short`>>                      | short           | SMALLINT    | 5
+| <<number, `integer`>>                    | integer         | INTEGER     | 10
+| <<number, `long`>>                       | long            | BIGINT      | 19
+| <<number, `double`>>                     | double          | DOUBLE      | 15
+| <<number, `float`>>                      | float           | REAL        | 7
+| <<number, `half_float`>>                 | half_float      | FLOAT       | 3
+| <<number, `scaled_float`>>               | scaled_float    | DOUBLE      | 15
+| <<keyword, `keyword`>>                   | keyword         | VARCHAR     | 32,766
+| <<constant-keyword, `constant_keyword`>> | constant_keyword| VARCHAR     | 32,766
+| <<text, `text`>>                         | text            | VARCHAR     | 2,147,483,647
+| <<binary, `binary`>>                     | binary          | VARBINARY   | 2,147,483,647
+| <<date, `date`>>                         | datetime        | TIMESTAMP   | 29
+| <<ip, `ip`>>                             | ip              | VARCHAR     | 39
 
 4+h| Complex types
 
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
index cbb50605f72e2..3439626887ac0 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
@@ -28,6 +28,7 @@
 import java.util.Objects;
 import java.util.StringJoiner;
 
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
@@ -213,6 +214,7 @@ protected Object unwrapMultiValue(Object values) {
     protected boolean isFromDocValuesOnly(DataType dataType) {
         return dataType == KEYWORD // because of ignore_above.
                     || dataType == DATETIME
+                    || dataType == CONSTANT_KEYWORD // because a non-existent value is considered the constant value itself
                     || dataType == SCALED_FLOAT; // because of scaling_factor
     }
     
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
index 5d003dd80bf08..2b8cddfcf8b15 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypeRegistry;
 import org.elasticsearch.xpack.ql.type.DateEsField;
@@ -60,6 +61,7 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.action.ActionListener.wrap;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
@@ -298,8 +300,13 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
             StringBuilder errorMessage = new StringBuilder();
 
             boolean hasUnmapped = types.containsKey(UNMAPPED);
+            // a keyword field and a constant_keyword field with the same name in two different indices are considered "compatible"
+            // since a common use case of constant_keyword field involves two indices with a field having the same name: one being
+            // a keyword, the other being a constant_keyword
+            boolean hasCompatibleKeywords = types.containsKey(KEYWORD.esType()) && types.containsKey(CONSTANT_KEYWORD.esType());
+            int allowedTypesCount = (hasUnmapped ? 2 : 1) + (hasCompatibleKeywords ? 1 : 0);
 
-            if (types.size() > (hasUnmapped ? 2 : 1)) {
+            if (types.size() > allowedTypesCount) {
                 // build the error message
                 // and create a MultiTypeField
 
@@ -344,6 +351,11 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
                 }
             }
 
+            // if there are both a keyword and a constant_keyword type for this field, only keep the keyword as a common compatible type
+            if (hasCompatibleKeywords) {
+                types.remove(CONSTANT_KEYWORD.esType());
+            }
+
             // everything checks
             return null;
         });
@@ -435,6 +447,9 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa
         if (esType == DATETIME) {
             return new DateEsField(fieldName, props, isAggregateable);
         }
+        if (esType == CONSTANT_KEYWORD) {
+            return new ConstantKeywordEsField(fieldName);
+        }
         if (esType == UNSUPPORTED) {
             return new UnsupportedEsField(fieldName, typeName, null, props);
         }
@@ -501,14 +516,14 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
 
         for (Entry<String, Map<String, FieldCapabilities>> entry : sortedFields) {
             String fieldName = entry.getKey();
-            Map<String, FieldCapabilities> types = entry.getValue();
 
             // ignore size added by the mapper plugin
             if (FIELD_NAMES_BLACKLIST.contains(fieldName)) {
                 continue;
             }
 
-            // apply verification
+            Map<String, FieldCapabilities> types = new LinkedHashMap<>(entry.getValue());
+            // apply verification and possibly remove the "duplicate" CONSTANT_KEYWORD field type
             final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types);
 
             // filter meta fields and unmapped
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
new file mode 100644
index 0000000000000..9983599dc4def
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ql.type;
+
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
+
+/**
+ * SQL-related information about an index field with a constant_keyword type
+ */
+public class ConstantKeywordEsField extends KeywordEsField {
+
+    public ConstantKeywordEsField(String name) {
+        super(name, CONSTANT_KEYWORD, Collections.emptyMap(), true, Short.MAX_VALUE, false, false);
+    }
+
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
index 5400659c75ab4..a325f9287355e 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
@@ -21,6 +21,7 @@
 
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -59,9 +60,12 @@ public static DataType commonType(DataType left, DataType right) {
             return left;
         }
         if (isString(left) && isString(right)) {
-            if (left == TEXT) {
+            if (left == TEXT || right == TEXT) {
                 return TEXT;
             }
+            if (left == KEYWORD) {
+                return KEYWORD;
+            }
             return right;
         }
         if (left.isNumeric() && right.isNumeric()) {
@@ -120,7 +124,7 @@ public static Converter converterFor(DataType from, DataType to) {
             return DefaultConverter.TO_NULL;
         }
         // proper converters
-        if (to == KEYWORD || to == TEXT) {
+        if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
             return conversionToString(from);
         }
         if (to == LONG) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
index 6253a5f89d7e2..bc04a3b0213a9 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
@@ -18,33 +18,34 @@
 public final class DataTypes {
 
     // @formatter:off
-    public static final DataType UNSUPPORTED  = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
+    public static final DataType UNSUPPORTED      = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
 
-    public static final DataType NULL         = new DataType("null",              0,                 false, false, false);
+    public static final DataType NULL             = new DataType("null",              0,                 false, false, false);
 
-    public static final DataType BOOLEAN      = new DataType("boolean",           1,                 false, false, false);
+    public static final DataType BOOLEAN          = new DataType("boolean",           1,                 false, false, false);
     // integer numeric
-    public static final DataType BYTE         = new DataType("byte",              Byte.BYTES,        true, false, true);
-    public static final DataType SHORT        = new DataType("short",             Short.BYTES,       true, false, true);
-    public static final DataType INTEGER      = new DataType("integer",           Integer.BYTES,     true, false, true);
-    public static final DataType LONG         = new DataType("long",              Long.BYTES,        true, false, true);
+    public static final DataType BYTE             = new DataType("byte",              Byte.BYTES,        true, false, true);
+    public static final DataType SHORT            = new DataType("short",             Short.BYTES,       true, false, true);
+    public static final DataType INTEGER          = new DataType("integer",           Integer.BYTES,     true, false, true);
+    public static final DataType LONG             = new DataType("long",              Long.BYTES,        true, false, true);
     // decimal numeric
-    public static final DataType DOUBLE       = new DataType("double",            Double.BYTES,      false, true, true);
-    public static final DataType FLOAT        = new DataType("float",             Float.BYTES,       false, true, true);
-    public static final DataType HALF_FLOAT   = new DataType("half_float",        Float.BYTES,       false, true, true);
-    public static final DataType SCALED_FLOAT = new DataType("scaled_float",      Long.BYTES,        false, true, true);
+    public static final DataType DOUBLE           = new DataType("double",            Double.BYTES,      false, true, true);
+    public static final DataType FLOAT            = new DataType("float",             Float.BYTES,       false, true, true);
+    public static final DataType HALF_FLOAT       = new DataType("half_float",        Float.BYTES,       false, true, true);
+    public static final DataType SCALED_FLOAT     = new DataType("scaled_float",      Long.BYTES,        false, true, true);
     // string
-    public static final DataType KEYWORD      = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
-    public static final DataType TEXT         = new DataType("text",              Integer.MAX_VALUE, false, false, false);
+    public static final DataType KEYWORD          = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
+    public static final DataType TEXT             = new DataType("text",              Integer.MAX_VALUE, false, false, false);
+    public static final DataType CONSTANT_KEYWORD = new DataType("constant_keyword",  Integer.MAX_VALUE, false, false, true);
     // date
-    public static final DataType DATETIME     = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
+    public static final DataType DATETIME         = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
     // ip
     public static final DataType IP           = new DataType("ip",                45,                false, false, true);
     // binary
-    public static final DataType BINARY       = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
+    public static final DataType BINARY           = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
     // complex types
-    public static final DataType OBJECT       = new DataType("object",            0,                 false, false, false);
-    public static final DataType NESTED       = new DataType("nested",            0,                 false, false, false);
+    public static final DataType OBJECT           = new DataType("object",            0,                 false, false, false);
+    public static final DataType NESTED           = new DataType("nested",            0,                 false, false, false);
     //@formatter:on
     
     private static final Collection<DataType> TYPES = Arrays.asList(
@@ -61,6 +62,7 @@ public final class DataTypes {
             SCALED_FLOAT,
             KEYWORD,
             TEXT,
+            CONSTANT_KEYWORD,
             DATETIME,
             IP,
             BINARY,
@@ -132,7 +134,7 @@ public static boolean isUnsupported(DataType from) {
     }
 
     public static boolean isString(DataType t) {
-        return t == KEYWORD || t == TEXT;
+        return t == KEYWORD || t == TEXT || t == CONSTANT_KEYWORD;
     }
 
     public static boolean isPrimitive(DataType t) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
index 78d84946da4c9..65c679a5ad034 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
@@ -29,7 +29,12 @@ public KeywordEsField(String name, Map<String, EsField> properties, boolean hasD
     
     public KeywordEsField(String name, Map<String, EsField> properties, boolean hasDocValues, int precision,
             boolean normalized, boolean isAlias) {
-        super(name, KEYWORD, properties, hasDocValues, isAlias);
+        this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias);
+    }
+
+    protected KeywordEsField(String name, DataType esDataType, Map<String, EsField> properties, boolean hasDocValues, int precision,
+            boolean normalized, boolean isAlias) {
+        super(name, esDataType, properties, hasDocValues, isAlias);
         this.precision = precision;
         this.normalized = normalized;
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
index e3e325be23f33..777fa9baea8c6 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
@@ -11,6 +11,7 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 
@@ -44,7 +45,7 @@ public Exact getExactInfo() {
     private Tuple<EsField, String> findExact() {
         EsField field = null;
         for (EsField property : getProperties().values()) {
-            if (property.getDataType() == KEYWORD && property.getExactInfo().hasExact()) {
+            if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD) && property.getExactInfo().hasExact()) {
                 if (field != null) {
                     return new Tuple<>(null, "Multiple exact keyword candidates available for [" + getName() +
                         "]; specify which one to use");
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
index 44202b006fcf6..136d6e8155451 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
@@ -14,6 +14,7 @@
 import java.util.Map.Entry;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
@@ -89,6 +90,8 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
                 int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
                 boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
                 field = new KeywordEsField(name, properties, docValues, length, normalized);
+            } else if (esDataType == CONSTANT_KEYWORD) {
+                field = new ConstantKeywordEsField(name);
             } else if (esDataType == DATETIME) {
                 field = new DateEsField(name, properties, docValues);
             } else if (esDataType == UNSUPPORTED) {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
index abc9fee4d97db..b40edfc381fc5 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
@@ -20,6 +20,7 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
@@ -53,7 +54,7 @@ static class ValueAndCompatibleTypes {
             new ValueAndCompatibleTypes(ESTestCase::randomLong, LONG, FLOAT, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomFloat, FLOAT, LONG, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomDouble, DOUBLE, LONG, FLOAT, BOOLEAN),
-            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD));
+            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD, CONSTANT_KEYWORD));
 
     public static Literal randomLiteral() {
         ValueAndCompatibleTypes gen = randomFrom(GENERATORS);
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
index f1188ed546e5a..0010e99fdf510 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
@@ -17,6 +17,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -361,6 +362,7 @@ public void testCommonType() {
         assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
+        assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -370,6 +372,11 @@ public void testCommonType() {
         // strings
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
+        assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
+        assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
+        assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
+        assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
+        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
     }
 
     public void testEsDataTypes() {
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
index 9958d00b02929..7adbf9306d7b8 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
@@ -14,6 +14,7 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
@@ -135,9 +136,10 @@ public void testMultiField() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field.getDataType(), is(TEXT));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(2));
+        assertThat(fields.size(), is(3));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
+        assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
     }
 
     public void testMultiFieldTooManyOptions() {
@@ -148,9 +150,10 @@ public void testMultiFieldTooManyOptions() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field, instanceOf(TextEsField.class));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(2));
+        assertThat(fields.size(), is(3));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
+        assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
     }
 
     public void testNestedDoc() {
@@ -173,6 +176,13 @@ public void testIpField() {
         assertThat(dt.getDataType().typeName(), is("ip"));
     }
 
+    public void testConstantKeywordField() {
+        Map<String, EsField> mapping = loadMapping("mapping-constant-keyword.json");
+        assertThat(mapping.size(), is(1));
+        EsField dt = mapping.get("full_name");
+        assertThat(dt.getDataType().typeName(), is("constant_keyword"));
+    }
+
     public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json b/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
new file mode 100644
index 0000000000000..7f248dc26fba6
--- /dev/null
+++ b/x-pack/plugin/ql/src/test/resources/mapping-constant-keyword.json
@@ -0,0 +1,8 @@
+{
+    "properties" : {
+        "full_name" : {
+            "type" : "constant_keyword",
+            "value" : "foo"
+        }
+    }
+}
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
index 9e293d4271333..8adc4c64e8476 100644
--- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
+++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
@@ -9,6 +9,10 @@
                 "english" : {
                     "type" : "text",
                     "analyzer" : "english"
+                },
+                "constant" : {
+                    "type" : "constant_keyword",
+                    "value" : "some constant value"
                 }
             }
         }
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
index 0b96204fd5d7e..9eae74d18019a 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java
@@ -25,6 +25,7 @@ public enum EsType implements SQLType {
     SCALED_FLOAT(             Types.FLOAT),
     KEYWORD(                  Types.VARCHAR),
     TEXT(                     Types.VARCHAR),
+    CONSTANT_KEYWORD(         Types.VARCHAR),
     OBJECT(                   Types.STRUCT),
     NESTED(                   Types.STRUCT),
     BINARY(                   Types.VARBINARY),
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
index bf72b4454e40e..888fb8632d217 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
@@ -210,6 +210,7 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
             case BOOLEAN:
             case TEXT:
             case KEYWORD:
+            case CONSTANT_KEYWORD:
                 return v; // These types are already represented correctly in JSON
             case BYTE:
                 return ((Number) v).byteValue(); // Parser might return it as integer or long - need to update to the correct type
@@ -323,6 +324,7 @@ private static Boolean asBoolean(Object val, EsType columnType, String typeStrin
                 return Boolean.valueOf(Integer.signum(((Number) val).intValue()) != 0);
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 return Boolean.valueOf((String) val);
             default:
                 return failConversion(val, columnType, typeString, Boolean.class);
@@ -345,6 +347,7 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr
                 return safeToByte(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Byte.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -372,6 +375,7 @@ private static Short asShort(Object val, EsType columnType, String typeString) t
                 return safeToShort(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Short.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -398,6 +402,7 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin
                 return safeToInt(safeToLong(((Number) val).doubleValue()));
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Integer.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -429,6 +434,7 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr
             //    return ((Number) val).longValue();
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Long.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -456,6 +462,7 @@ private static Float asFloat(Object val, EsType columnType, String typeString) t
                 return Float.valueOf(((Number) val).floatValue());
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Float.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -482,6 +489,7 @@ private static Double asDouble(Object val, EsType columnType, String typeString)
                 return Double.valueOf(((Number) val).doubleValue());
             case KEYWORD:
             case TEXT:
+            case CONSTANT_KEYWORD:
                 try {
                     return Double.valueOf((String) val);
                 } catch (NumberFormatException e) {
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
index 511cf92be5665..0fc929c900bfe 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java
@@ -76,6 +76,7 @@ private TypeUtils() {}
         types.put(EsType.SCALED_FLOAT, Double.class);
         types.put(EsType.KEYWORD, String.class);
         types.put(EsType.TEXT, String.class);
+        types.put(EsType.CONSTANT_KEYWORD, String.class);
         types.put(EsType.BINARY, byte[].class);
         types.put(EsType.DATETIME, Timestamp.class);
         types.put(EsType.IP, String.class);
@@ -157,7 +158,7 @@ static EsType of(String name) throws SQLException {
     }
 
     static boolean isString(EsType dataType) {
-        return dataType == EsType.KEYWORD || dataType == EsType.TEXT;
+        return dataType == EsType.KEYWORD || dataType == EsType.TEXT || dataType == EsType.CONSTANT_KEYWORD;
     }
 
     static EsType of(Class<? extends Object> clazz) throws SQLException {
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
index 4a760b97cd23c..150db29a2f58e 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
@@ -103,6 +103,40 @@ public void testKeywordField() throws IOException {
         assertResponse(expected, runSql("SELECT keyword_field FROM test"));
     }
 
+    /*
+     *    "constant_keyword_field": {
+     *       "type": "constant_keyword",
+     *       "value": "foo"
+     *    }
+     */
+    public void testConstantKeywordField() throws IOException {
+        String value = randomAlphaOfLength(20);
+        // _source for `constant_keyword` fields doesn't matter, as they should be taken from docvalue_fields
+        boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting
+        boolean enableSource = randomBoolean();          // enable _source at index level
+        
+        Map<String, Object> indexProps = new HashMap<>(1);
+        indexProps.put("_source", enableSource);
+        
+        Map<String, Map<String, Object>> fieldProps = null;
+        if (randomBoolean()) {
+            fieldProps = new HashMap<>(1);
+            Map<String, Object> fieldProp = new HashMap<>(1);
+            fieldProp.put("value", value);
+            fieldProps.put("constant_keyword_field", fieldProp);
+        }
+        
+        createIndexWithFieldTypeAndProperties("constant_keyword", fieldProps, explicitSourceSetting ? indexProps : null);
+        index("{\"constant_keyword_field\":\"" + value + "\"}");
+        
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("columns", Arrays.asList(
+                columnInfo("plain", "constant_keyword_field", "constant_keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)
+        ));
+        expected.put("rows", singletonList(singletonList(value)));
+        assertResponse(expected, runSql("SELECT constant_keyword_field FROM test"));
+    }
+
     /*
      *    "long/integer/short/byte_field": {
      *       "type": "long/integer/short/byte"
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
index d2dffe21e1013..e362349f332d3 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
@@ -123,13 +123,11 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                 createString("first_name", createIndex);
                 createString("last_name", createIndex);
                 createIndex.startObject("gender").field("type", "keyword");
-                if (extraFields) {
-                    createIndex.field("copy_to", "extra_gender");
-                }
                 createIndex.endObject();
 
                 if (extraFields) {
-                    createIndex.startObject("extra_gender").field("type", "keyword").endObject();
+                    createIndex.startObject("extra_gender").field("type", "constant_keyword").endObject();
+                    createIndex.startObject("null_constant").field("type", "constant_keyword").endObject();
                     createIndex.startObject("extra.info.gender")
                         .field("type", "alias")
                         .field("path", "gender")
@@ -198,6 +196,9 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                     }
                     hadLastItem = true;
                     bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"');
+                    if (titles.get(f).equals("gender") && extraFields) {
+                        bulk.append(",\"extra_gender\":\"Female\"");
+                    }
                 }
             }
             // append department
diff --git a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
index 6c51bbf3dc69b..440434796a163 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
@@ -41,7 +41,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -50,6 +50,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
@@ -69,7 +70,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -78,6 +79,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
index b17ba988472be..c8ea838745c46 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
@@ -276,7 +276,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -285,6 +285,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
@@ -304,7 +305,7 @@ emp_no              |INTEGER        |integer
 extra               |STRUCT         |object
 extra.info          |STRUCT         |object
 extra.info.gender   |VARCHAR        |keyword
-extra_gender        |VARCHAR        |keyword
+extra_gender        |VARCHAR        |constant_keyword
 extra_no            |INTEGER        |integer
 first_name          |VARCHAR        |text
 first_name.keyword  |VARCHAR        |keyword
@@ -313,6 +314,7 @@ hire_date           |TIMESTAMP      |datetime
 languages           |TINYINT        |byte
 last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
+null_constant       |VARCHAR        |constant_keyword
 salary              |INTEGER        |integer
 ;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
new file mode 100644
index 0000000000000..42257a8890ebd
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/resources/constant-keyword.csv-spec
@@ -0,0 +1,163 @@
+// To mute tests follow example in file: example.csv-spec
+
+//
+// Tests testing constant_keyword (introduced in ES 7.7)
+//
+
+// filtering
+
+filterEquals
+SELECT extra_gender g FROM "test_emp_copy" WHERE g = 'Female' LIMIT 5;
+
+    g     
+---------------
+Female              
+Female              
+Female             
+Female         
+Female
+;
+
+filterNotEquals
+SELECT COUNT(*) AS c FROM "test_emp_copy" WHERE extra_gender <> 'Female';
+
+    c:l     
+---------------
+0
+;
+
+aggWithNullFilter
+SELECT COUNT(*) count FROM test_emp_copy WHERE extra_gender IS NOT NULL;
+
+     count:l     
+---------------
+100
+;
+
+// Awaits for https://github.com/elastic/elasticsearch/issues/53545
+aggWithNullFilter-Ignore
+SELECT COUNT(*) count FROM test_emp_copy WHERE null_constant IS NULL;
+
+     count:l     
+---------------
+100
+;
+
+functionOverAlias
+SELECT BIT_LENGTH(extra_gender) bit FROM test_emp_copy ORDER BY extra_gender LIMIT 1;
+
+      bit      
+---------------
+48
+;
+
+
+singlePercentileWithoutComma
+SELECT extra_gender AS gender, PERCENTILE(emp_no, 97) p1 FROM test_emp_copy GROUP BY extra_gender;
+
+gender:s             | p1:d
+Female               |10097.5        
+;
+
+percentileRank
+SELECT extra_gender AS gender, PERCENTILE_RANK(emp_no, 10025) rank FROM test_emp_copy GROUP BY extra_gender;
+
+gender:s       | rank:d
+---------------+---------------
+Female         |24.5           
+;
+
+aggSumWithAliasWithColumnRepeatedWithOrderDesc
+SELECT extra_gender AS g, extra_gender, SUM(salary) AS s3, SUM(salary), SUM(salary) AS s5 FROM test_emp_copy GROUP BY extra_gender;
+
+       g:s     | extra_gender:s |      s3:i     | SUM(salary):i |      s5:i       
+---------------+----------------+---------------+---------------+---------------
+Female         |Female          |4824855        |4824855        |4824855      
+;
+
+topHitsAsMinAndMaxAndGroupBy
+schema::extra_gender:s|min:s|max:s|first:s|last:s
+SELECT extra_gender, MIN(first_name) as min, MAX(first_name) as max, FIRST(first_name) as first, LAST(first_name) as last FROM test_emp_copy GROUP BY extra_gender ORDER BY extra_gender;
+
+ extra_gender  |      min      |      max      |     first     |     last      
+---------------+---------------+---------------+---------------+---------------
+Female         |Alejandro      |Zvonko         |Alejandro      |Zvonko         
+;
+
+medianAbsoluteDeviationOnTwoFields
+schema::extra_gender:s|avg:l|mad_s:l|mad_l:d
+SELECT extra_gender, FLOOR(AVG(salary)) AS avg, FLOOR(MAD(salary)) AS mad_s, MAD(languages) AS mad_l FROM test_emp_copy GROUP BY extra_gender ORDER BY extra_gender;
+
+ extra_gender  |      avg    |     mad_s   |     mad_l     
+---------------+-------------+-------------+---------------
+Female         |48248        |10096        |1            
+;
+
+caseGroupByAndHaving
+schema::count:l|extra_gender:s|languages:byte
+SELECT count(*) AS count, extra_gender, languages FROM test_emp_copy
+GROUP BY 2, 3 HAVING CASE WHEN count(*) > 10 THEN 'many' ELSE 'a few' END = 'many'
+ORDER BY 1, 3;
+
+     count     | extra_gender  |   languages   
+---------------+---------------+---------------
+15             |Female         |1              
+17             |Female         |3              
+18             |Female         |4              
+19             |Female         |2              
+21             |Female         |5              
+;
+
+whereFieldWithRLikeAndGroupByOrderBy
+SELECT last_name l, extra_gender g, COUNT(*) c, MAX(salary) AS sal FROM test_emp_copy WHERE emp_no < 10050 AND (last_name RLIKE 'B.*' OR extra_gender = 'F') GROUP BY g, l ORDER BY sal;
+
+      l:s      |      g:s      |      c:l      |     sal:i      
+---------------+---------------+---------------+---------------
+Berztiss       |Female         |1              |28336        
+Brender        |Female         |1              |36051        
+Bridgland      |Female         |1              |48942        
+Bouloucos      |Female         |1              |58715        
+Bamford        |Female         |1              |61805        
+;
+
+multipleGroupingsAndOrderingByGroupsWithFunctions_2
+SELECT first_name f, last_name l, LCASE(extra_gender) g, CONCAT(UCASE(first_name), LCASE(last_name)) c FROM test_emp_copy GROUP BY f, LCASE(extra_gender), l, c ORDER BY c DESC, first_name, l ASC, g LIMIT 3;
+
+      f:s      |      l:s      |      g:s      |       c:s       
+---------------+---------------+---------------+---------------
+null           |Swan           |female         |swan           
+null           |Reistad        |female         |reistad        
+null           |Portugali      |female         |portugali      
+;
+
+isNotNullWithCount
+SELECT extra_gender IS NOT NULL AS bool, COUNT(*) FROM test_emp_copy GROUP BY extra_gender;
+
+          bool:b        |   COUNT(*):l    
+------------------------+---------------
+true                    |100            
+;
+
+likeWithCount1
+SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender LIKE 'F%';
+
+   COUNT(*):l    
+---------------
+100            
+;
+
+likeWithCount2
+SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender LIKE '%m%';
+
+   COUNT(*):l    
+---------------
+100            
+;
+
+likeWithCount3
+SELECT COUNT(*) FROM test_emp_copy WHERE extra_gender NOT LIKE '%m%';
+
+   COUNT(*):l    
+---------------
+0         
+;
\ No newline at end of file
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
index 711df01849103..dee74303b9bda 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
@@ -29,19 +29,20 @@ SYS COLUMNS TABLE LIKE 'test_emp';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest  |null           |test_emp            |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp            |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |birth_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |emp_no             |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra.info.gender  |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_gender       |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_no           |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |gender             |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |hire_date          |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |languages          |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name          |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name.keyword  |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
+integTest  |null           |test_emp            |null_constant      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |salary             |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsWithCatalogAndLike
@@ -49,19 +50,20 @@ SYS COLUMNS CATALOG 'integTest' TABLE LIKE 'test\_emp\_copy' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s  |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest  |null           |test_emp_copy|salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name        |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|gender            |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|languages         |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
+integTest  |null           |test_emp_copy|null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|salary            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsOnAliasWithTableLike
@@ -69,55 +71,57 @@ SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-integTest   |null           |test_alias    |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-integTest   |null           |test_alias    |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |birth_date        |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |emp_no            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_no          |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name        |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |gender            |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |hire_date         |93             |DATETIME        |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |languages         |-6             |BYTE            |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name         |12             |TEXT            |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO
+integTest   |null           |test_alias    |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |salary            |4              |INTEGER         |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsAllTables
 SYS COLUMNS TABLE LIKE '%';
 
-   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
----------------+---------------+---------------+------------------+---------------+---------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
-integTest      |null           |logs           |@timestamp        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_in          |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_out         |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_ip         |12             |IP             |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_port       |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |dest_ip           |12             |IP             |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |id                |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |status            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |birth_date        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |emp_no            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name        |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |gender            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |hire_date         |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |languages         |-6             |BYTE           |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name         |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |salary            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_gender      |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name        |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME       |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |languages         |-6             |BYTE           |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name         |12             |TEXT           |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD        |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |salary            |4              |INTEGER        |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |16                |YES          |null           |null           |null           |null            |NO                |NO
+   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s  |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
+---------------+---------------+---------------+------------------+---------------+----------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
+integTest      |null           |logs           |@timestamp        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |bytes_in          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |bytes_out         |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |client_ip         |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |client_port       |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |dest_ip           |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |id                |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |logs           |status            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp       |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |16                |YES          |null           |null           |null           |null            |NO                |NO
+integTest      |null           |test_emp_copy  |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |17                |YES          |null           |null           |null           |null            |NO                |NO
 ;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
index e958ce1aa830d..bb5156a47b35b 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java
@@ -36,6 +36,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.TO_NULL;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -159,7 +160,7 @@ public static Converter converterFor(DataType from, DataType to) {
         }
         // extend the default converter with DATE and TIME
         if (from == DATE || from == TIME) {
-            if (to == KEYWORD || to == TEXT) {
+            if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
                 return conversionToString(from);
             }
             if (to == LONG) {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
index 6c321bea80de8..616b2de40810a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
@@ -29,6 +29,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -268,6 +269,7 @@ public static boolean isFromDocValuesOnly(DataType dataType) {
                 || dataType == DATE         // because of date formats
                 || dataType == DATETIME
                 || dataType == SCALED_FLOAT // because of scaling_factor
+                || dataType == CONSTANT_KEYWORD
                 || dataType == GEO_POINT
                 || dataType == GEO_SHAPE
                 || dataType == SHAPE;
@@ -333,6 +335,9 @@ public static SQLType sqlType(DataType dataType) {
         if (dataType == TEXT) {
             return JDBCType.VARCHAR;
         }
+        if (dataType == CONSTANT_KEYWORD) {
+            return JDBCType.VARCHAR;
+        }
         if (dataType == DATETIME) {
             return JDBCType.TIMESTAMP;
         }
@@ -456,6 +461,9 @@ public static int defaultPrecision(DataType dataType) {
         if (dataType == TEXT) {
             return 32766;
         }
+        if (dataType == CONSTANT_KEYWORD) {
+            return 15;
+        }
         if (dataType == DATETIME) {
             return 3;
         }
@@ -576,6 +584,9 @@ public static int displaySize(DataType dataType) {
         if (dataType == TEXT) {
             return dataType.size();
         }
+        if (dataType == CONSTANT_KEYWORD) {
+            return 32766;
+        }
         if (dataType == DATETIME) {
             return 29;
         }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
index b53d1cbdf8133..614cc24ec44aa 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
@@ -10,10 +10,12 @@
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
+import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.ql.type.InvalidMappedField;
 import org.elasticsearch.xpack.ql.type.KeywordEsField;
+import org.elasticsearch.xpack.ql.type.TextEsField;
 import org.elasticsearch.xpack.sql.type.SqlDataTypeRegistry;
 
 import java.util.ArrayList;
@@ -26,6 +28,7 @@
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
@@ -268,6 +271,52 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
                 ((InvalidMappedField) esField).errorMessage());
     }
 
+    public void testConstantKeywordTwoIndicesScenario() throws Exception {
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "name", "text", true, false);
+        addFieldCaps(fieldCaps, "wheels_count", "constant_keyword", true, true);
+        addFieldCaps(fieldCaps, "motor", "keyword", true, true);
+
+        Map<String, FieldCapabilities> multi = new HashMap<>();
+        multi.put("keyword", new FieldCapabilities("cycle_type", "keyword", true, true, new String[] { "other_cycles" }, null, null,
+                Collections.emptyMap()));
+        multi.put("constant_keyword", new FieldCapabilities("cycle_type", "constant_keyword", true, false, new String[] { "bicycles" },
+                null, null, Collections.emptyMap()));
+        fieldCaps.put("cycle_type", multi);
+
+        String wildcard = "*";
+        IndexResolution resolution = mergedMappings(wildcard, new String[] { "other_cycles", "bicycles" }, fieldCaps);
+
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        EsField esField = null;
+        Map<String, EsField> props = esIndex.mapping();
+
+        assertEquals(4, props.size());
+        assertTrue(props.containsKey("cycle_type"));
+        assertTrue(props.containsKey("name"));
+        assertTrue(props.containsKey("wheels_count"));
+        assertTrue(props.containsKey("motor"));
+        
+        esField = props.get("cycle_type");
+        assertTrue(esField instanceof KeywordEsField);
+        assertEquals(KEYWORD, ((KeywordEsField) esField).getDataType());
+        
+        esField = props.get("name");
+        assertTrue(esField instanceof TextEsField);
+        assertEquals(TEXT, ((TextEsField) esField).getDataType());
+        
+        esField = props.get("wheels_count");
+        assertTrue(esField instanceof ConstantKeywordEsField);
+        assertEquals(CONSTANT_KEYWORD, ((ConstantKeywordEsField) esField).getDataType());
+        
+        esField = props.get("motor");
+        assertTrue(esField instanceof KeywordEsField);
+        assertEquals(KEYWORD, ((KeywordEsField) esField).getDataType());
+    }
+
     public void testSeparateSameMappingDifferentIndices() throws Exception {
         Map<String, EsField> oneMapping = loadMapping("mapping-basic.json", true);
         Map<String, EsField> sameMapping = loadMapping("mapping-basic.json", true);
@@ -383,7 +432,7 @@ private static boolean isSearchable(DataType type) {
     }
 
     private static boolean isAggregatable(DataType type) {
-        return type.isNumeric() || type == KEYWORD || type == DATETIME;
+        return type.isNumeric() || type == KEYWORD || type == DATETIME || type == CONSTANT_KEYWORD;
     }
 
     private static class UpdateableFieldCapabilities extends FieldCapabilities {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
index 17bc17bbec4b8..e0bf79b830e00 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
@@ -46,7 +46,7 @@ public void testSysTypes() {
         Command cmd = sql("SYS TYPES").v1();
 
         List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", "SCALED_FLOAT",
-                "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME",
+                "CONSTANT_KEYWORD", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME",
                 "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND",
                 "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND",
                 "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND",
@@ -107,7 +107,9 @@ public void testSysTypesMultipleMatches() {
 
         cmd.execute(session(), wrap(p -> {
             SchemaRowSet r = (SchemaRowSet) p.rowSet();
-            assertEquals(3, r.size());
+            assertEquals(4, r.size());
+            assertEquals("CONSTANT_KEYWORD", r.column(0));
+            assertTrue(r.advanceRow());
             assertEquals("IP", r.column(0));
             assertTrue(r.advanceRow());
             assertEquals("KEYWORD", r.column(0));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
index b39dd8bcaa897..c445a84b236ca 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
@@ -22,6 +22,7 @@
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
+import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -631,6 +632,7 @@ public void testCommonType() {
         assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
+        assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -640,6 +642,11 @@ public void testCommonType() {
         // strings
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
+        assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
+        assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
+        assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
+        assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
+        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
 
         // numeric and intervals
         assertEquals(INTERVAL_YEAR_TO_MONTH, commonType(INTERVAL_YEAR_TO_MONTH, LONG));

From f5f925c7bd78f2a1fec6518890fa785b7540c54f Mon Sep 17 00:00:00 2001
From: Tom Veasey <tveasey@users.noreply.github.com>
Date: Mon, 16 Mar 2020 14:13:29 +0000
Subject: [PATCH 051/285] Reenable tests and update version (#53556)

---
 .../xpack/core/ml/dataframe/analyses/Classification.java     | 4 ++--
 .../core/ml/dataframe/analyses/ClassificationTests.java      | 2 +-
 .../test/mixed_cluster/90_ml_data_frame_analytics_crud.yml   | 5 -----
 .../test/old_cluster/90_ml_data_frame_analytics_crud.yml     | 3 ---
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml     | 5 -----
 5 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index c0965f8d7ebb5..ef0b7e19d100e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -142,7 +142,7 @@ public Classification(StreamInput in) throws IOException {
         dependentVariable = in.readString();
         boostedTreeParams = new BoostedTreeParams(in);
         predictionFieldName = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
             classAssignmentObjective = in.readEnum(ClassAssignmentObjective.class);
         } else {
             classAssignmentObjective = ClassAssignmentObjective.MAXIMIZE_MINIMUM_RECALL;
@@ -194,7 +194,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(dependentVariable);
         boostedTreeParams.writeTo(out);
         out.writeOptionalString(predictionFieldName);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
             out.writeEnum(classAssignmentObjective);
         }
         out.writeOptionalVInt(numTopClasses);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 7429fe08d180d..96961706aa4ca 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -67,7 +67,7 @@ public static Classification mutateForVersion(Classification instance, Version v
         return new Classification(instance.getDependentVariable(),
             BoostedTreeParamsTests.mutateForVersion(instance.getBoostedTreeParams(), version),
             instance.getPredictionFieldName(),
-            version.onOrAfter(Version.V_8_0_0) ? instance.getClassAssignmentObjective() : null,
+            version.onOrAfter(Version.V_7_7_0) ? instance.getClassAssignmentObjective() : null,
             instance.getNumTopClasses(),
             instance.getTrainingPercent(),
             instance.getRandomizeSeed());
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index be388da19be07..5049f3360a42e 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
-
 ---
 "Get old outlier_detection job":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index 10ab98a738e1c..923a56395e8a4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,7 +1,4 @@
 setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
 
   - do:
       index:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index aa4ef717263f9..b197d0c5cf433 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/52763"
-
 ---
 "Get old cluster outlier_detection job":
 

From e8ed337b2ad9048c51310a02bead3fb4a2facdff Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Mon, 16 Mar 2020 11:21:20 -0400
Subject: [PATCH 052/285] [DOCS] Reformat `remove_duplicates` token filter
 (#53608)

Makes the following changes to the `remove_duplicates` token filter
docs:

* Rewrites description and adds Lucene link
* Adds detailed analyze example
* Adds custom analyzer example
---
 .../remove-duplicates-tokenfilter.asciidoc    | 150 +++++++++++++++++-
 1 file changed, 148 insertions(+), 2 deletions(-)

diff --git a/docs/reference/analysis/tokenfilters/remove-duplicates-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/remove-duplicates-tokenfilter.asciidoc
index e9dbf1ed15303..76f43bffc72de 100644
--- a/docs/reference/analysis/tokenfilters/remove-duplicates-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/remove-duplicates-tokenfilter.asciidoc
@@ -4,5 +4,151 @@
 <titleabbrev>Remove duplicates</titleabbrev>
 ++++
 
-A token filter of type `remove_duplicates` that drops identical tokens at the
-same position.
+Removes duplicate tokens in the same position.
+
+The `remove_duplicates` filter uses Lucene's
+{lucene-analysis-docs}/miscellaneous/RemoveDuplicatesTokenFilter.html[RemoveDuplicatesTokenFilter].
+
+[[analysis-remove-duplicates-tokenfilter-analyze-ex]]
+==== Example
+
+To see how the `remove_duplicates` filter works, you first need to produce a
+token stream containing duplicate tokens in the same position.
+
+The following <<indices-analyze,analyze API>> request uses the
+<<analysis-keyword-repeat-tokenfilter,`keyword_repeat`>> and
+<<analysis-stemmer-tokenfilter,`stemmer`>> filters to create stemmed and
+unstemmed tokens for `jumping dog`.
+
+[source,console]
+----
+GET _analyze
+{
+  "tokenizer": "whitespace",
+  "filter": [
+    "keyword_repeat",
+    "stemmer"
+  ],
+  "text": "jumping dog"
+}
+----
+
+The API returns the following response. Note that the `dog` token in position
+`1` is duplicated.
+
+[source,console-result]
+----
+{
+  "tokens": [
+    {
+      "token": "jumping",
+      "start_offset": 0,
+      "end_offset": 7,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "jump",
+      "start_offset": 0,
+      "end_offset": 7,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "dog",
+      "start_offset": 8,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "dog",
+      "start_offset": 8,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    }
+  ]
+}
+----
+
+To remove one of the duplicate `dog` tokens, add the `remove_duplicates` filter
+to the previous analyze API request.
+
+[source,console]
+----
+GET _analyze
+{
+  "tokenizer": "whitespace",
+  "filter": [
+    "keyword_repeat",
+    "stemmer",
+    "remove_duplicates"
+  ],
+  "text": "jumping dog"
+}
+----
+
+The API returns the following response. There is now only one `dog` token in
+position `1`.
+
+[source,console-result]
+----
+{
+  "tokens": [
+    {
+      "token": "jumping",
+      "start_offset": 0,
+      "end_offset": 7,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "jump",
+      "start_offset": 0,
+      "end_offset": 7,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "dog",
+      "start_offset": 8,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    }
+  ]
+}
+----
+
+[[analysis-remove-duplicates-tokenfilter-analyzer-ex]]
+==== Add to an analyzer
+
+The following <<indices-create-index,create index API>> request uses the
+`remove_duplicates` filter to configure a new <<analysis-custom-analyzer,custom
+analyzer>>.
+
+This custom analyzer uses the `keyword_repeat` and `stemmer` filters to create a
+stemmed and unstemmed version of each token in a stream. The `remove_duplicates`
+filter then removes any duplicate tokens in the same position.
+
+[source,console]
+----
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_custom_analyzer": {
+          "tokenizer": "standard",
+          "filter": [
+            "keyword_repeat",
+            "stemmer",
+            "remove_duplicates"
+          ]
+        }
+      }
+    }
+  }
+}
+----
\ No newline at end of file

From 887541e865f68fbbe13b27172b7fd230638aae93 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Mon, 16 Mar 2020 10:25:17 -0500
Subject: [PATCH 053/285] Add Watcher to available rest resources (#53319)

Prior to this commit Watcher explicitly copied test between two
projects with a copy task. This commit removes the explicit copy in favor
of adding the Watcher tests to the available restResources that may be
copied between projects.

This is how inter-project dependencies should be modeled. However, only
Watcher is included here since it is (currently) the only project with
inter-project test dependencies.
---
 .../gradle/test/rest/CopyRestTestsTask.java     |  4 ++--
 .../gradle/test/rest/RestResourcesPlugin.java   | 17 ++++++++++++-----
 x-pack/plugin/watcher/qa/rest/build.gradle      |  2 ++
 .../watcher/qa/with-security/build.gradle       | 12 +++---------
 4 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 9e9d1e0d9e795..4c86b86b56f5b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -114,7 +114,7 @@ void copy() {
             if (BuildParams.isInternal()) {
                 getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
                 project.copy(c -> {
-                    c.from(coreConfig.getSingleFile());
+                    c.from(coreConfig.getAsFileTree());
                     c.into(getOutputDir());
                     c.include(corePatternSet.getIncludes());
                 });
@@ -138,7 +138,7 @@ void copy() {
         if (includeXpack.get().isEmpty() == false) {
             getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
-                c.from(xpackConfig.getSingleFile());
+                c.from(xpackConfig.getAsFileTree());
                 c.into(getOutputDir());
                 c.include(xpackPatternSet.getIncludes());
             });
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
index a512b9b1fc025..57ee2c1f9037f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
@@ -86,21 +86,27 @@ public class RestResourcesPlugin implements Plugin<Project> {
     public void apply(Project project) {
         RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class);
 
+        // tests
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
-                task.coreConfig = project.getConfigurations().create("restTest");
+                task.coreConfig = project.getConfigurations().maybeCreate("restTests");
                 if (BuildParams.isInternal()) {
+                    // core
                     Dependency restTestdependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
                     project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
-
-                    task.xpackConfig = project.getConfigurations().create("restXpackTest");
+                    // x-pack
+                    task.xpackConfig = project.getConfigurations().maybeCreate("restXpackTests");
                     Dependency restXPackTestdependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency);
                     task.dependsOn(task.xpackConfig);
+                    // watcher
+                    Dependency restWatcherTests = project.getDependencies()
+                        .project(Map.of("path", ":x-pack:plugin:watcher:qa:rest", "configuration", "restXpackTests"));
+                    project.getDependencies().add(task.xpackConfig.getName(), restWatcherTests);
                 } else {
                     Dependency dependency = project.getDependencies()
                         .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
@@ -109,18 +115,19 @@ public void apply(Project project) {
                 task.dependsOn(task.coreConfig);
             });
 
+        // api
         Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.includeXpack.set(extension.restApi.getIncludeXpack());
                 task.dependsOn(copyRestYamlTestTask);
-                task.coreConfig = project.getConfigurations().create("restSpec");
+                task.coreConfig = project.getConfigurations().maybeCreate("restSpecs");
                 if (BuildParams.isInternal()) {
                     Dependency restSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
                     project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
 
-                    task.xpackConfig = project.getConfigurations().create("restXpackSpec");
+                    task.xpackConfig = project.getConfigurations().maybeCreate("restXpackSpecs");
                     Dependency restXpackSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency);
diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle
index df648262e9973..4fb1de7f6cbed 100644
--- a/x-pack/plugin/watcher/qa/rest/build.gradle
+++ b/x-pack/plugin/watcher/qa/rest/build.gradle
@@ -8,6 +8,7 @@ dependencies {
 
 configurations {
   testArtifacts.extendsFrom testRuntime
+  restXpackTests
 }
 
 task testJar(type: Jar) {
@@ -17,6 +18,7 @@ task testJar(type: Jar) {
 
 artifacts {
   testArtifacts testJar
+  restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
 }
 
 restResources {
diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle
index c80b1894d0c73..5a8b03ca68d37 100644
--- a/x-pack/plugin/watcher/qa/with-security/build.gradle
+++ b/x-pack/plugin/watcher/qa/with-security/build.gradle
@@ -7,21 +7,15 @@ dependencies {
   testCompile project(path: ':x-pack:plugin:watcher:qa:rest', configuration: 'testArtifacts')
 }
 
-
-// bring in watcher rest test suite from the rest project
-task copyWatcherRestTests(type: Copy) {
-  into project.sourceSets.test.output.resourcesDir
-  from project(xpackProject('plugin:watcher:qa:rest').path).sourceSets.test.resources.srcDirs
-  include 'rest-api-spec/test/watcher/**'
-}
-
 restResources {
   restApi {
     includeXpack 'watcher', 'security', 'xpack'
   }
+  restTests {
+    includeXpack 'watcher'
+  }
 }
 
-integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'

From 4f48e053f92b2ae7677a2e2ba0e5464ede7128d1 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Mon, 16 Mar 2020 10:32:47 -0500
Subject: [PATCH 054/285] Revert "Add Watcher to available rest resources
 (#53319)"

This reverts commit 887541e865f68fbbe13b27172b7fd230638aae93.
---
 .../gradle/test/rest/CopyRestTestsTask.java     |  4 ++--
 .../gradle/test/rest/RestResourcesPlugin.java   | 17 +++++------------
 x-pack/plugin/watcher/qa/rest/build.gradle      |  2 --
 .../watcher/qa/with-security/build.gradle       | 12 +++++++++---
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 4c86b86b56f5b..9e9d1e0d9e795 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -114,7 +114,7 @@ void copy() {
             if (BuildParams.isInternal()) {
                 getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
                 project.copy(c -> {
-                    c.from(coreConfig.getAsFileTree());
+                    c.from(coreConfig.getSingleFile());
                     c.into(getOutputDir());
                     c.include(corePatternSet.getIncludes());
                 });
@@ -138,7 +138,7 @@ void copy() {
         if (includeXpack.get().isEmpty() == false) {
             getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
-                c.from(xpackConfig.getAsFileTree());
+                c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
                 c.include(xpackPatternSet.getIncludes());
             });
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
index 57ee2c1f9037f..a512b9b1fc025 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
@@ -86,27 +86,21 @@ public class RestResourcesPlugin implements Plugin<Project> {
     public void apply(Project project) {
         RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class);
 
-        // tests
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
-                task.coreConfig = project.getConfigurations().maybeCreate("restTests");
+                task.coreConfig = project.getConfigurations().create("restTest");
                 if (BuildParams.isInternal()) {
-                    // core
                     Dependency restTestdependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
                     project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
-                    // x-pack
-                    task.xpackConfig = project.getConfigurations().maybeCreate("restXpackTests");
+
+                    task.xpackConfig = project.getConfigurations().create("restXpackTest");
                     Dependency restXPackTestdependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency);
                     task.dependsOn(task.xpackConfig);
-                    // watcher
-                    Dependency restWatcherTests = project.getDependencies()
-                        .project(Map.of("path", ":x-pack:plugin:watcher:qa:rest", "configuration", "restXpackTests"));
-                    project.getDependencies().add(task.xpackConfig.getName(), restWatcherTests);
                 } else {
                     Dependency dependency = project.getDependencies()
                         .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
@@ -115,19 +109,18 @@ public void apply(Project project) {
                 task.dependsOn(task.coreConfig);
             });
 
-        // api
         Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.includeXpack.set(extension.restApi.getIncludeXpack());
                 task.dependsOn(copyRestYamlTestTask);
-                task.coreConfig = project.getConfigurations().maybeCreate("restSpecs");
+                task.coreConfig = project.getConfigurations().create("restSpec");
                 if (BuildParams.isInternal()) {
                     Dependency restSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
                     project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
 
-                    task.xpackConfig = project.getConfigurations().maybeCreate("restXpackSpecs");
+                    task.xpackConfig = project.getConfigurations().create("restXpackSpec");
                     Dependency restXpackSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency);
diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle
index 4fb1de7f6cbed..df648262e9973 100644
--- a/x-pack/plugin/watcher/qa/rest/build.gradle
+++ b/x-pack/plugin/watcher/qa/rest/build.gradle
@@ -8,7 +8,6 @@ dependencies {
 
 configurations {
   testArtifacts.extendsFrom testRuntime
-  restXpackTests
 }
 
 task testJar(type: Jar) {
@@ -18,7 +17,6 @@ task testJar(type: Jar) {
 
 artifacts {
   testArtifacts testJar
-  restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
 }
 
 restResources {
diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle
index 5a8b03ca68d37..c80b1894d0c73 100644
--- a/x-pack/plugin/watcher/qa/with-security/build.gradle
+++ b/x-pack/plugin/watcher/qa/with-security/build.gradle
@@ -7,15 +7,21 @@ dependencies {
   testCompile project(path: ':x-pack:plugin:watcher:qa:rest', configuration: 'testArtifacts')
 }
 
+
+// bring in watcher rest test suite from the rest project
+task copyWatcherRestTests(type: Copy) {
+  into project.sourceSets.test.output.resourcesDir
+  from project(xpackProject('plugin:watcher:qa:rest').path).sourceSets.test.resources.srcDirs
+  include 'rest-api-spec/test/watcher/**'
+}
+
 restResources {
   restApi {
     includeXpack 'watcher', 'security', 'xpack'
   }
-  restTests {
-    includeXpack 'watcher'
-  }
 }
 
+integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'

From 0f0091e7998f5274ab0529469b7b1040c968ee9a Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Mon, 16 Mar 2020 18:07:14 +0100
Subject: [PATCH 055/285] Disable Watcher script optimization for stored
 scripts (#53497)

The watcher TextTemplateEngine uses a fast path mechanism where it
checks for the existence of `{{` to decide if a mustache script
required compilation. This does not work for stored script, as the field
that is checked contains the id of the script, which means, the name of
the script is returned as its value.

This commit checks for the script type and does not involve this fast
path check if a stored script is used.

Closes #40212
---
 .../watcher/common/text/TextTemplate.java     | 18 ++++++++-----
 .../common/text/TextTemplateEngine.java       |  2 +-
 .../notification/email/EmailTemplate.java     |  2 +-
 .../common/text/TextTemplateTests.java        | 26 +++++++++++++++++++
 4 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java
index 224dcc7140d2a..a261e6d01af1c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java
@@ -29,12 +29,12 @@ public class TextTemplate implements ToXContent {
 
     private final Script script;
     private final String inlineTemplate;
-    private final boolean isUsingMustache;
+    private final boolean mayRequireCompilation;
 
     public TextTemplate(String template) {
         this.script = null;
         this.inlineTemplate = template;
-        this.isUsingMustache = template.contains("{{");
+        this.mayRequireCompilation = template.contains("{{");
     }
 
     public TextTemplate(String template, @Nullable XContentType contentType, ScriptType type,
@@ -50,14 +50,14 @@ public TextTemplate(String template, @Nullable XContentType contentType, ScriptT
             params = new HashMap<>();
         }
         this.script = new Script(type, type == ScriptType.STORED ? null : Script.DEFAULT_TEMPLATE_LANG, template, options, params);
-        this.isUsingMustache = template.contains("{{");
         this.inlineTemplate = null;
+        this.mayRequireCompilation = script.getType() == ScriptType.STORED || script.getIdOrCode().contains("{{");
     }
 
     public TextTemplate(Script script) {
         this.script = script;
         this.inlineTemplate = null;
-        this.isUsingMustache = script.getIdOrCode().contains("{{");
+        this.mayRequireCompilation = script.getType() == ScriptType.STORED || script.getIdOrCode().contains("{{");
     }
 
     public Script getScript() {
@@ -68,8 +68,14 @@ public String getTemplate() {
         return script != null ? script.getIdOrCode() : inlineTemplate;
     }
 
-    public boolean isUsingMustache() {
-        return isUsingMustache;
+    /**
+     * Check if compilation may be required.
+     * If a stored script is used, we cannot tell at this stage, so we always assume
+     * that stored scripts require compilation.
+     * If an inline script is used, we checked for the mustache opening brackets
+     */
+    public boolean mayRequireCompilation() {
+        return mayRequireCompilation;
     }
 
     public XContentType getContentType() {
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java
index 2f2d3d7b9f31a..fed102f1d5c1f 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java
@@ -32,7 +32,7 @@ public String render(TextTemplate textTemplate, Map<String, Object> model) {
         String mediaType = compileParams(detectContentType(template));
         template = trimContentType(textTemplate);
 
-        if (textTemplate.isUsingMustache() == false) {
+        if (textTemplate.mayRequireCompilation() == false) {
             return template;
         }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java
index dc48dd2b9f23f..2735ad9fc93b1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java
@@ -455,7 +455,7 @@ public boolean handle(String fieldName, XContentParser parser) throws IOExceptio
         static void validateEmailAddresses(TextTemplate ... emails) {
             for (TextTemplate emailTemplate : emails) {
                 // no mustache, do validation
-                if (emailTemplate.isUsingMustache() == false) {
+                if (emailTemplate.mayRequireCompilation() == false) {
                     String email = emailTemplate.getTemplate();
                     try {
                         for (Email.Address address : Email.AddressList.parse(email)) {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java
index 2ebaccc61801c..a563695732a18 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java
@@ -225,6 +225,32 @@ public void testParserInvalidMissingText() throws Exception {
         assertEquals("[1:2] [script] unknown field [type]", ex.getMessage());
     }
 
+    public void testMustacheTemplateRequiresCompilation() {
+        final TextTemplate inlineTemplateRequiresCompilation = createTextTemplate(ScriptType.INLINE, "{{foo.bar}}");
+        assertThat(inlineTemplateRequiresCompilation.mayRequireCompilation(), is(true));
+
+        final TextTemplate inlineTemplateNoRequiresCompilation = createTextTemplate(ScriptType.INLINE, "script without mustache");
+        assertThat(inlineTemplateNoRequiresCompilation.mayRequireCompilation(), is(false));
+
+        final TextTemplate storedScriptTemplate = createTextTemplate(ScriptType.STORED, "stored_script_id");
+        assertThat(storedScriptTemplate.mayRequireCompilation(), is(true));
+    }
+
+    private TextTemplate createTextTemplate(ScriptType type, String idOrCode) {
+        final TextTemplate template;
+        if (randomBoolean()) {
+            if (type == ScriptType.STORED) {
+                template = new TextTemplate(new Script(type, null, idOrCode, null, Collections.emptyMap()));
+            } else {
+                template = new TextTemplate(new Script(type, lang, idOrCode, Collections.emptyMap(), Collections.emptyMap()));
+            }
+        } else {
+            template = new TextTemplate(idOrCode, null, type,  null);
+        }
+
+        return template;
+    }
+
     public void testNullObject() throws Exception {
         assertThat(engine.render(null ,new HashMap<>()), is(nullValue()));
     }

From f63295bd8feb3c10859a2811788a088e752d25c0 Mon Sep 17 00:00:00 2001
From: Andrei Stefan <astefan@users.noreply.github.com>
Date: Mon, 16 Mar 2020 19:20:04 +0200
Subject: [PATCH 056/285] Remove CONSTANT_KEYWORD from LiteralTests.java
 (#53606)

---
 .../org/elasticsearch/xpack/ql/expression/LiteralTests.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
index b40edfc381fc5..abc9fee4d97db 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/LiteralTests.java
@@ -20,7 +20,6 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
@@ -54,7 +53,7 @@ static class ValueAndCompatibleTypes {
             new ValueAndCompatibleTypes(ESTestCase::randomLong, LONG, FLOAT, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomFloat, FLOAT, LONG, DOUBLE, BOOLEAN),
             new ValueAndCompatibleTypes(ESTestCase::randomDouble, DOUBLE, LONG, FLOAT, BOOLEAN),
-            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD, CONSTANT_KEYWORD));
+            new ValueAndCompatibleTypes(() -> randomAlphaOfLength(5), KEYWORD));
 
     public static Literal randomLiteral() {
         ValueAndCompatibleTypes gen = randomFrom(GENERATORS);

From 230a9a89756a2c733a625f22ebed022640d7a041 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 16 Mar 2020 13:23:22 -0400
Subject: [PATCH 057/285] Improve top_metrics docs (#53521)

* Removes experimental.
* Replaces `"v"` (for value) with `"m"` (for metric).
* Move the note about tiebreaking into the list of limitations of the
  sort.
* Explain how you ask for `metrics`.
* Clean up some wording.
* Link to the docs from `top_metrics`.

Closes #51813
---
 .../metrics/top-metrics-aggregation.asciidoc  | 93 +++++++++----------
 .../metrics/tophits-aggregation.asciidoc      |  3 +
 2 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
index 5958d09f4ed71..6dbc8adfd3ff8 100644
--- a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
@@ -3,26 +3,24 @@
 [[search-aggregations-metrics-top-metrics]]
 === Top Metrics Aggregation
 
-experimental[We expect to change the response format of this aggregation as we add more features., https://github.com/elastic/elasticsearch/issues/51813]
-
 The `top_metrics` aggregation selects metrics from the document with the largest or smallest "sort"
-value. For example, This gets the value of the `v` field on the document with the largest value of `s`:
+value. For example, this gets the value of the `m` field on the document with the largest value of `s`:
 
 [source,console,id=search-aggregations-metrics-top-metrics-simple]
 ----
 POST /test/_bulk?refresh
 {"index": {}}
-{"s": 1, "v": 3.1415}
+{"s": 1, "m": 3.1415}
 {"index": {}}
-{"s": 2, "v": 1.0}
+{"s": 2, "m": 1.0}
 {"index": {}}
-{"s": 3, "v": 2.71828}
+{"s": 3, "m": 2.71828}
 POST /test/_search?filter_path=aggregations
 {
   "aggs": {
     "tm": {
       "top_metrics": {
-        "metrics": {"field": "v"},
+        "metrics": {"field": "m"},
         "sort": {"s": "desc"}
       }
     }
@@ -37,7 +35,7 @@ Which returns:
 {
   "aggregations": {
     "tm": {
-      "top": [ {"sort": [3], "metrics": {"v": 2.718280076980591 } } ]
+      "top": [ {"sort": [3], "metrics": {"m": 2.718280076980591 } } ]
     }
   }
 }
@@ -54,7 +52,7 @@ The `sort` field in the metric request functions exactly the same as the `sort`
 <<request-body-search-sort, search>> request except:
 * It can't be used on <<binary,binary>>, <<flattened,flattened>, <<ip,ip>>,
 <<keyword,keyword>>, or <<text,text>> fields.
-* It only supports a single sort value.
+* It only supports a single sort value so which document wins ties is not specified.
 
 The metrics that the aggregation returns is the first hit that would be returned by the search
 request. So,
@@ -65,14 +63,12 @@ request. So,
   gets metrics from the documents with `location` *closest* to `35.7796, -78.6382`
 `"sort": "_score"`:: gets metrics from the document with the highest score
 
-NOTE: This aggregation doesn't support any sort of "tie breaking". If two documents have
-the same sort values then this aggregation could return either document's fields.
-
 ==== `metrics`
 
-`metrics` selects the fields to of the "top" document to return.
-
-You can return multiple metrics by providing a list:
+`metrics` selects the fields to of the "top" document to return. You can return
+a request a single metric with something like `"metric": {"field": "m"}` or multiple
+metrics by requesting a list of metrics like `"metric": [{"field": "m"}, {"field": "i"}`.
+Here is a more complete example:
 
 [source,console,id=search-aggregations-metrics-top-metrics-list-of-metrics]
 ----
@@ -86,19 +82,19 @@ PUT /test
 }
 POST /test/_bulk?refresh
 {"index": {}}
-{"s": 1, "v": 3.1415, "m": 1, "d": "2020-01-01T00:12:12Z"}
+{"s": 1, "m": 3.1415, "i": 1, "d": "2020-01-01T00:12:12Z"}
 {"index": {}}
-{"s": 2, "v": 1.0, "m": 6, "d": "2020-01-02T00:12:12Z"}
+{"s": 2, "m": 1.0, "i": 6, "d": "2020-01-02T00:12:12Z"}
 {"index": {}}
-{"s": 3, "v": 2.71828, "m": -12, "d": "2019-12-31T00:12:12Z"}
+{"s": 3, "m": 2.71828, "i": -12, "d": "2019-12-31T00:12:12Z"}
 POST /test/_search?filter_path=aggregations
 {
   "aggs": {
     "tm": {
       "top_metrics": {
         "metrics": [
-          {"field": "v"},
           {"field": "m"},
+          {"field": "i"},
           {"field": "d"}
         ],
         "sort": {"s": "desc"}
@@ -118,8 +114,8 @@ Which returns:
       "top": [ {
         "sort": [3],
         "metrics": {
-          "v": 2.718280076980591,
-          "m": -12,
+          "m": 2.718280076980591,
+          "i": -12,
           "d": "2019-12-31T00:12:12.000Z"
         }
       } ]
@@ -137,19 +133,19 @@ Which returns:
 ----
 POST /test/_bulk?refresh
 {"index": {}}
-{"s": 1, "v": 3.1415}
+{"s": 1, "m": 3.1415}
 {"index": {}}
-{"s": 2, "v": 1.0}
+{"s": 2, "m": 1.0}
 {"index": {}}
-{"s": 3, "v": 2.71828}
+{"s": 3, "m": 2.71828}
 POST /test/_search?filter_path=aggregations
 {
   "aggs": {
     "tm": {
       "top_metrics": {
-        "metrics": {"field": "v"},
+        "metrics": {"field": "m"},
         "sort": {"s": "desc"},
-        "size": 2
+        "size": 3
       }
     }
   }
@@ -164,8 +160,9 @@ Which returns:
   "aggregations": {
     "tm": {
       "top": [
-        {"sort": [3], "metrics": {"v": 2.718280076980591 } },
-        {"sort": [2], "metrics": {"v": 1.0 } }
+        {"sort": [3], "metrics": {"m": 2.718280076980591 } },
+        {"sort": [2], "metrics": {"m": 1.0 } },
+        {"sort": [1], "metrics": {"m": 3.1414999961853027 } }
       ]
     }
   }
@@ -179,7 +176,8 @@ is a *very* conservative default maximum and you can raise it if you need to by
 changing the `top_metrics_max_size` index setting. But know that large sizes can
 take a fair bit of memory, especially if they are inside of an aggregation which
 makes many buckes like a large
-<<search-aggregations-metrics-top-metrics-example-terms, terms aggregation>>.
+<<search-aggregations-metrics-top-metrics-example-terms, terms aggregation>>. If
+you till want to raise it, use something like:
 
 [source,console]
 ----
@@ -190,8 +188,7 @@ PUT /test/_settings
 ----
 // TEST[continued]
 
-NOTE: If `size` is more than `1` the `top_metrics` aggregation can't be the target of a sort.
-
+NOTE: If `size` is more than `1` the `top_metrics` aggregation can't be the *target* of a sort.
 
 ==== Examples
 
@@ -214,11 +211,11 @@ PUT /node
 }
 POST /node/_bulk?refresh
 {"index": {}}
-{"ip": "192.168.0.1", "date": "2020-01-01T01:01:01", "v": 1}
+{"ip": "192.168.0.1", "date": "2020-01-01T01:01:01", "m": 1}
 {"index": {}}
-{"ip": "192.168.0.1", "date": "2020-01-01T02:01:01", "v": 2}
+{"ip": "192.168.0.1", "date": "2020-01-01T02:01:01", "m": 2}
 {"index": {}}
-{"ip": "192.168.0.2", "date": "2020-01-01T02:01:01", "v": 3}
+{"ip": "192.168.0.2", "date": "2020-01-01T02:01:01", "m": 3}
 POST /node/_search?filter_path=aggregations
 {
   "aggs": {
@@ -229,7 +226,7 @@ POST /node/_search?filter_path=aggregations
       "aggs": {
         "tm": {
           "top_metrics": {
-            "metrics": {"field": "v"},
+            "metrics": {"field": "m"},
             "sort": {"date": "desc"}
           }
         }
@@ -251,14 +248,14 @@ Which returns:
           "key": "192.168.0.1",
           "doc_count": 2,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"m": 2 } } ]
           }
         },
         {
           "key": "192.168.0.2",
           "doc_count": 1,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"m": 3 } } ]
           }
         }
       ],
@@ -280,12 +277,12 @@ POST /node/_search?filter_path=aggregations
     "ip": {
       "terms": {
         "field": "ip",
-        "order": {"tm.v": "desc"}
+        "order": {"tm.m": "desc"}
       },
       "aggs": {
         "tm": {
           "top_metrics": {
-            "metrics": {"field": "v"},
+            "metrics": {"field": "m"},
             "sort": {"date": "desc"}
           }
         }
@@ -308,14 +305,14 @@ Which returns:
           "key": "192.168.0.2",
           "doc_count": 1,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 3 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"m": 3 } } ]
           }
         },
         {
           "key": "192.168.0.1",
           "doc_count": 2,
           "tm": {
-            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"v": 2 } } ]
+            "top": [ {"sort": ["2020-01-01T02:01:01.000Z"], "metrics": {"m": 2 } } ]
           }
         }
       ],
@@ -337,17 +334,17 @@ always sorted independantly of whole numbered fields.
 ----
 POST /test/_bulk?refresh
 {"index": {"_index": "test1"}}
-{"s": 1, "v": 3.1415}
+{"s": 1, "m": 3.1415}
 {"index": {"_index": "test1"}}
-{"s": 2, "v": 1}
+{"s": 2, "m": 1}
 {"index": {"_index": "test2"}}
-{"s": 3.1, "v": 2.71828}
+{"s": 3.1, "m": 2.71828}
 POST /test*/_search?filter_path=aggregations
 {
   "aggs": {
     "tm": {
       "top_metrics": {
-        "metrics": {"field": "v"},
+        "metrics": {"field": "m"},
         "sort": {"s": "asc"}
       }
     }
@@ -362,7 +359,7 @@ Which returns:
 {
   "aggregations": {
     "tm": {
-      "top": [ {"sort": [3.0999999046325684], "metrics": {"v": 2.718280076980591 } } ]
+      "top": [ {"sort": [3.0999999046325684], "metrics": {"m": 2.718280076980591 } } ]
     }
   }
 }
@@ -380,7 +377,7 @@ POST /test*/_search?filter_path=aggregations
   "aggs": {
     "tm": {
       "top_metrics": {
-        "metrics": {"field": "v"},
+        "metrics": {"field": "m"},
         "sort": {"s": {"order": "asc", "numeric_type": "double"}}
       }
     }
@@ -396,7 +393,7 @@ Which returns the much more expected:
 {
   "aggregations": {
     "tm": {
-      "top": [ {"sort": [1.0], "metrics": {"v": 3.1414999961853027 } } ]
+      "top": [ {"sort": [1.0], "metrics": {"m": 3.1414999961853027 } } ]
     }
   }
 }
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index e3cc9a8ce4efd..326b6faf47436 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -27,6 +27,9 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<request-body-search-version,Include versions>>
 * <<request-body-search-seq-no-primary-term,Include Sequence Numbers and Primary Terms>>
 
+IMPORTANT: If you *only* need `docvalue_fields`, `size`, and `sort` then
+<<search-aggregations-metrics-top-metrics>> might be a more efficient choice than the Top Hits Aggregation.
+
 ==== Example
 
 In the following example we group the sales by type and per type we show the last sale. 

From 3495239b7ff5e481677c712c93883e96cab00d4d Mon Sep 17 00:00:00 2001
From: Gordon Brown <gordon.brown@elastic.co>
Date: Mon, 16 Mar 2020 11:24:23 -0600
Subject: [PATCH 058/285] Hide I/SLM history aliases (#53564)

This commit adjusts the aliases used for the ILM and SLM history indices
to be hidden aliases.

Also tweaks the configuration of the `IndexTemplateRegistry`s used by
these history system to only upgrade the template from the master node,
as documents are indexed from the master node, so the template version
should only be upgraded from the master node.
---
 .../xpack/core/slm/history/SnapshotHistoryStore.java         | 3 ++-
 .../core/slm/history/SnapshotLifecycleTemplateRegistry.java  | 5 +++++
 .../org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java | 2 +-
 .../xpack/ilm/history/ILMHistoryTemplateRegistry.java        | 5 +++++
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotHistoryStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotHistoryStore.java
index b2a5891558617..7327aa3fa81c4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotHistoryStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotHistoryStore.java
@@ -98,7 +98,8 @@ static void ensureHistoryIndex(Client client, ClusterState state, ActionListener
             client.admin().indices().prepareCreate(initialHistoryIndexName)
                 .setWaitForActiveShards(1)
                 .addAlias(new Alias(SLM_HISTORY_ALIAS)
-                    .writeIndex(true))
+                    .writeIndex(true)
+                    .isHidden(true))
                 .execute(new ActionListener<>() {
                     @Override
                     public void onResponse(CreateIndexResponse response) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java
index ef768cd992c37..a166f9ecb022c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java
@@ -43,6 +43,11 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry {
 
     public static final String SLM_POLICY_NAME = "slm-history-ilm-policy";
 
+    @Override
+    protected boolean requiresMasterNode() {
+        return true;
+    }
+
     public static final IndexTemplateConfig TEMPLATE_SLM_HISTORY = new IndexTemplateConfig(
         SLM_TEMPLATE_NAME,
         "/slm-history.json",
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
index cf80312c2d658..a2bb00ea5c73d 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
@@ -199,7 +199,7 @@ static void ensureHistoryIndex(Client client, ClusterState state, ActionListener
                 .setSettings((Map<String, ?>) templateAsMap.get("settings"))
                 .setMapping((Map<String, Object>) templateAsMap.get("mappings"))
                 .setWaitForActiveShards(1)
-                .addAlias(new Alias(ILM_HISTORY_ALIAS).writeIndex(true))
+                .addAlias(new Alias(ILM_HISTORY_ALIAS).writeIndex(true).isHidden(true))
                 .execute(new ActionListener<>() {
                     @Override
                     public void onResponse(CreateIndexResponse response) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
index 149d4297cafeb..838da9d04045b 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
@@ -35,6 +35,11 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry {
 
     public static final String ILM_POLICY_NAME = "ilm-history-ilm-policy";
 
+    @Override
+    protected boolean requiresMasterNode() {
+        return true;
+    }
+
     public static final IndexTemplateConfig TEMPLATE_ILM_HISTORY = new IndexTemplateConfig(
         ILM_TEMPLATE_NAME,
         "/ilm-history.json",

From 3b7843d77400911910bd18ecb93eb81aee5e5cab Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 16 Mar 2020 14:41:56 -0400
Subject: [PATCH 059/285] Fix sorting agg buckets by doc_count (#53617)

I broke sorting aggregations by `doc_count` in #51271 by mixing up true
and false. This flips that comparison and adds a few tests to double
check that we don't so this again.
---
 .../test/search.aggregation/50_filter.yml     | 79 ++++++++++++++++++-
 .../bucket/BucketsAggregator.java             |  2 +-
 .../bucket/filter/FilterAggregatorTests.java  | 38 +++++++++
 3 files changed, 114 insertions(+), 5 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml
index 1c88758173936..b059e0892be08 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml
@@ -14,10 +14,10 @@ setup:
                   type: keyword
 
   - do:
-        index:
-          index: test
-          id: foo|bar|baz0
-          body: { "notifications" : ["abc"] }
+      index:
+        index: test
+        id: foo|bar|baz0
+        body: { "notifications" : ["abc"] }
 
   - do:
       index:
@@ -73,3 +73,74 @@ setup:
   - match: { _all.total.request_cache.hit_count: 0 }
   - match: { _all.total.request_cache.miss_count: 1 }
   - is_true: indices.test
+
+---
+"As a child of terms":
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body: |
+          {"index":{}}
+          {"category": "bar", "val": 8}
+          {"index":{}}
+          {"category": "bar", "val": 0}
+  - do:
+      search:
+        size: 0
+        body:
+          aggs:
+            category:
+              terms:
+                field: category.keyword
+              aggs:
+                high:
+                  filter:
+                    range:
+                      val:
+                        gte: 7
+
+  - match: { hits.total.value: 4 }
+  - match: { aggregations.category.buckets.0.key: bar }
+  - match: { aggregations.category.buckets.0.doc_count: 2 }
+  - match: { aggregations.category.buckets.0.high.doc_count: 1 }
+
+---
+"Sorting terms":
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body: |
+          {"index":{}}
+          {"category": "foo", "val": 7}
+          {"index":{}}
+          {"category": "bar", "val": 8}
+          {"index":{}}
+          {"category": "bar", "val": 9}
+          {"index":{}}
+          {"category": "bar", "val": 0}
+  - do:
+      search:
+        size: 0
+        body:
+          aggs:
+            category:
+              terms:
+                field: category.keyword
+                order:
+                  high.doc_count: desc
+              aggs:
+                high:
+                  filter:
+                    range:
+                      val:
+                        gte: 7
+
+  - match: { hits.total.value: 6 }
+  - match: { aggregations.category.buckets.0.key: bar }
+  - match: { aggregations.category.buckets.0.doc_count: 3 }
+  - match: { aggregations.category.buckets.0.high.doc_count: 2 }
+  - match: { aggregations.category.buckets.1.key: foo }
+  - match: { aggregations.category.buckets.1.doc_count: 1 }
+  - match: { aggregations.category.buckets.1.high.doc_count: 1 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
index f26cd0478a036..acf4d6c7bcb98 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
@@ -176,7 +176,7 @@ public Aggregator resolveSortPath(AggregationPath.PathElement next, Iterator<Agg
 
     @Override
     public BucketComparator bucketComparator(String key, SortOrder order) {
-        if (key == null || false == "doc_count".equals(key)) {
+        if (key == null || "doc_count".equals(key)) {
             return (lhs, rhs) -> order.reverseMul() * Integer.compare(bucketDocCount(lhs), bucketDocCount(rhs));
         }
         throw new IllegalArgumentException("Ordering on a single-bucket aggregation can only be done on its doc_count. " +
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java
index 56153d2d35f13..9e57545a47bc5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java
@@ -29,12 +29,23 @@
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.Aggregator.BucketComparator;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.sort.SortOrder;
 import org.junit.Before;
 
+import java.io.IOException;
+
+import static java.util.Collections.singleton;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+
 public class FilterAggregatorTests extends AggregatorTestCase {
     private MappedFieldType fieldType;
 
@@ -110,6 +121,33 @@ public void testRandom() throws Exception {
             indexReader.close();
             directory.close();
         }
+    }
 
+    public void testBucketComparator() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                indexWriter.addDocument(singleton(new Field("field", "1", fieldType)));
+            }
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+                FilterAggregationBuilder builder = new FilterAggregationBuilder("test", new MatchAllQueryBuilder());
+                FilterAggregator agg = createAggregator(builder, indexSearcher, fieldType);
+                agg.preCollection();
+                LeafBucketCollector collector = agg.getLeafCollector(indexReader.leaves().get(0));
+                collector.collect(0, 0);
+                collector.collect(0, 0);
+                collector.collect(0, 1);
+                BucketComparator c = agg.bucketComparator(null, SortOrder.ASC);
+                assertThat(c.compare(0, 1), greaterThan(0));
+                assertThat(c.compare(1, 0), lessThan(0));
+                c = agg.bucketComparator("doc_count", SortOrder.ASC);
+                assertThat(c.compare(0, 1), greaterThan(0));
+                assertThat(c.compare(1, 0), lessThan(0));
+                Exception e = expectThrows(IllegalArgumentException.class, () ->
+                    agg.bucketComparator("garbage", randomFrom(SortOrder.values())));
+                assertThat(e.getMessage(), equalTo("Ordering on a single-bucket aggregation can only be done on its doc_count. "
+                        + "Either drop the key (a la \"test\") or change it to \"doc_count\" (a la \"test.doc_count\") or \"key\"."));
+            }
+        }
     }
 }

From 4d81edb6257732a73adc3e5bc5157ad01d1a42c4 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 16 Mar 2020 14:51:54 -0400
Subject: [PATCH 060/285] Stop using round-tripped PipelineAggregators (#53423)

This begins to clean up how `PipelineAggregator`s and executed.
Previously, we would create the `PipelineAggregator`s on the data nodes
and embed them in the aggregation tree. When it came time to execute the
pipeline aggregation we'd use the `PipelineAggregator`s that were on the
first shard's results. This is inefficient because:
1. The data node needs to make the `PipelineAggregator` only to
   serialize it and then throw it away.
2. The coordinating node needs to deserialize all of the
   `PipelineAggregator`s even though it only needs one of them.
3. You end up with many `PipelineAggregator` instances when you only
   really *need* one per pipeline.
4. `PipelineAggregator` needs to implement serialization.

This begins to undo these by building the `PipelineAggregator`s directly
on the coordinating node and using those instead of the
`PipelineAggregator`s in the aggregtion tree. In a follow up change
we'll stop serializing the `PipelineAggregator`s to node versions that
support this behavior. And, one day, we'll be able to remove
`PipelineAggregator` from the aggregation result tree entirely.

Importantly, this doesn't change how pipeline aggregations are declared
or parsed or requested. They are still part of the `AggregationBuilder`
tree because *that* makes sense.
---
 .../stats/InternalMatrixStatsTests.java       |  5 +-
 .../action/search/SearchPhaseController.java  | 55 +++++++++++-------
 .../action/search/SearchResponseMerger.java   | 15 +++--
 .../action/search/TransportSearchAction.java  | 16 ++---
 .../java/org/elasticsearch/node/Node.java     |  2 +-
 .../elasticsearch/search/SearchService.java   | 33 +++++++++--
 .../aggregations/AggregationBuilder.java      | 10 ++++
 .../aggregations/AggregatorFactories.java     | 42 ++++++++------
 .../aggregations/InternalAggregation.java     | 56 ++++++++++++++----
 .../aggregations/InternalAggregations.java    |  9 ++-
 .../InternalMultiBucketAggregation.java       | 18 +++---
 .../InternalSingleBucketAggregation.java      | 14 +++--
 .../pipeline/PipelineAggregator.java          | 44 ++++++++++++++
 .../action/search/DfsQueryPhaseTests.java     | 18 +++---
 .../action/search/FetchSearchPhaseTests.java  | 21 +++----
 .../search/SearchPhaseControllerTests.java    | 58 ++++++++++++-------
 .../search/SearchResponseMergerTests.java     | 29 +++++-----
 .../search/TransportSearchActionTests.java    | 50 ++++++++++------
 .../search/SearchServiceTests.java            |  7 ++-
 .../AggregatorFactoriesTests.java             | 14 +++++
 .../BasePipelineAggregationTestCase.java      |  2 +-
 .../InternalAggregationsTests.java            | 27 ++++-----
 .../composite/InternalCompositeTests.java     |  9 +--
 .../histogram/InternalHistogramTests.java     |  5 +-
 .../SignificanceHeuristicTests.java           |  3 +-
 .../bucket/terms/TermsAggregatorTests.java    |  7 ++-
 .../snapshots/SnapshotResiliencyTests.java    |  2 +-
 .../aggregations/AggregatorTestCase.java      | 24 ++++----
 .../test/InternalAggregationTestCase.java     | 31 +++++++---
 .../xpack/search/AsyncSearchTask.java         | 14 ++---
 .../xpack/search/MutableSearchResponse.java   | 14 ++---
 .../TransportSubmitAsyncSearchAction.java     | 11 ++--
 .../rollup/RollupResponseTranslator.java      | 11 ++--
 .../action/TransportRollupSearchAction.java   |  3 +-
 .../RollupResponseTranslationTests.java       | 33 ++++++-----
 35 files changed, 458 insertions(+), 254 deletions(-)

diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
index 428889b1f67fd..2d0997df268c3 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.search.aggregations.ParsedAggregation;
 import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats.Fields;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.test.InternalAggregationTestCase;
 
 import java.io.IOException;
@@ -162,8 +163,8 @@ public void testReduceRandom() {
 
         ScriptService mockScriptService = mockScriptService();
         MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
-        InternalAggregation.ReduceContext context =
-            new InternalAggregation.ReduceContext(bigArrays, mockScriptService, true);
+        InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                bigArrays, mockScriptService, b -> {}, PipelineTree.EMPTY);
         InternalMatrixStats reduced = (InternalMatrixStats) shardResults.get(0).reduce(shardResults, context);
         multiPassStats.assertNearlyEqual(reduced.getResults());
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 423f930b78fbe..d8a3e26b8d82b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.IntArrayList;
 import com.carrotsearch.hppc.ObjectObjectHashMap;
+
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.FieldDoc;
@@ -69,17 +70,13 @@
 import java.util.stream.Collectors;
 
 public final class SearchPhaseController {
-
     private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
 
-    private final Function<Boolean, ReduceContext> reduceContextFunction;
+    private final Function<SearchRequest, InternalAggregation.ReduceContextBuilder> requestToAggReduceContextBuilder;
 
-    /**
-     * Constructor.
-     * @param reduceContextFunction A function that builds a context for the reduce of an {@link InternalAggregation}
-     */
-    public SearchPhaseController(Function<Boolean, ReduceContext> reduceContextFunction) {
-        this.reduceContextFunction = reduceContextFunction;
+    public SearchPhaseController(
+            Function<SearchRequest, InternalAggregation.ReduceContextBuilder> requestToAggReduceContextBuilder) {
+        this.requestToAggReduceContextBuilder = requestToAggReduceContextBuilder;
     }
 
     public AggregatedDfs aggregateDfs(Collection<DfsSearchResult> results) {
@@ -394,7 +391,18 @@ private SearchHits getHits(ReducedQueryPhase reducedQueryPhase, boolean ignoreFr
      * @param queryResults a list of non-null query shard results
      */
     ReducedQueryPhase reducedScrollQueryPhase(Collection<? extends SearchPhaseResult> queryResults) {
-        return reducedQueryPhase(queryResults, true, SearchContext.TRACK_TOTAL_HITS_ACCURATE, true);
+        InternalAggregation.ReduceContextBuilder aggReduceContextBuilder = new InternalAggregation.ReduceContextBuilder() {
+            @Override
+            public ReduceContext forPartialReduction() {
+                throw new UnsupportedOperationException("Scroll requests don't have aggs");
+            }
+
+            @Override
+            public ReduceContext forFinalReduction() {
+                throw new UnsupportedOperationException("Scroll requests don't have aggs");
+            }
+        };
+        return reducedQueryPhase(queryResults, true, SearchContext.TRACK_TOTAL_HITS_ACCURATE, aggReduceContextBuilder, true);
     }
 
     /**
@@ -402,9 +410,11 @@ ReducedQueryPhase reducedScrollQueryPhase(Collection<? extends SearchPhaseResult
      * @param queryResults a list of non-null query shard results
      */
     public ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResult> queryResults,
-                                               boolean isScrollRequest, int trackTotalHitsUpTo, boolean performFinalReduce) {
+                                               boolean isScrollRequest, int trackTotalHitsUpTo,
+                                               InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
+                                               boolean performFinalReduce) {
         return reducedQueryPhase(queryResults, null, new ArrayList<>(), new TopDocsStats(trackTotalHitsUpTo),
-            0, isScrollRequest, performFinalReduce);
+            0, isScrollRequest, aggReduceContextBuilder, performFinalReduce);
     }
 
     /**
@@ -421,6 +431,7 @@ public ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResul
     private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResult> queryResults,
                                                 List<InternalAggregations> bufferedAggs, List<TopDocs> bufferedTopDocs,
                                                 TopDocsStats topDocsStats, int numReducePhases, boolean isScrollRequest,
+                                                InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
                                                 boolean performFinalReduce) {
         assert numReducePhases >= 0 : "num reduce phases must be >= 0 but was: " + numReducePhases;
         numReducePhases++; // increment for this phase
@@ -496,9 +507,8 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
             reducedSuggest = new Suggest(Suggest.reduce(groupedSuggestions));
             reducedCompletionSuggestions = reducedSuggest.filter(CompletionSuggestion.class);
         }
-        ReduceContext reduceContext = reduceContextFunction.apply(performFinalReduce);
-        final InternalAggregations aggregations = aggregationsList.isEmpty() ? null :
-            InternalAggregations.topLevelReduce(aggregationsList, reduceContext);
+        final InternalAggregations aggregations = aggregationsList.isEmpty() ? null : InternalAggregations.topLevelReduce(aggregationsList,
+                    performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction());
         final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
         final SortedTopDocs sortedTopDocs = sortDocs(isScrollRequest, queryResults, bufferedTopDocs, topDocsStats, from, size,
             reducedCompletionSuggestions);
@@ -600,6 +610,7 @@ static final class QueryPhaseResultConsumer extends ArraySearchPhaseResults<Sear
         private int numReducePhases = 0;
         private final TopDocsStats topDocsStats;
         private final int topNSize;
+        private final InternalAggregation.ReduceContextBuilder aggReduceContextBuilder;
         private final boolean performFinalReduce;
 
         /**
@@ -613,7 +624,9 @@ static final class QueryPhaseResultConsumer extends ArraySearchPhaseResults<Sear
          */
         private QueryPhaseResultConsumer(SearchProgressListener progressListener, SearchPhaseController controller,
                                          int expectedResultSize, int bufferSize, boolean hasTopDocs, boolean hasAggs,
-                                         int trackTotalHitsUpTo, int topNSize, boolean performFinalReduce) {
+                                         int trackTotalHitsUpTo, int topNSize,
+                                         InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
+                                         boolean performFinalReduce) {
             super(expectedResultSize);
             if (expectedResultSize != 1 && bufferSize < 2) {
                 throw new IllegalArgumentException("buffer size must be >= 2 if there is more than one expected result");
@@ -635,6 +648,7 @@ private QueryPhaseResultConsumer(SearchProgressListener progressListener, Search
             this.bufferSize = bufferSize;
             this.topDocsStats = new TopDocsStats(trackTotalHitsUpTo);
             this.topNSize = topNSize;
+            this.aggReduceContextBuilder = aggReduceContextBuilder;
             this.performFinalReduce = performFinalReduce;
         }
 
@@ -650,7 +664,7 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
             if (querySearchResult.isNull() == false) {
                 if (index == bufferSize) {
                     if (hasAggs) {
-                        ReduceContext reduceContext = controller.reduceContextFunction.apply(false);
+                        ReduceContext reduceContext = aggReduceContextBuilder.forPartialReduction();
                         InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(Arrays.asList(aggsBuffer), reduceContext);
                         Arrays.fill(aggsBuffer, null);
                         aggsBuffer[0] = reducedAggs;
@@ -693,8 +707,8 @@ private synchronized List<TopDocs> getRemainingTopDocs() {
 
         @Override
         public ReducedQueryPhase reduce() {
-            ReducedQueryPhase reducePhase = controller.reducedQueryPhase(results.asList(),
-                getRemainingAggs(), getRemainingTopDocs(), topDocsStats, numReducePhases, false, performFinalReduce);
+            ReducedQueryPhase reducePhase = controller.reducedQueryPhase(results.asList(), getRemainingAggs(), getRemainingTopDocs(),
+                    topDocsStats, numReducePhases, false, aggReduceContextBuilder, performFinalReduce);
             progressListener.notifyFinalReduce(SearchProgressListener.buildSearchShards(results.asList()),
                 reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases);
             return reducePhase;
@@ -730,13 +744,14 @@ ArraySearchPhaseResults<SearchPhaseResult> newSearchPhaseResults(SearchProgressL
         final boolean hasAggs = source != null && source.aggregations() != null;
         final boolean hasTopDocs = source == null || source.size() != 0;
         final int trackTotalHitsUpTo = resolveTrackTotalHits(request);
+        InternalAggregation.ReduceContextBuilder aggReduceContextBuilder = requestToAggReduceContextBuilder.apply(request);
         if (isScrollRequest == false && (hasAggs || hasTopDocs)) {
             // no incremental reduce if scroll is used - we only hit a single shard or sometimes more...
             if (request.getBatchedReduceSize() < numShards) {
                 int topNSize = getTopDocsSize(request);
                 // only use this if there are aggs and if there are more shards than we should reduce at once
                 return new QueryPhaseResultConsumer(listener, this, numShards, request.getBatchedReduceSize(), hasTopDocs, hasAggs,
-                    trackTotalHitsUpTo, topNSize, request.isFinalReduce());
+                    trackTotalHitsUpTo, topNSize, aggReduceContextBuilder, request.isFinalReduce());
             }
         }
         return new ArraySearchPhaseResults<SearchPhaseResult>(numShards) {
@@ -750,7 +765,7 @@ void consumeResult(SearchPhaseResult result) {
             ReducedQueryPhase reduce() {
                 List<SearchPhaseResult> resultList = results.asList();
                 final ReducedQueryPhase reducePhase =
-                    reducedQueryPhase(resultList, isScrollRequest, trackTotalHitsUpTo, request.isFinalReduce());
+                    reducedQueryPhase(resultList, isScrollRequest, trackTotalHitsUpTo, aggReduceContextBuilder, request.isFinalReduce());
                 listener.notifyFinalReduce(SearchProgressListener.buildSearchShards(resultList),
                     reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases);
                 return reducePhase;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java
index 16092434ab17b..5c1cacab559fd 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java
@@ -27,13 +27,15 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchPhaseController.TopDocsStats;
+import org.elasticsearch.action.search.SearchResponse.Clusters;
 import org.elasticsearch.action.search.TransportSearchAction.SearchTimeProvider;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.profile.ProfileShardResult;
@@ -51,11 +53,8 @@
 import java.util.Objects;
 import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.function.Function;
 
-import static org.elasticsearch.action.search.SearchPhaseController.TopDocsStats;
 import static org.elasticsearch.action.search.SearchPhaseController.mergeTopDocs;
-import static org.elasticsearch.action.search.SearchResponse.Clusters;
 
 /**
  * Merges multiple search responses into one. Used in cross-cluster search when reduction is performed locally on each cluster.
@@ -81,16 +80,16 @@ final class SearchResponseMerger {
     final int size;
     final int trackTotalHitsUpTo;
     private final SearchTimeProvider searchTimeProvider;
-    private final Function<Boolean, ReduceContext> reduceContextFunction;
+    private final InternalAggregation.ReduceContextBuilder aggReduceContextBuilder;
     private final List<SearchResponse> searchResponses = new CopyOnWriteArrayList<>();
 
     SearchResponseMerger(int from, int size, int trackTotalHitsUpTo, SearchTimeProvider searchTimeProvider,
-                         Function<Boolean, ReduceContext> reduceContextFunction) {
+                         InternalAggregation.ReduceContextBuilder aggReduceContextBuilder) {
         this.from = from;
         this.size = size;
         this.trackTotalHitsUpTo = trackTotalHitsUpTo;
         this.searchTimeProvider = Objects.requireNonNull(searchTimeProvider);
-        this.reduceContextFunction = Objects.requireNonNull(reduceContextFunction);
+        this.aggReduceContextBuilder = Objects.requireNonNull(aggReduceContextBuilder);
     }
 
     /**
@@ -196,7 +195,7 @@ SearchResponse getMergedResponse(Clusters clusters) {
         SearchHits mergedSearchHits = topDocsToSearchHits(topDocs, topDocsStats);
         setSuggestShardIndex(shards, groupedSuggestions);
         Suggest suggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions));
-        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, reduceContextFunction.apply(true));
+        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, aggReduceContextBuilder.forFinalReduction());
         ShardSearchFailure[] shardFailures = failures.toArray(ShardSearchFailure.EMPTY_ARRAY);
         SearchProfileShardResults profileShardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
         //make failures ordering consistent between ordinary search and CCS by looking at the shard they come from
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index d7f60de9cffc0..61bbaa0a5097c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -212,9 +212,10 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
                 executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener);
             } else {
                 if (shouldMinimizeRoundtrips(searchRequest)) {
-                    ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider, searchService::createReduceContext,
-                        remoteClusterService, threadPool, listener,
-                        (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l));
+                    ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider,
+                            searchService.aggReduceContextBuilder(searchRequest),
+                            remoteClusterService, threadPool, listener,
+                            (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l));
                 } else {
                     AtomicInteger skippedClusters = new AtomicInteger(0);
                     collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
@@ -260,7 +261,7 @@ static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) {
     }
 
     static void ccsRemoteReduce(SearchRequest searchRequest, OriginalIndices localIndices, Map<String, OriginalIndices> remoteIndices,
-                                SearchTimeProvider timeProvider, Function<Boolean, InternalAggregation.ReduceContext> reduceContext,
+                                SearchTimeProvider timeProvider, InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
                                 RemoteClusterService remoteClusterService, ThreadPool threadPool, ActionListener<SearchResponse> listener,
                                 BiConsumer<SearchRequest, ActionListener<SearchResponse>> localSearchConsumer) {
 
@@ -298,7 +299,8 @@ public void onFailure(Exception e) {
                 }
             });
         } else {
-            SearchResponseMerger searchResponseMerger = createSearchResponseMerger(searchRequest.source(), timeProvider, reduceContext);
+            SearchResponseMerger searchResponseMerger = createSearchResponseMerger(
+                    searchRequest.source(), timeProvider, aggReduceContextBuilder);
             AtomicInteger skippedClusters = new AtomicInteger(0);
             final AtomicReference<Exception> exceptions = new AtomicReference<>();
             int totalClusters = remoteIndices.size() + (localIndices == null ? 0 : 1);
@@ -325,7 +327,7 @@ public void onFailure(Exception e) {
     }
 
     static SearchResponseMerger createSearchResponseMerger(SearchSourceBuilder source, SearchTimeProvider timeProvider,
-                                                           Function<Boolean, InternalAggregation.ReduceContext> reduceContextFunction) {
+                                                           InternalAggregation.ReduceContextBuilder aggReduceContextBuilder) {
         final int from;
         final int size;
         final int trackTotalHitsUpTo;
@@ -342,7 +344,7 @@ static SearchResponseMerger createSearchResponseMerger(SearchSourceBuilder sourc
             source.from(0);
             source.size(from + size);
         }
-        return new SearchResponseMerger(from, size, trackTotalHitsUpTo, timeProvider, reduceContextFunction);
+        return new SearchResponseMerger(from, size, trackTotalHitsUpTo, timeProvider, aggReduceContextBuilder);
     }
 
     static void collectSearchShards(IndicesOptions indicesOptions, String preference, String routing, AtomicInteger skippedClusters,
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index a6fa9e9faa492..224c640c941ff 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -575,7 +575,7 @@ protected Node(final Environment initialEnvironment,
                     b.bind(MetaDataCreateIndexService.class).toInstance(metaDataCreateIndexService);
                     b.bind(SearchService.class).toInstance(searchService);
                     b.bind(SearchTransportService.class).toInstance(searchTransportService);
-                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(searchService::createReduceContext));
+                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(searchService::aggReduceContextBuilder));
                     b.bind(Transport.class).toInstance(transport);
                     b.bind(TransportService.class).toInstance(transportService);
                     b.bind(NetworkService.class).toInstance(networkService);
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index c7fa46bf9a353..451c906500449 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
@@ -69,8 +70,10 @@
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsPhase;
@@ -84,11 +87,11 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhase;
@@ -1200,9 +1203,31 @@ public IndicesService getIndicesService() {
         return indicesService;
     }
 
-    public InternalAggregation.ReduceContext createReduceContext(boolean finalReduce) {
-        return new InternalAggregation.ReduceContext(bigArrays, scriptService,
-            finalReduce ? multiBucketConsumerService.create() : bucketCount -> {}, finalReduce);
+    /**
+     * Returns a builder for {@link InternalAggregation.ReduceContext}. This
+     * builder retains a reference to the provided {@link SearchRequest}.
+     */
+    public InternalAggregation.ReduceContextBuilder aggReduceContextBuilder(SearchRequest request) {
+        return new InternalAggregation.ReduceContextBuilder() {
+            @Override
+            public InternalAggregation.ReduceContext forPartialReduction() {
+                return InternalAggregation.ReduceContext.forPartialReduction(bigArrays, scriptService);
+            }
+
+            @Override
+            public ReduceContext forFinalReduction() {
+                PipelineTree pipelineTree = requestToPipelineTree(request);
+                return InternalAggregation.ReduceContext.forFinalReduction(
+                        bigArrays, scriptService, multiBucketConsumerService.create(), pipelineTree);
+            }
+        };
+    }
+
+    private static PipelineTree requestToPipelineTree(SearchRequest request) {
+        if (request.source() == null || request.source().aggregations() == null) {
+            return PipelineTree.EMPTY;
+        }
+        return request.source().aggregations().buildPipelineTree();
     }
 
     static class SearchRewriteContext {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index 8372d66bd67f7..9cf2ff4414d4c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -26,6 +26,8 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -144,6 +146,14 @@ static AggregationBuilder rewriteAggregation(AggregationBuilder original, QueryR
         return builder;
     }
 
+    /**
+     * Build a tree of {@link PipelineAggregator}s to modify the tree of
+     * aggregation results after the final reduction.
+     */
+    public PipelineTree buildPipelineTree() {
+        return factoriesBuilder.buildPipelineTree();
+    }
+
     /** Common xcontent fields shared among aggregator builders */
     public static final class CommonFields extends ParseField.CommonFields {
         public static final ParseField VALUE_TYPE = new ParseField("value_type");
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 6e3e71a51489d..1998a53ad06b0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.search.aggregations.support.AggregationPath.PathElement;
 import org.elasticsearch.search.internal.SearchContext;
@@ -52,6 +53,9 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
+
 public class AggregatorFactories {
     public static final Pattern VALID_AGG_NAME = Pattern.compile("[^\\[\\]>]+");
 
@@ -232,7 +236,6 @@ public static class Builder implements Writeable, ToXContentObject {
         // ordered nicely, although technically order does not matter
         private final Collection<AggregationBuilder> aggregationBuilders = new LinkedHashSet<>();
         private final Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders = new LinkedHashSet<>();
-        private boolean skipResolveOrder;
 
         /**
          * Create an empty builder.
@@ -295,24 +298,14 @@ public Builder addPipelineAggregator(PipelineAggregationBuilder pipelineAggregat
             return this;
         }
 
-        /**
-         * FOR TESTING ONLY
-         */
-        Builder skipResolveOrder() {
-            this.skipResolveOrder = true;
-            return this;
-        }
-
         public AggregatorFactories build(QueryShardContext queryShardContext, AggregatorFactory parent) throws IOException {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
             List<PipelineAggregationBuilder> orderedpipelineAggregators = null;
-            if (skipResolveOrder) {
-                orderedpipelineAggregators = new ArrayList<>(pipelineAggregatorBuilders);
-            } else {
-                orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregationBuilders,
-                        parent);
+            orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregationBuilders);
+            for (PipelineAggregationBuilder builder : orderedpipelineAggregators) {
+                builder.validate(parent, aggregationBuilders, pipelineAggregatorBuilders);
             }
             AggregatorFactory[] aggFactories = new AggregatorFactory[aggregationBuilders.size()];
 
@@ -325,8 +318,7 @@ public AggregatorFactories build(QueryShardContext queryShardContext, Aggregator
         }
 
         private List<PipelineAggregationBuilder> resolvePipelineAggregatorOrder(
-                Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders, Collection<AggregationBuilder> aggregationBuilders,
-                AggregatorFactory parent) {
+                Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders, Collection<AggregationBuilder> aggregationBuilders) {
             Map<String, PipelineAggregationBuilder> pipelineAggregatorBuildersMap = new HashMap<>();
             for (PipelineAggregationBuilder builder : pipelineAggregatorBuilders) {
                 pipelineAggregatorBuildersMap.put(builder.getName(), builder);
@@ -340,7 +332,6 @@ private List<PipelineAggregationBuilder> resolvePipelineAggregatorOrder(
             Collection<PipelineAggregationBuilder> temporarilyMarked = new HashSet<>();
             while (!unmarkedBuilders.isEmpty()) {
                 PipelineAggregationBuilder builder = unmarkedBuilders.get(0);
-                builder.validate(parent, aggregationBuilders, pipelineAggregatorBuilders);
                 resolvePipelineAggregatorOrder(aggBuildersMap, pipelineAggregatorBuildersMap, orderedPipelineAggregatorrs, unmarkedBuilders,
                         temporarilyMarked, builder);
             }
@@ -494,5 +485,22 @@ public Builder rewrite(QueryRewriteContext context) throws IOException {
                 return this;
             }
         }
+
+        /**
+         * Build a tree of {@link PipelineAggregator}s to modify the tree of
+         * aggregation results after the final reduction.
+         */
+        public PipelineTree buildPipelineTree() {
+            if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
+                return PipelineTree.EMPTY;
+            }
+            Map<String, PipelineTree> subTrees = aggregationBuilders.stream()
+                    .collect(toMap(AggregationBuilder::getName, AggregationBuilder::buildPipelineTree));
+            List<PipelineAggregator> aggregators = resolvePipelineAggregatorOrder(pipelineAggregatorBuilders, aggregationBuilders)
+                    .stream()
+                    .map(PipelineAggregationBuilder::create)
+                    .collect(toList());
+            return new PipelineTree(subTrees, aggregators);
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 77833c98cae0e..0015756d4d933 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 
 import java.io.IOException;
@@ -37,27 +38,54 @@
 import java.util.function.Function;
 import java.util.function.IntConsumer;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations.
  */
 public abstract class InternalAggregation implements Aggregation, NamedWriteable {
-
+    /**
+     * Builds {@link ReduceContext}.
+     */
+    public interface ReduceContextBuilder {
+        /**
+         * Build a {@linkplain ReduceContext} to perform a partial reduction.
+         */
+        ReduceContext forPartialReduction();
+        /**
+         * Build a {@linkplain ReduceContext} to perform the final reduction.
+         */
+        ReduceContext forFinalReduction();
+    }
     public static class ReduceContext {
-
         private final BigArrays bigArrays;
         private final ScriptService scriptService;
         private final IntConsumer multiBucketConsumer;
-        private final boolean isFinalReduce;
+        private final PipelineTree pipelineTreeRoot;
 
-        public ReduceContext(BigArrays bigArrays, ScriptService scriptService, boolean isFinalReduce) {
-            this(bigArrays, scriptService, (s) -> {}, isFinalReduce);
+        /**
+         * Build a {@linkplain ReduceContext} to perform a partial reduction.
+         */
+        public static ReduceContext forPartialReduction(BigArrays bigArrays, ScriptService scriptService) {
+            return new ReduceContext(bigArrays, scriptService, (s) -> {}, null);
+        }
+
+        /**
+         * Build a {@linkplain ReduceContext} to perform the final reduction.
+         * @param pipelineTreeRoot The root of tree of pipeline aggregations for this request
+         */
+        public static ReduceContext forFinalReduction(BigArrays bigArrays, ScriptService scriptService,
+                IntConsumer multiBucketConsumer, PipelineTree pipelineTreeRoot) {
+            return new ReduceContext(bigArrays, scriptService, multiBucketConsumer,
+                    requireNonNull(pipelineTreeRoot, "prefer EMPTY to null"));
         }
 
-        public ReduceContext(BigArrays bigArrays, ScriptService scriptService, IntConsumer multiBucketConsumer, boolean isFinalReduce) {
+        private ReduceContext(BigArrays bigArrays, ScriptService scriptService, IntConsumer multiBucketConsumer,
+                PipelineTree pipelineTreeRoot) {
             this.bigArrays = bigArrays;
             this.scriptService = scriptService;
             this.multiBucketConsumer = multiBucketConsumer;
-            this.isFinalReduce = isFinalReduce;
+            this.pipelineTreeRoot = pipelineTreeRoot;
         }
 
         /**
@@ -66,7 +94,7 @@ public ReduceContext(BigArrays bigArrays, ScriptService scriptService, IntConsum
          * Operations that are potentially losing information can only be applied during the final reduce phase.
          */
         public boolean isFinalReduce() {
-            return isFinalReduce;
+            return pipelineTreeRoot != null;
         }
 
         public BigArrays bigArrays() {
@@ -77,6 +105,13 @@ public ScriptService scriptService() {
             return scriptService;
         }
 
+        /**
+         * The root of the tree of pipeline aggregations for this request.
+         */
+        public PipelineTree pipelineTreeRoot() {
+            return pipelineTreeRoot;
+        }
+
         /**
          * Adds {@code count} buckets to the global count for the request and fails if this number is greater than
          * the maximum number of buckets allowed in a response
@@ -155,9 +190,10 @@ public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations
      * Creates the output from all pipeline aggs that this aggregation is associated with.  Should only
      * be called after all aggregations have been fully reduced
      */
-    public InternalAggregation reducePipelines(InternalAggregation reducedAggs, ReduceContext reduceContext) {
+    public InternalAggregation reducePipelines(
+            InternalAggregation reducedAggs, ReduceContext reduceContext, PipelineTree pipelinesForThisAgg) {
         assert reduceContext.isFinalReduce();
-        for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
+        for (PipelineAggregator pipelineAggregator : pipelinesForThisAgg.aggregators()) {
             reducedAggs = pipelineAggregator.reduce(reducedAggs, reduceContext);
         }
         return reducedAggs;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 07a992d197cd1..f763ececfaf36 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -139,13 +139,12 @@ public static InternalAggregations topLevelReduce(List<InternalAggregations> agg
         if (context.isFinalReduce()) {
             List<InternalAggregation> reducedInternalAggs = reduced.getInternalAggregations();
             reducedInternalAggs = reducedInternalAggs.stream()
-                .map(agg -> agg.reducePipelines(agg, context))
+                .map(agg -> agg.reducePipelines(agg, context, context.pipelineTreeRoot().subTree(agg.getName())))
                 .collect(Collectors.toList());
 
-            List<SiblingPipelineAggregator> topLevelPipelineAggregators = aggregationsList.get(0).getTopLevelPipelineAggregators();
-            for (SiblingPipelineAggregator pipelineAggregator : topLevelPipelineAggregators) {
-                InternalAggregation newAgg
-                    = pipelineAggregator.doReduce(new InternalAggregations(reducedInternalAggs), context);
+            for (PipelineAggregator pipelineAggregator : context.pipelineTreeRoot().aggregators()) {
+                SiblingPipelineAggregator sib = (SiblingPipelineAggregator) pipelineAggregator;
+                InternalAggregation newAgg = sib.doReduce(new InternalAggregations(reducedInternalAggs), context);
                 reducedInternalAggs.add(newAgg);
             }
             return new InternalAggregations(reducedInternalAggs);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index 61176fbf29f82..753648de9af95 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -144,15 +145,15 @@ public static int countInnerBucket(Aggregation agg) {
     }
 
     /**
-     * Unlike {@link InternalAggregation#reducePipelines(InternalAggregation, ReduceContext)}, a multi-bucket
-     * agg needs to first reduce the buckets (and their parent pipelines) before allowing sibling pipelines
-     * to materialize
+     * Amulti-bucket agg needs to first reduce the buckets and *their* pipelines
+     * before allowing sibling pipelines to materialize.
      */
     @Override
-    public final InternalAggregation reducePipelines(InternalAggregation reducedAggs, ReduceContext reduceContext) {
+    public final InternalAggregation reducePipelines(
+            InternalAggregation reducedAggs, ReduceContext reduceContext, PipelineTree pipelineTree) {
         assert reduceContext.isFinalReduce();
-        List<B> materializedBuckets = reducePipelineBuckets(reduceContext);
-        return super.reducePipelines(create(materializedBuckets), reduceContext);
+        List<B> materializedBuckets = reducePipelineBuckets(reduceContext, pipelineTree);
+        return super.reducePipelines(create(materializedBuckets), reduceContext, pipelineTree);
     }
 
     @Override
@@ -172,12 +173,13 @@ public InternalAggregation copyWithRewritenBuckets(Function<InternalAggregations
         return modified ? create(newBuckets) : this;
     }
 
-    private List<B> reducePipelineBuckets(ReduceContext reduceContext) {
+    private List<B> reducePipelineBuckets(ReduceContext reduceContext, PipelineTree pipelineTree) {
         List<B> reducedBuckets = new ArrayList<>();
         for (B bucket : getBuckets()) {
             List<InternalAggregation> aggs = new ArrayList<>();
             for (Aggregation agg : bucket.getAggregations()) {
-                aggs.add(((InternalAggregation)agg).reducePipelines((InternalAggregation)agg, reduceContext));
+                PipelineTree subTree = pipelineTree.subTree(agg.getName());
+                aggs.add(((InternalAggregation)agg).reducePipelines((InternalAggregation)agg, reduceContext, subTree));
             }
             reducedBuckets.add(createBucket(new InternalAggregations(aggs), bucket));
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
index 437cd1466d688..5df2d1d03ce1d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 
 import java.io.IOException;
@@ -113,19 +114,20 @@ public InternalAggregation reduce(List<InternalAggregation> aggregations, Reduce
     }
 
     /**
-     * Unlike {@link InternalAggregation#reducePipelines(InternalAggregation, ReduceContext)}, a single-bucket
-     * agg needs to first reduce the aggs in it's bucket (and their parent pipelines) before allowing sibling pipelines
-     * to reduce
+     * Amulti-bucket agg needs to first reduce the buckets and *their* pipelines
+     * before allowing sibling pipelines to materialize.
      */
     @Override
-    public final InternalAggregation reducePipelines(InternalAggregation reducedAggs, ReduceContext reduceContext) {
+    public final InternalAggregation reducePipelines(
+            InternalAggregation reducedAggs, ReduceContext reduceContext, PipelineTree pipelineTree) {
         assert reduceContext.isFinalReduce();
         List<InternalAggregation> aggs = new ArrayList<>();
         for (Aggregation agg : getAggregations().asList()) {
-            aggs.add(((InternalAggregation)agg).reducePipelines((InternalAggregation)agg, reduceContext));
+            PipelineTree subTree = pipelineTree.subTree(agg.getName());
+            aggs.add(((InternalAggregation)agg).reducePipelines((InternalAggregation)agg, reduceContext, subTree));
         }
         InternalAggregations reducedSubAggs = new InternalAggregations(aggs);
-        return super.reducePipelines(create(reducedSubAggs), reduceContext);
+        return super.reducePipelines(create(reducedSubAggs), reduceContext, pipelineTree);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
index de5a8ac998a29..e12fabbdd3645 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
@@ -30,8 +30,12 @@
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+
 public abstract class PipelineAggregator implements NamedWriteable {
     /**
      * Parse the {@link PipelineAggregationBuilder} from a {@link XContentParser}.
@@ -57,6 +61,46 @@ PipelineAggregationBuilder parse(String pipelineAggregatorName, XContentParser p
                 throws IOException;
     }
 
+    /**
+     * Tree of {@link PipelineAggregator}s to modify a tree of aggregations
+     * after their final reduction.
+     */
+    public static class PipelineTree {
+        /**
+         * An empty tree of {@link PipelineAggregator}s.
+         */
+        public static final PipelineTree EMPTY = new PipelineTree(emptyMap(), emptyList());
+
+        private final Map<String, PipelineTree> subTrees;
+        private final List<PipelineAggregator> aggregators;
+
+        public PipelineTree(Map<String, PipelineTree> subTrees, List<PipelineAggregator> aggregators) {
+            this.subTrees = subTrees;
+            this.aggregators = aggregators;
+        }
+
+        /**
+         * The {@link PipelineAggregator}s for the aggregation at this
+         * position in the tree.
+         */
+        public List<PipelineAggregator> aggregators() {
+            return aggregators;
+        }
+
+        /**
+         * Get the sub-tree at for the named sub-aggregation or {@link #EMPTY}
+         * if there are no pipeline aggragations for that sub-aggregator.
+         */
+        public PipelineTree subTree(String name) {
+            return subTrees.getOrDefault(name, EMPTY);
+        }
+
+        @Override
+        public String toString() {
+            return "PipelineTree[" + aggregators + "," + subTrees + "]";
+        }
+    }
+
     private String name;
     private String[] bucketsPaths;
     private Map<String, Object> metaData;
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index a2496c19d2a9f..a09a81aa54349 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -27,18 +27,17 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.transport.Transport;
 
 import java.io.IOException;
@@ -63,8 +62,6 @@ public void testDfsWith2Shards() throws IOException {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
@@ -92,7 +89,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
         };
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         mockSearchPhaseContext.searchTransport = searchTransportService;
-        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+        DfsQueryPhase phase = new DfsQueryPhase(results, searchPhaseController(),
             (response) -> new SearchPhase("test") {
             @Override
             public void run() throws IOException {
@@ -125,8 +122,6 @@ public void testDfsWith1ShardFailed() throws IOException {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
@@ -148,7 +143,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
         };
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         mockSearchPhaseContext.searchTransport = searchTransportService;
-        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+        DfsQueryPhase phase = new DfsQueryPhase(results, searchPhaseController(),
             (response) -> new SearchPhase("test") {
                 @Override
                 public void run() throws IOException {
@@ -184,8 +179,6 @@ public void testFailPhaseOnException() throws IOException {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
@@ -207,7 +200,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
         };
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         mockSearchPhaseContext.searchTransport = searchTransportService;
-        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+        DfsQueryPhase phase = new DfsQueryPhase(results, searchPhaseController(),
             (response) -> new SearchPhase("test") {
                 @Override
                 public void run() throws IOException {
@@ -219,4 +212,7 @@ public void run() throws IOException {
         assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty()); // phase execution will clean up on the contexts
     }
 
+    private SearchPhaseController searchPhaseController() {
+        return new SearchPhaseController(request -> InternalAggregationTestCase.emptyReduceContextBuilder());
+    }
  }
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 8978f43842cfe..df9d810302ff1 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -26,20 +26,19 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.transport.Transport;
 
 import java.util.concurrent.CountDownLatch;
@@ -50,8 +49,7 @@
 public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testShortcutQueryAndFetchOptimization() {
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
+        SearchPhaseController controller = new SearchPhaseController(s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 1);
         boolean hasHits = randomBoolean();
@@ -94,8 +92,7 @@ public void run() {
 
     public void testFetchTwoDocument() {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
+        SearchPhaseController controller = new SearchPhaseController(s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
         final SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
@@ -154,8 +151,7 @@ public void run() {
 
     public void testFailFetchOneDoc() {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
+        SearchPhaseController controller = new SearchPhaseController(s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
@@ -218,8 +214,7 @@ public void testFetchDocsConcurrently() throws InterruptedException {
         int resultSetSize = randomIntBetween(0, 100);
         // we use at least 2 hits otherwise this is subject to single shard optimization and we trip an assert...
         int numHits = randomIntBetween(2, 100); // also numshards --> 1 hit per shard
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
+        SearchPhaseController controller = new SearchPhaseController(s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(numHits);
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP,
             mockSearchPhaseContext.getRequest(), numHits);
@@ -276,8 +271,7 @@ public void run() {
 
     public void testExceptionFailsPhase() {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
+        SearchPhaseController controller = new SearchPhaseController(s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
@@ -333,8 +327,7 @@ public void run() {
 
     public void testCleanupIrrelevantContexts() { // contexts that are not fetched should be cleaned up
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(
-            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
+        SearchPhaseController controller = new SearchPhaseController(s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = 1;
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 5e65487fe5927..7388d44e6b785 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.search;
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
+
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.SortField;
@@ -45,8 +46,10 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.metrics.InternalMax;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.internal.InternalSearchResponse;
@@ -59,6 +62,7 @@
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.junit.Before;
 
 import java.util.ArrayList;
@@ -89,11 +93,19 @@ public class SearchPhaseControllerTests extends ESTestCase {
     @Before
     public void setup() {
         reductions = new CopyOnWriteArrayList<>();
-        searchPhaseController = new SearchPhaseController(
-            (finalReduce) -> {
-                reductions.add(finalReduce);
-                return new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, finalReduce);
-            });
+        searchPhaseController = new SearchPhaseController(s -> new InternalAggregation.ReduceContextBuilder() {
+            @Override
+            public ReduceContext forPartialReduction() {
+                reductions.add(false);
+                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null);
+            }
+
+            public ReduceContext forFinalReduction() {
+                reductions.add(true);
+                return InternalAggregation.ReduceContext.forFinalReduction(
+                        BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineTree.EMPTY);
+            };
+        });
     }
 
     public void testSortDocs() {
@@ -176,8 +188,8 @@ public void testMerge() {
         int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2);
         AtomicArray<SearchPhaseResult> queryResults = generateQueryResults(nShards, suggestions, queryResultSize, false);
         for (int trackTotalHits : new int[] {SearchContext.TRACK_TOTAL_HITS_DISABLED, SearchContext.TRACK_TOTAL_HITS_ACCURATE}) {
-            SearchPhaseController.ReducedQueryPhase reducedQueryPhase =
-                searchPhaseController.reducedQueryPhase(queryResults.asList(), false, trackTotalHits, true);
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = searchPhaseController.reducedQueryPhase(
+                    queryResults.asList(), false, trackTotalHits, InternalAggregationTestCase.emptyReduceContextBuilder(), true);
             AtomicArray<SearchPhaseResult> fetchResults = generateFetchResults(nShards,
                 reducedQueryPhase.sortedTopDocs.scoreDocs, reducedQueryPhase.suggest);
             InternalSearchResponse mergedResponse = searchPhaseController.merge(false,
@@ -221,8 +233,8 @@ public void testMerge() {
      * Generate random query results received from the provided number of shards, including the provided
      * number of search hits and randomly generated completion suggestions based on the name and size of the provided ones.
      * Note that <code>shardIndex</code> is already set to the generated completion suggestions to simulate what
-     * {@link SearchPhaseController#reducedQueryPhase(Collection, boolean, int, boolean)} does, meaning that the returned query results
-     * can be fed directly to
+     * {@link SearchPhaseController#reducedQueryPhase(Collection, boolean, int, InternalAggregation.ReduceContextBuilder, boolean)} does,
+     * meaning that the returned query results can be fed directly to
      * {@link SearchPhaseController#sortDocs(boolean, Collection, Collection, SearchPhaseController.TopDocsStats, int, int, List)}
      */
     private static AtomicArray<SearchPhaseResult> generateQueryResults(int nShards, List<CompletionSuggestion> suggestions,
@@ -431,7 +443,7 @@ private void consumerTestCase(int numEmptyResponses) {
         SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
         assertEquals(numTotalReducePhases, reduce.numReducePhases);
         assertEquals(numTotalReducePhases, reductions.size());
-        assertFinalReduction(request);
+        assertAggReduction(request);
         InternalMax max = (InternalMax) reduce.aggregations.asList().get(0);
         assertEquals(3.0D, max.getValue(), 0.0D);
         assertFalse(reduce.sortedTopDocs.isSortedByField);
@@ -475,7 +487,7 @@ public void testConsumerConcurrently() throws InterruptedException {
             threads[i].join();
         }
         SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
-        assertFinalReduction(request);
+        assertAggReduction(request);
         InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
         assertEquals(max.get(), internalMax.getValue(), 0.0D);
         assertEquals(1, reduce.sortedTopDocs.scoreDocs.length);
@@ -512,7 +524,7 @@ public void testConsumerOnlyAggs() {
             consumer.consumeResult(result);
         }
         SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
-        assertFinalReduction(request);
+        assertAggReduction(request);
         InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
         assertEquals(max.get(), internalMax.getValue(), 0.0D);
         assertEquals(0, reduce.sortedTopDocs.scoreDocs.length);
@@ -547,7 +559,7 @@ public void testConsumerOnlyHits() {
             consumer.consumeResult(result);
         }
         SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
-        assertFinalReduction(request);
+        assertAggReduction(request);
         assertEquals(1, reduce.sortedTopDocs.scoreDocs.length);
         assertEquals(max.get(), reduce.maxScore, 0.0f);
         assertEquals(expectedNumResults, reduce.totalHits.value);
@@ -558,9 +570,15 @@ public void testConsumerOnlyHits() {
         assertNull(reduce.sortedTopDocs.collapseValues);
     }
 
-    private void assertFinalReduction(SearchRequest searchRequest) {
-        assertThat(reductions.size(), greaterThanOrEqualTo(1));
-        assertEquals(searchRequest.isFinalReduce(), reductions.get(reductions.size() - 1));
+    private void assertAggReduction(SearchRequest searchRequest) {
+        if (searchRequest.source() == null || searchRequest.source().aggregations() == null ||
+                searchRequest.source().aggregations().getAggregatorFactories().isEmpty()) {
+            // When there aren't any aggregations we don't perform any aggregation reductions.
+            assertThat(reductions.size(), equalTo(0));
+        } else {
+            assertThat(reductions.size(), greaterThanOrEqualTo(1));
+            assertEquals(searchRequest.isFinalReduce(), reductions.get(reductions.size() - 1));
+        }
     }
 
     public void testNewSearchPhaseResults() {
@@ -655,7 +673,7 @@ public void testConsumerSortByField() {
             consumer.consumeResult(result);
         }
         SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
-        assertFinalReduction(request);
+        assertAggReduction(request);
         assertEquals(Math.min(expectedNumResults, size), reduce.sortedTopDocs.scoreDocs.length);
         assertEquals(expectedNumResults, reduce.totalHits.value);
         assertEquals(max.get(), ((FieldDoc)reduce.sortedTopDocs.scoreDocs[0]).fields[0]);
@@ -693,7 +711,7 @@ public void testConsumerFieldCollapsing() {
             consumer.consumeResult(result);
         }
         SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
-        assertFinalReduction(request);
+        assertAggReduction(request);
         assertEquals(3, reduce.sortedTopDocs.scoreDocs.length);
         assertEquals(expectedNumResults, reduce.totalHits.value);
         assertEquals(a, ((FieldDoc)reduce.sortedTopDocs.scoreDocs[0]).fields[0]);
@@ -787,7 +805,7 @@ public void testConsumerSuggestions() {
             CompletionSuggestion.Entry.Option option = completion.getOptions().get(0);
             assertEquals(maxScoreCompletion, option.getScore(), 0f);
         }
-        assertFinalReduction(request);
+        assertAggReduction(request);
         assertEquals(1, reduce.sortedTopDocs.scoreDocs.length);
         assertEquals(maxScoreCompletion, reduce.sortedTopDocs.scoreDocs[0].score, 0f);
         assertEquals(0, reduce.sortedTopDocs.scoreDocs[0].doc);
@@ -862,7 +880,7 @@ public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, Interna
                 threads[i].join();
             }
             SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
-            assertFinalReduction(request);
+            assertAggReduction(request);
             InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
             assertEquals(max.get(), internalMax.getValue(), 0.0D);
             assertEquals(1, reduce.sortedTopDocs.scoreDocs.length);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
index ab31c44ff2d59..df04549ddee88 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
@@ -64,6 +63,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.test.InternalAggregationTestCase.emptyReduceContextBuilder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -97,7 +97,7 @@ public void testMergeTookInMillis() throws InterruptedException {
         long currentRelativeTime = randomLong();
         SearchTimeProvider timeProvider = new SearchTimeProvider(randomLong(), 0, () -> currentRelativeTime);
         SearchResponseMerger merger = new SearchResponseMerger(randomIntBetween(0, 1000), randomIntBetween(0, 10000),
-            SearchContext.TRACK_TOTAL_HITS_ACCURATE, timeProvider, flag -> null);
+            SearchContext.TRACK_TOTAL_HITS_ACCURATE, timeProvider, emptyReduceContextBuilder());
         for (int i = 0; i < numResponses; i++) {
             SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null, 1, 1, 0, randomLong(),
                 ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters());
@@ -111,7 +111,7 @@ public void testMergeTookInMillis() throws InterruptedException {
     public void testMergeShardFailures() throws InterruptedException {
         SearchTimeProvider searchTimeProvider = new SearchTimeProvider(0, 0, () -> 0);
         SearchResponseMerger merger = new SearchResponseMerger(0, 0, SearchContext.TRACK_TOTAL_HITS_ACCURATE,
-            searchTimeProvider, flag -> null);
+            searchTimeProvider, emptyReduceContextBuilder());
         PriorityQueue<Tuple<SearchShardTarget, ShardSearchFailure>> priorityQueue = new PriorityQueue<>(Comparator.comparing(Tuple::v1,
             (o1, o2) -> {
                 int compareTo = o1.getShardId().compareTo(o2.getShardId());
@@ -159,7 +159,7 @@ public void testMergeShardFailures() throws InterruptedException {
     public void testMergeShardFailuresNullShardTarget() throws InterruptedException {
         SearchTimeProvider searchTimeProvider = new SearchTimeProvider(0, 0, () -> 0);
         SearchResponseMerger merger = new SearchResponseMerger(0, 0, SearchContext.TRACK_TOTAL_HITS_ACCURATE,
-            searchTimeProvider, flag -> null);
+            searchTimeProvider, emptyReduceContextBuilder());
         PriorityQueue<Tuple<ShardId, ShardSearchFailure>> priorityQueue = new PriorityQueue<>(Comparator.comparing(Tuple::v1));
         for (int i = 0; i < numResponses; i++) {
             int numFailures = randomIntBetween(1, 10);
@@ -197,7 +197,7 @@ public void testMergeShardFailuresNullShardTarget() throws InterruptedException
     public void testMergeShardFailuresNullShardId() throws InterruptedException {
         SearchTimeProvider searchTimeProvider = new SearchTimeProvider(0, 0, () -> 0);
         SearchResponseMerger merger = new SearchResponseMerger(0, 0, SearchContext.TRACK_TOTAL_HITS_ACCURATE,
-            searchTimeProvider, flag -> null);
+            searchTimeProvider, emptyReduceContextBuilder());
         List<ShardSearchFailure> expectedFailures = new ArrayList<>();
         for (int i = 0; i < numResponses; i++) {
             int numFailures = randomIntBetween(1, 50);
@@ -220,7 +220,7 @@ public void testMergeShardFailuresNullShardId() throws InterruptedException {
     public void testMergeProfileResults() throws InterruptedException {
         SearchTimeProvider searchTimeProvider = new SearchTimeProvider(0, 0, () -> 0);
         SearchResponseMerger merger = new SearchResponseMerger(0, 0, SearchContext.TRACK_TOTAL_HITS_ACCURATE,
-            searchTimeProvider, flag -> null);
+            searchTimeProvider, emptyReduceContextBuilder());
         Map<String, ProfileShardResult> expectedProfile = new HashMap<>();
         for (int i = 0; i < numResponses; i++) {
             SearchProfileShardResults profile = SearchProfileShardResultsTests.createTestItem();
@@ -247,7 +247,8 @@ public void testMergeProfileResults() throws InterruptedException {
     public void testMergeCompletionSuggestions()  throws InterruptedException {
         String suggestionName = randomAlphaOfLengthBetween(4, 8);
         int size = randomIntBetween(1, 100);
-        SearchResponseMerger searchResponseMerger = new SearchResponseMerger(0, 0, 0, new SearchTimeProvider(0, 0, () -> 0), flag -> null);
+        SearchResponseMerger searchResponseMerger = new SearchResponseMerger(0, 0, 0, new SearchTimeProvider(0, 0, () -> 0),
+                emptyReduceContextBuilder());
         for (int i = 0; i < numResponses; i++) {
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
@@ -296,7 +297,8 @@ public void testMergeCompletionSuggestions()  throws InterruptedException {
     public void testMergeCompletionSuggestionsTieBreak()  throws InterruptedException {
         String suggestionName = randomAlphaOfLengthBetween(4, 8);
         int size = randomIntBetween(1, 100);
-        SearchResponseMerger searchResponseMerger = new SearchResponseMerger(0, 0, 0, new SearchTimeProvider(0, 0, () -> 0), flag -> null);
+        SearchResponseMerger searchResponseMerger = new SearchResponseMerger(0, 0, 0, new SearchTimeProvider(0, 0, () -> 0),
+                emptyReduceContextBuilder());
         for (int i = 0; i < numResponses; i++) {
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
@@ -351,7 +353,7 @@ public void testMergeCompletionSuggestionsTieBreak()  throws InterruptedExceptio
 
     public void testMergeAggs() throws InterruptedException {
         SearchResponseMerger searchResponseMerger = new SearchResponseMerger(0, 0, 0, new SearchTimeProvider(0, 0, () -> 0),
-            flag -> new InternalAggregation.ReduceContext(null, null, flag));
+                emptyReduceContextBuilder());
         String maxAggName = randomAlphaOfLengthBetween(5, 8);
         String rangeAggName = randomAlphaOfLengthBetween(5, 8);
         int totalCount = 0;
@@ -429,7 +431,8 @@ public void testMergeSearchHits() throws InterruptedException {
         TotalHits.Relation totalHitsRelation = randomTrackTotalHits.v2();
 
         PriorityQueue<SearchHit> priorityQueue = new PriorityQueue<>(new SearchHitComparator(sortFields));
-        SearchResponseMerger searchResponseMerger = new SearchResponseMerger(from, size, trackTotalHitsUpTo, timeProvider, flag -> null);
+        SearchResponseMerger searchResponseMerger = new SearchResponseMerger(
+                from, size, trackTotalHitsUpTo, timeProvider, emptyReduceContextBuilder());
 
         TotalHits expectedTotalHits = null;
         int expectedTotal = 0;
@@ -556,7 +559,7 @@ public void testMergeSearchHits() throws InterruptedException {
     public void testMergeNoResponsesAdded() {
         long currentRelativeTime = randomLong();
         final SearchTimeProvider timeProvider = new SearchTimeProvider(randomLong(), 0, () -> currentRelativeTime);
-        SearchResponseMerger merger = new SearchResponseMerger(0, 10, Integer.MAX_VALUE, timeProvider, flag -> null);
+        SearchResponseMerger merger = new SearchResponseMerger(0, 10, Integer.MAX_VALUE, timeProvider, emptyReduceContextBuilder());
         SearchResponse.Clusters clusters = SearchResponseTests.randomClusters();
         assertEquals(0, merger.numResponses());
         SearchResponse response = merger.getMergedResponse(clusters);
@@ -583,7 +586,7 @@ public void testMergeNoResponsesAdded() {
     public void testMergeEmptySearchHitsWithNonEmpty() {
         long currentRelativeTime = randomLong();
         final SearchTimeProvider timeProvider = new SearchTimeProvider(randomLong(), 0, () -> currentRelativeTime);
-        SearchResponseMerger merger = new SearchResponseMerger(0, 10, Integer.MAX_VALUE, timeProvider, flag -> null);
+        SearchResponseMerger merger = new SearchResponseMerger(0, 10, Integer.MAX_VALUE, timeProvider, emptyReduceContextBuilder());
         SearchResponse.Clusters clusters = SearchResponseTests.randomClusters();
         int numFields = randomIntBetween(1, 3);
         SortField[] sortFields = new SortField[numFields];
@@ -626,7 +629,7 @@ public void testMergeOnlyEmptyHits() {
         Tuple<Integer, TotalHits.Relation> randomTrackTotalHits = randomTrackTotalHits();
         int trackTotalHitsUpTo = randomTrackTotalHits.v1();
         TotalHits.Relation totalHitsRelation = randomTrackTotalHits.v2();
-        SearchResponseMerger merger = new SearchResponseMerger(0, 10, trackTotalHitsUpTo, timeProvider, flag -> null);
+        SearchResponseMerger merger = new SearchResponseMerger(0, 10, trackTotalHitsUpTo, timeProvider, emptyReduceContextBuilder());
         int numResponses = randomIntBetween(1, 5);
         TotalHits expectedTotalHits = null;
         for (int i = 0; i < numResponses; i++) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 105bcc66ac1c3..32bca4d0dc18c 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -52,6 +52,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
@@ -89,6 +90,8 @@
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -385,8 +388,8 @@ public void testCCSRemoteReduceMergeFails() throws Exception {
             AtomicReference<Exception> failure = new AtomicReference<>();
             LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                 ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
-            TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext,
-                remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+            TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
+                    aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
             if (localIndices == null) {
                 assertNull(setOnce.get());
             } else {
@@ -419,8 +422,6 @@ public void testCCSRemoteReduce() throws Exception {
         OriginalIndices localIndices = local ? new OriginalIndices(new String[]{"index"}, SearchRequest.DEFAULT_INDICES_OPTIONS) : null;
         int totalClusters = numClusters + (local ? 1 : 0);
         TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0);
-        Function<Boolean, InternalAggregation.ReduceContext> reduceContext =
-            finalReduce -> new InternalAggregation.ReduceContext(null, null, finalReduce);
         try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
             service.start();
             service.acceptIncomingRequests();
@@ -432,8 +433,8 @@ public void testCCSRemoteReduce() throws Exception {
                 AtomicReference<SearchResponse> response = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext,
-                    remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
+                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -458,8 +459,8 @@ public void testCCSRemoteReduce() throws Exception {
                 AtomicReference<Exception> failure = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext,
-                    remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
+                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -505,8 +506,8 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 AtomicReference<Exception> failure = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext,
-                    remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
+                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -534,8 +535,8 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 AtomicReference<SearchResponse> response = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext,
-                    remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, 
+                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -574,8 +575,8 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 AtomicReference<SearchResponse> response = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext,
-                    remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
+                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -751,13 +752,12 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
 
     public void testCreateSearchResponseMerger() {
         TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0);
-        Function<Boolean, InternalAggregation.ReduceContext> reduceContext = flag -> null;
         {
             SearchSourceBuilder source = new SearchSourceBuilder();
             assertEquals(-1, source.size());
             assertEquals(-1, source.from());
             assertNull(source.trackTotalHitsUpTo());
-            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, reduceContext);
+            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, aggReduceContextBuilder());
             assertEquals(0, merger.from);
             assertEquals(10, merger.size);
             assertEquals(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO, merger.trackTotalHitsUpTo);
@@ -766,7 +766,7 @@ public void testCreateSearchResponseMerger() {
             assertNull(source.trackTotalHitsUpTo());
         }
         {
-            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(null, timeProvider, reduceContext);
+            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(null, timeProvider, aggReduceContextBuilder());
             assertEquals(0, merger.from);
             assertEquals(10, merger.size);
             assertEquals(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO, merger.trackTotalHitsUpTo);
@@ -779,7 +779,7 @@ public void testCreateSearchResponseMerger() {
             source.size(originalSize);
             int trackTotalHitsUpTo = randomIntBetween(0, Integer.MAX_VALUE);
             source.trackTotalHitsUpTo(trackTotalHitsUpTo);
-            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, reduceContext);
+            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, aggReduceContextBuilder());
             assertEquals(0, source.from());
             assertEquals(originalFrom + originalSize, source.size());
             assertEquals(trackTotalHitsUpTo, (int)source.trackTotalHitsUpTo());
@@ -837,4 +837,18 @@ public void testShouldMinimizeRoundtrips() throws Exception {
             assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest));
         }
     }
+
+    private InternalAggregation.ReduceContextBuilder aggReduceContextBuilder() {
+        return new InternalAggregation.ReduceContextBuilder() {
+            @Override
+            public InternalAggregation.ReduceContext forPartialReduction() {
+                return InternalAggregation.ReduceContext.forPartialReduction(null, null);
+            };
+
+            @Override
+            public InternalAggregation.ReduceContext forFinalReduction() {
+                return InternalAggregation.ReduceContext.forFinalReduction(null, null, b -> {}, new PipelineTree(emptyMap(), emptyList()));
+            }
+        };
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 7607d519d1d8c..e1e6003695d3a 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -777,14 +777,15 @@ public void testExpandSearchThrottled() {
     }
 
     public void testCreateReduceContext() {
-        final SearchService service = getInstanceFromNode(SearchService.class);
+        SearchService service = getInstanceFromNode(SearchService.class);
+        InternalAggregation.ReduceContextBuilder reduceContextBuilder = service.aggReduceContextBuilder(new SearchRequest());
         {
-            InternalAggregation.ReduceContext reduceContext = service.createReduceContext(true);
+            InternalAggregation.ReduceContext reduceContext = reduceContextBuilder.forFinalReduction();
             expectThrows(MultiBucketConsumerService.TooManyBucketsException.class,
                 () -> reduceContext.consumeBucketsAndMaybeBreak(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS + 1));
         }
         {
-            InternalAggregation.ReduceContext reduceContext = service.createReduceContext(false);
+            InternalAggregation.ReduceContext reduceContext = reduceContextBuilder.forPartialReduction();
             reduceContext.consumeBucketsAndMaybeBreak(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS + 1);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
index e5b45d094e620..cff1e7c874b5c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
@@ -36,15 +36,19 @@
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Random;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -228,6 +232,16 @@ public void testRewrite() throws Exception {
         assertSame(rewritten, secondRewritten);
     }
 
+    public void testBuildPipelineTreeResolvesPipelineOrder() {
+        AggregatorFactories.Builder builder = new AggregatorFactories.Builder();
+        builder.addPipelineAggregator(PipelineAggregatorBuilders.avgBucket("bar", "foo"));
+        builder.addPipelineAggregator(PipelineAggregatorBuilders.avgBucket("foo", "real"));
+        builder.addAggregator(AggregationBuilders.avg("real").field("target"));
+        PipelineTree tree = builder.buildPipelineTree();
+        assertThat(tree.aggregators().stream().map(PipelineAggregator::name).collect(toList()),
+                equalTo(List.of("foo", "bar")));
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         return xContentRegistry;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index ccbb4b9a97ba2..1d59ba476986d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -96,7 +96,7 @@ public void setUp() throws Exception {
      */
     public void testFromXContent() throws IOException {
         AF testAgg = createTestAggregatorFactory();
-        AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().skipResolveOrder().addPipelineAggregator(testAgg);
+        AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().addPipelineAggregator(testAgg);
         logger.info("Content string: {}", factoriesBuilder);
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         if (randomBoolean()) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
index 7236b354ef92b..0fe02c5e89fd9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests;
@@ -32,9 +33,11 @@
 import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValueTests;
 import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
@@ -42,6 +45,10 @@
 import java.util.Collections;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+
 public class InternalAggregationsTests extends ESTestCase {
 
     private final NamedWriteableRegistry registry = new NamedWriteableRegistry(
@@ -49,7 +56,8 @@ public class InternalAggregationsTests extends ESTestCase {
 
     public void testReduceEmptyAggs() {
         List<InternalAggregations> aggs = Collections.emptyList();
-        InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(null, null, randomBoolean());
+        InternalAggregation.ReduceContextBuilder builder = InternalAggregationTestCase.emptyReduceContextBuilder();
+        InternalAggregation.ReduceContext reduceContext = randomBoolean() ? builder.forFinalReduction() : builder.forPartialReduction();
         assertNull(InternalAggregations.reduce(aggs, reduceContext));
     }
 
@@ -61,7 +69,7 @@ public void testNonFinalReduceTopLevelPipelineAggs()  {
         topLevelPipelineAggs.add((SiblingPipelineAggregator)maxBucketPipelineAggregationBuilder.create());
         List<InternalAggregations> aggs = Collections.singletonList(new InternalAggregations(Collections.singletonList(terms),
             topLevelPipelineAggs));
-        InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(null, null, false);
+        InternalAggregation.ReduceContext reduceContext = InternalAggregationTestCase.emptyReduceContextBuilder().forPartialReduction();
         InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, reduceContext);
         assertEquals(1, reducedAggs.getTopLevelPipelineAggregators().size());
         assertEquals(1, reducedAggs.aggregations.size());
@@ -73,17 +81,10 @@ public void testFinalReduceTopLevelPipelineAggs()  {
 
         MaxBucketPipelineAggregationBuilder maxBucketPipelineAggregationBuilder = new MaxBucketPipelineAggregationBuilder("test", "test");
         SiblingPipelineAggregator siblingPipelineAggregator = (SiblingPipelineAggregator) maxBucketPipelineAggregationBuilder.create();
-        InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(null, null, true);
-        final InternalAggregations reducedAggs;
-        if (randomBoolean()) {
-            InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms),
-                Collections.singletonList(siblingPipelineAggregator));
-            reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(aggs), reduceContext);
-        } else {
-            InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms),
-                Collections.singletonList(siblingPipelineAggregator));
-            reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(aggs), reduceContext);
-        }
+        InternalAggregation.ReduceContext reduceContext = InternalAggregation.ReduceContext.forFinalReduction(
+                BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, new PipelineTree(emptyMap(), singletonList(siblingPipelineAggregator)));
+        InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms), emptyList());
+        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(aggs), reduceContext);
         assertEquals(0, reducedAggs.getTopLevelPipelineAggregators().size());
         assertEquals(2, reducedAggs.aggregations.size());
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java
index 67ac4d9163dcb..980be089e96e3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -241,7 +240,7 @@ public void testReduceSame() throws IOException {
         for (int i = 0; i < numSame; i++) {
             toReduce.add(result);
         }
-        InternalComposite finalReduce = (InternalComposite) result.reduce(toReduce, reduceContext());
+        InternalComposite finalReduce = (InternalComposite) result.reduce(toReduce, emptyReduceContextBuilder().forFinalReduction());
         assertThat(finalReduce.getBuckets().size(), equalTo(result.getBuckets().size()));
         Iterator<InternalComposite.InternalBucket> expectedIt = result.getBuckets().iterator();
         for (InternalComposite.InternalBucket bucket : finalReduce.getBuckets()) {
@@ -261,7 +260,7 @@ public void testReduceUnmapped() throws IOException {
                 rawFormats, emptyList(), null, reverseMuls, true, emptyList(), emptyMap());
         List<InternalAggregation> toReduce = Arrays.asList(unmapped, mapped);
         Collections.shuffle(toReduce, random());
-        InternalComposite finalReduce = (InternalComposite) unmapped.reduce(toReduce, reduceContext());
+        InternalComposite finalReduce = (InternalComposite) unmapped.reduce(toReduce, emptyReduceContextBuilder().forFinalReduction());
         assertThat(finalReduce.getBuckets().size(), equalTo(mapped.getBuckets().size()));
         if (false == mapped.getBuckets().isEmpty()) {
             assertThat(finalReduce.getFormats(), equalTo(mapped.getFormats()));
@@ -407,8 +406,4 @@ private InternalComposite.ArrayMap createMap(List<String> fields, Comparable[] v
             values
         );
     }
-
-    private InternalAggregation.ReduceContext reduceContext() {
-        return new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, true);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java
index 5b8264b9e71ea..82c63aca7a446 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogramTests.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.BucketOrder;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.test.InternalMultiBucketAggregationTestCase;
 
 import java.util.ArrayList;
@@ -109,7 +109,8 @@ public void testHandlesNaN() {
         newBuckets.add(new InternalHistogram.Bucket(Double.NaN, b.docCount, keyed, b.format, b.aggregations));
         
         InternalHistogram newHistogram = histogram.create(newBuckets);
-        newHistogram.reduce(Arrays.asList(newHistogram, histogram2), new InternalAggregation.ReduceContext(null, null, false));
+        newHistogram.reduce(Arrays.asList(newHistogram, histogram2),
+                InternalAggregationTestCase.emptyReduceContextBuilder().forPartialReduction());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index 510d06579205c..222b3f21cb2c2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.test.TestSearchContext;
 
 import java.io.ByteArrayInputStream;
@@ -147,7 +148,7 @@ public static SignificanceHeuristic getRandomSignificanceheuristic() {
 
     public void testReduce() {
         List<InternalAggregation> aggs = createInternalAggregations();
-        InternalAggregation.ReduceContext context = new InternalAggregation.ReduceContext(null, null, true);
+        InternalAggregation.ReduceContext context = InternalAggregationTestCase.emptyReduceContextBuilder().forFinalReduction();
         SignificantTerms reducedAgg = (SignificantTerms) aggs.get(0).reduce(aggs, context);
         assertThat(reducedAgg.getBuckets().size(), equalTo(2));
         assertThat(reducedAgg.getBuckets().get(0).getSubsetDf(), equalTo(8L));
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index b50f76f1de59b..514f7549e757d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -79,6 +79,7 @@
 import org.elasticsearch.search.aggregations.metrics.InternalTopHits;
 import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -1084,9 +1085,9 @@ public void testMixLongAndDouble() throws Exception {
                 }
                 dir.close();
             }
-            InternalAggregation.ReduceContext ctx =
-                new InternalAggregation.ReduceContext(new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY),
-                    new NoneCircuitBreakerService()), null, true);
+            InternalAggregation.ReduceContext ctx = InternalAggregation.ReduceContext.forFinalReduction(
+                    new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()),
+                    null, b -> {}, PipelineTree.EMPTY);
             for (InternalAggregation internalAgg : aggs) {
                 InternalAggregation mergedAggs = internalAgg.reduce(aggs, ctx);
                 assertTrue(mergedAggs instanceof DoubleTerms);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 7c7e1498c5b34..062e64ba55b74 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1358,7 +1358,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi
                     bigArrays, new FetchPhase(Collections.emptyList()), responseCollectorService, new NoneCircuitBreakerService());
                 actions.put(SearchAction.INSTANCE,
                     new TransportSearchAction(threadPool, transportService, searchService,
-                        searchTransportService, new SearchPhaseController(searchService::createReduceContext), clusterService,
+                        searchTransportService, new SearchPhaseController(searchService::aggReduceContextBuilder), clusterService,
                         actionFilters, indexNameExpressionResolver));
                 actions.put(RestoreSnapshotAction.INSTANCE,
                     new TransportRestoreSnapshotAction(transportService, clusterService, threadPool, restoreService, actionFilters,
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index a4cb547b5b81a..40678a58d6e92 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -94,6 +94,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.fetch.FetchPhase;
@@ -454,7 +455,8 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
             }
         }
 
-        List<InternalAggregation> aggs = new ArrayList<> ();
+        PipelineTree pipelines = builder.buildPipelineTree();
+        List<InternalAggregation> aggs = new ArrayList<>();
         Query rewritten = searcher.rewrite(query);
         Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f);
         MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket,
@@ -481,33 +483,27 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
                 Collections.shuffle(aggs, random());
                 int r = randomIntBetween(1, toReduceSize);
                 List<InternalAggregation> toReduce = aggs.subList(0, r);
-                MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket,
-                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST));
-                InternalAggregation.ReduceContext context =
-                    new InternalAggregation.ReduceContext(root.context().bigArrays(), getMockScriptService(),
-                        reduceBucketConsumer, false);
+                InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(
+                        root.context().bigArrays(), getMockScriptService());
                 A reduced = (A) aggs.get(0).reduce(toReduce, context);
-                doAssertReducedMultiBucketConsumer(reduced, reduceBucketConsumer);
                 aggs = new ArrayList<>(aggs.subList(r, toReduceSize));
                 aggs.add(reduced);
             }
             // now do the final reduce
             MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket,
                 new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST));
-            InternalAggregation.ReduceContext context =
-                new InternalAggregation.ReduceContext(root.context().bigArrays(), getMockScriptService(), reduceBucketConsumer, true);
+            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                    root.context().bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines);
 
             @SuppressWarnings("unchecked")
             A internalAgg = (A) aggs.get(0).reduce(aggs, context);
 
             // materialize any parent pipelines
-            internalAgg = (A) internalAgg.reducePipelines(internalAgg, context);
+            internalAgg = (A) internalAgg.reducePipelines(internalAgg, context, pipelines);
 
             // materialize any sibling pipelines at top level
-            if (internalAgg.pipelineAggregators().size() > 0) {
-                for (PipelineAggregator pipelineAggregator : internalAgg.pipelineAggregators()) {
-                    internalAgg = (A) pipelineAggregator.reduce(internalAgg, context);
-                }
+            for (PipelineAggregator pipelineAggregator : pipelines.aggregators()) {
+                internalAgg = (A) pipelineAggregator.reduce(internalAgg, context);
             }
             doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer);
             return internalAgg;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index 3c6c2ea378b87..ebf60bbfb4b6d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.ContextParser;
@@ -42,6 +43,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
@@ -137,6 +139,7 @@
 import org.elasticsearch.search.aggregations.pipeline.PercentilesBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -159,6 +162,24 @@
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public abstract class InternalAggregationTestCase<T extends InternalAggregation> extends AbstractWireSerializingTestCase<T> {
+    /**
+     * Builds an {@link InternalAggregation.ReduceContextBuilder} that is valid but empty.
+     */
+    public static InternalAggregation.ReduceContextBuilder emptyReduceContextBuilder() {
+        return new InternalAggregation.ReduceContextBuilder() {
+            @Override
+            public InternalAggregation.ReduceContext forPartialReduction() {
+                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null);
+            }
+
+            @Override
+            public ReduceContext forFinalReduction() {
+                return InternalAggregation.ReduceContext.forFinalReduction(
+                        BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineTree.EMPTY);
+            }
+        };
+    }
+
     public static final int DEFAULT_MAX_BUCKETS = 100000;
     protected static final double TOLERANCE = 1e-10;
 
@@ -270,10 +291,7 @@ public void testReduceRandom() {
             Collections.shuffle(toReduce, random());
             int r = randomIntBetween(1, toReduceSize);
             List<InternalAggregation> internalAggregations = toReduce.subList(0, r);
-            MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(DEFAULT_MAX_BUCKETS,
-                new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST));
-            InternalAggregation.ReduceContext context =
-                new InternalAggregation.ReduceContext(bigArrays, mockScriptService, bucketConsumer,false);
+            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(bigArrays, mockScriptService);
             @SuppressWarnings("unchecked")
             T reduced = (T) inputs.get(0).reduce(internalAggregations, context);
             int initialBucketCount = 0;
@@ -283,14 +301,13 @@ public void testReduceRandom() {
             int reducedBucketCount = countInnerBucket(reduced);
             //check that non final reduction never adds buckets
             assertThat(reducedBucketCount, lessThanOrEqualTo(initialBucketCount));
-            assertMultiBucketConsumer(reducedBucketCount, bucketConsumer);
             toReduce = new ArrayList<>(toReduce.subList(r, toReduceSize));
             toReduce.add(reduced);
         }
         MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(DEFAULT_MAX_BUCKETS,
             new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST));
-        InternalAggregation.ReduceContext context =
-            new InternalAggregation.ReduceContext(bigArrays, mockScriptService, bucketConsumer, true);
+        InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                bigArrays, mockScriptService, bucketConsumer, PipelineTree.EMPTY);
         @SuppressWarnings("unchecked")
         T reduced = (T) inputs.get(0).reduce(toReduce, context);
         assertMultiBucketConsumer(reduced, bucketConsumer);
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
index cb2862a087774..479daffc832b5 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.tasks.TaskId;
@@ -45,7 +45,7 @@ final class AsyncSearchTask extends SearchTask {
     private final AsyncSearchId searchId;
     private final Client client;
     private final ThreadPool threadPool;
-    private final Supplier<ReduceContext> reduceContextSupplier;
+    private final Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier;
     private final Listener progressListener;
 
     private final Map<String, String> originHeaders;
@@ -72,7 +72,7 @@ final class AsyncSearchTask extends SearchTask {
      * @param taskHeaders The filtered request headers for the task.
      * @param searchId The {@link AsyncSearchId} of the task.
      * @param threadPool The threadPool to schedule runnable.
-     * @param reduceContextSupplier A supplier to create final reduce contexts.
+     * @param aggReduceContextSupplier A supplier to create final reduce contexts.
      */
     AsyncSearchTask(long id,
                     String type,
@@ -84,14 +84,14 @@ final class AsyncSearchTask extends SearchTask {
                     AsyncSearchId searchId,
                     Client client,
                     ThreadPool threadPool,
-                    Supplier<ReduceContext> reduceContextSupplier) {
+                    Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier) {
         super(id, type, action, "async_search", parentTaskId, taskHeaders);
         this.expirationTimeMillis = getStartTime() + keepAlive.getMillis();
         this.originHeaders = originHeaders;
         this.searchId = searchId;
         this.client = client;
         this.threadPool = threadPool;
-        this.reduceContextSupplier = reduceContextSupplier;
+        this.aggReduceContextSupplier = aggReduceContextSupplier;
         this.progressListener = new Listener();
         this.searchResponse = new AtomicReference<>();
         setProgressListener(progressListener);
@@ -327,7 +327,7 @@ protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped,
             // best effort to cancel expired tasks
             checkExpiration();
             searchResponse.compareAndSet(null,
-                new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier));
+                new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, aggReduceContextSupplier));
             executeInitListeners();
         }
 
@@ -360,7 +360,7 @@ public void onFailure(Exception exc) {
             if (searchResponse.get() == null) {
                 // if the failure occurred before calling onListShards
                 searchResponse.compareAndSet(null,
-                    new MutableSearchResponse(-1, -1, null, reduceContextSupplier));
+                    new MutableSearchResponse(-1, -1, null, aggReduceContextSupplier));
             }
             searchResponse.get().updateWithFailure(exc);
             executeInitListeners();
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
index 1949aceb6bd91..82e311fc8bb8a 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
@@ -14,12 +14,11 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
 
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
@@ -39,7 +38,7 @@ class MutableSearchResponse {
     private final int skippedShards;
     private final Clusters clusters;
     private final AtomicArray<ShardSearchFailure> shardFailures;
-    private final Supplier<ReduceContext> reduceContextSupplier;
+    private final Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier;
 
     private int version;
     private boolean isPartial;
@@ -56,13 +55,14 @@ class MutableSearchResponse {
      * @param totalShards The number of shards that participate in the request, or -1 to indicate a failure.
      * @param skippedShards The number of skipped shards, or -1 to indicate a failure.
      * @param clusters The remote clusters statistics.
-     * @param reduceContextSupplier A supplier to run final reduce on partial aggregations.
+     * @param aggReduceContextSupplier A supplier to run final reduce on partial aggregations.
      */
-    MutableSearchResponse(int totalShards, int skippedShards, Clusters clusters, Supplier<ReduceContext> reduceContextSupplier) {
+    MutableSearchResponse(int totalShards, int skippedShards, Clusters clusters,
+            Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier) {
         this.totalShards = totalShards;
         this.skippedShards = skippedShards;
         this.clusters = clusters;
-        this.reduceContextSupplier = reduceContextSupplier;
+        this.aggReduceContextSupplier = aggReduceContextSupplier;
         this.version = 0;
         this.shardFailures = totalShards == -1 ? null : new AtomicArray<>(totalShards-skippedShards);
         this.isPartial = true;
@@ -136,7 +136,7 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, lon
         if (totalShards != -1) {
             if (sections.aggregations() != null && isFinalReduce == false) {
                 InternalAggregations oldAggs = (InternalAggregations) sections.aggregations();
-                InternalAggregations newAggs = topLevelReduce(singletonList(oldAggs), reduceContextSupplier.get());
+                InternalAggregations newAggs = topLevelReduce(singletonList(oldAggs), aggReduceContextSupplier.get());
                 sections = new InternalSearchResponse(sections.hits(), newAggs, sections.suggest(),
                     null, sections.timedOut(), sections.terminatedEarly(), sections.getNumReducePhases());
                 isFinalReduce = true;
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
index a89f166866a9c..032c6114a1a44 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.search.SearchService;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
@@ -36,13 +36,14 @@
 import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
 
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public class TransportSubmitAsyncSearchAction extends HandledTransportAction<SubmitAsyncSearchRequest, AsyncSearchResponse> {
     private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class);
 
     private final NodeClient nodeClient;
-    private final Supplier<ReduceContext> reduceContextSupplier;
+    private final Function<SearchRequest, InternalAggregation.ReduceContext> requestToAggReduceContextBuilder;
     private final TransportSearchAction searchAction;
     private final AsyncSearchIndexService store;
 
@@ -57,7 +58,7 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService,
                                             TransportSearchAction searchAction) {
         super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new);
         this.nodeClient = nodeClient;
-        this.reduceContextSupplier = () -> searchService.createReduceContext(true);
+        this.requestToAggReduceContextBuilder = request -> searchService.aggReduceContextBuilder(request).forFinalReduction();
         this.searchAction = searchAction;
         this.store = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(), client, registry);
     }
@@ -135,8 +136,10 @@ private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long
             @Override
             public AsyncSearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> taskHeaders) {
                 AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id));
+                Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier =
+                        () -> requestToAggReduceContextBuilder.apply(request.getSearchRequest());
                 return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId,
-                    store.getClient(), nodeClient.threadPool(), reduceContextSupplier);
+                    store.getClient(), nodeClient.threadPool(), aggReduceContextSupplier);
             }
         };
         searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), parentTaskId));
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
index 4a8d007e3b898..e93442af52cec 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation.SingleValue;
 import org.elasticsearch.search.aggregations.metrics.InternalSum;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 
@@ -272,6 +273,8 @@ private static SearchResponse doCombineResponse(SearchResponse liveResponse, Lis
         // which means we can use aggregation's reduce method to combine, just as if
         // it was a result from another shard
         InternalAggregations currentTree = new InternalAggregations(Collections.emptyList());
+        InternalAggregation.ReduceContext finalReduceContext = InternalAggregation.ReduceContext.forFinalReduction(
+                reduceContext.bigArrays(), reduceContext.scriptService(), b -> {}, PipelineTree.EMPTY);
         for (SearchResponse rolledResponse : rolledResponses) {
             List<InternalAggregation> unrolledAggs = new ArrayList<>(rolledResponse.getAggregations().asList().size());
             for (Aggregation agg : rolledResponse.getAggregations()) {
@@ -289,14 +292,14 @@ private static SearchResponse doCombineResponse(SearchResponse liveResponse, Lis
             // Iteratively merge in each new set of unrolled aggs, so that we can identify/fix overlapping doc_counts
             // in the next round of unrolling
             InternalAggregations finalUnrolledAggs = new InternalAggregations(unrolledAggs);
-            currentTree = InternalAggregations.reduce(Arrays.asList(currentTree, finalUnrolledAggs),
-                    new InternalAggregation.ReduceContext(reduceContext.bigArrays(), reduceContext.scriptService(), true));
+            currentTree = InternalAggregations.reduce(Arrays.asList(currentTree, finalUnrolledAggs), finalReduceContext);
         }
 
         // Add in the live aggregations if they exist
         if (liveAggs.asList().size() != 0) {
-            currentTree = InternalAggregations.reduce(Arrays.asList(currentTree, liveAggs),
-                    new InternalAggregation.ReduceContext(reduceContext.bigArrays(), reduceContext.scriptService(), true));
+            // TODO it looks like this passes the "final" reduce context more than once.
+            // Once here and once in the for above. That is bound to cause trouble.
+            currentTree = InternalAggregations.reduce(Arrays.asList(currentTree, liveAggs), finalReduceContext);
         }
 
         return mergeFinalResponse(liveResponse, rolledResponses, currentTree);
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
index 6dd40d8b4e88d..60fbdc314c96c 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
@@ -105,8 +105,7 @@ protected void doExecute(Task task, SearchRequest request, ActionListener<Search
         MultiSearchRequest msearch = createMSearchRequest(request, registry, rollupSearchContext);
 
         client.multiSearch(msearch, ActionListener.wrap(msearchResponse -> {
-            InternalAggregation.ReduceContext context
-                    = new InternalAggregation.ReduceContext(bigArrays, scriptService, false);
+            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(bigArrays, scriptService);
             listener.onResponse(processResponses(rollupSearchContext, msearchResponse, context));
         }, listener::onFailure));
     }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
index 91953295bd523..6c369c3c94f24 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
@@ -66,6 +66,7 @@
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.xpack.core.rollup.RollupField;
@@ -96,12 +97,12 @@ public void testLiveFailure() {
 
         Exception e = expectThrows(RuntimeException.class,
                 () -> RollupResponseTranslator.combineResponses(failure,
-                        new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                        InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), equalTo("foo"));
 
         e = expectThrows(RuntimeException.class,
                 () -> RollupResponseTranslator.translateResponse(failure,
-                        new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                        InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), equalTo("foo"));
 
         e = expectThrows(RuntimeException.class,
@@ -118,7 +119,7 @@ public void testRollupFailure() {
 
         Exception e = expectThrows(RuntimeException.class,
                 () -> RollupResponseTranslator.translateResponse(failure,
-                        new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                        InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), equalTo("rollup failure"));
     }
 
@@ -132,7 +133,7 @@ public void testLiveMissingRollupMissing() {
 
         ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class,
                 () -> RollupResponseTranslator.combineResponses(failure,
-                        new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                        InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), equalTo("Index [[foo]] was not found, likely because it was deleted while the request was in-flight. " +
             "Rollup does not support partial search results, please try the request again."));
     }
@@ -177,7 +178,7 @@ public void testMissingLiveIndex() throws Exception {
         ScriptService scriptService = mock(ScriptService.class);
 
         ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, () -> RollupResponseTranslator.combineResponses(msearch,
-                new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), equalTo("Index [[foo]] was not found, likely because it was deleted while the request was in-flight. " +
             "Rollup does not support partial search results, please try the request again."));
     }
@@ -196,7 +197,7 @@ public void testRolledMissingAggs() throws Exception {
         ScriptService scriptService = mock(ScriptService.class);
 
         SearchResponse response = RollupResponseTranslator.translateResponse(msearch,
-            new InternalAggregation.ReduceContext(bigArrays, scriptService, true));
+                InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY));
         assertNotNull(response);
         Aggregations responseAggs = response.getAggregations();
         assertThat(responseAggs.asList().size(), equalTo(0));
@@ -213,7 +214,7 @@ public void testMissingRolledIndex() {
         ScriptService scriptService = mock(ScriptService.class);
 
         ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, () -> RollupResponseTranslator.combineResponses(msearch,
-            new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), equalTo("Index [[foo]] was not found, likely because it was deleted while the request was in-flight. " +
             "Rollup does not support partial search results, please try the request again."));
     }
@@ -268,7 +269,8 @@ public void testTranslateRollup() throws Exception {
 
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         ScriptService scriptService = mock(ScriptService.class);
-        InternalAggregation.ReduceContext context = new InternalAggregation.ReduceContext(bigArrays, scriptService, true);
+        InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                bigArrays, scriptService, b -> {}, PipelineTree.EMPTY);
 
         SearchResponse finalResponse = RollupResponseTranslator.translateResponse(new MultiSearchResponse.Item[]{item}, context);
         assertNotNull(finalResponse);
@@ -282,7 +284,8 @@ public void testTranslateMissingRollup() {
         MultiSearchResponse.Item missing = new MultiSearchResponse.Item(null, new IndexNotFoundException("foo"));
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         ScriptService scriptService = mock(ScriptService.class);
-        InternalAggregation.ReduceContext context = new InternalAggregation.ReduceContext(bigArrays, scriptService, true);
+        InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                bigArrays, scriptService, b -> {}, PipelineTree.EMPTY);
 
         ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class,
                 () -> RollupResponseTranslator.translateResponse(new MultiSearchResponse.Item[]{missing}, context));
@@ -316,7 +319,7 @@ public void testMissingFilter() {
 
         Exception e = expectThrows(RuntimeException.class,
                 () -> RollupResponseTranslator.combineResponses(msearch,
-                        new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                        InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(), containsString("Expected [bizzbuzz] to be a FilterAggregation"));
     }
 
@@ -345,7 +348,7 @@ public void testMatchingNameNotFilter() {
 
         Exception e = expectThrows(RuntimeException.class,
                 () -> RollupResponseTranslator.combineResponses(msearch,
-                        new InternalAggregation.ReduceContext(bigArrays, scriptService, true)));
+                        InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY)));
         assertThat(e.getMessage(),
                 equalTo("Expected [filter_foo] to be a FilterAggregation, but was [InternalMax]"));
     }
@@ -399,7 +402,7 @@ public void testSimpleReduction() throws Exception {
 
 
         SearchResponse response = RollupResponseTranslator.combineResponses(msearch,
-                new InternalAggregation.ReduceContext(bigArrays, scriptService, true));
+                InternalAggregation.ReduceContext.forFinalReduction(bigArrays, scriptService, b -> {}, PipelineTree.EMPTY));
         assertNotNull(response);
         Aggregations responseAggs = response.getAggregations();
         assertNotNull(responseAggs);
@@ -507,7 +510,8 @@ public void testMismatch() throws IOException {
 
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         ScriptService scriptService = mock(ScriptService.class);
-        InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(bigArrays, scriptService, true);
+        InternalAggregation.ReduceContext reduceContext = InternalAggregation.ReduceContext.forFinalReduction(
+                bigArrays, scriptService, b -> {}, PipelineTree.EMPTY);
         ClassCastException e = expectThrows(ClassCastException.class,
                 () -> RollupResponseTranslator.combineResponses(msearch, reduceContext));
         assertThat(e.getMessage(),
@@ -608,7 +612,8 @@ public void testDateHistoWithGap() throws IOException {
         // Reduce the InternalDateHistogram response so we can fill buckets
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         ScriptService scriptService = mock(ScriptService.class);
-        InternalAggregation.ReduceContext context = new InternalAggregation.ReduceContext(bigArrays, scriptService, true);
+        InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                bigArrays, scriptService, b -> {}, PipelineTree.EMPTY);
 
         InternalAggregation reduced = ((InternalDateHistogram)unrolled).reduce(Collections.singletonList(unrolled), context);
         assertThat(reduced.toString(), equalTo("{\"histo\":{\"buckets\":[{\"key_as_string\":\"1970-01-01T00:00:00.100Z\",\"key\":100," +

From c87da3e2c4296cbcfea094c0c6cb94c0635f8ba7 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 16 Mar 2020 21:01:16 +0200
Subject: [PATCH 061/285] [ML] Unmute a couple accidentally muted bwc tests for
 DF analytics (#53613)

---
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml    | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index b197d0c5cf433..3af10eb33e286 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -80,9 +80,6 @@
 
 ---
 "Get mixed cluster outlier_detection job":
-  - skip:
-      reason: "Waiting for backport of https://github.com/elastic/elasticsearch/pull/47935"
-      version: all
   - do:
       ml.get_data_frame_analytics:
         id: "mixed_cluster_outlier_detection_job"
@@ -101,9 +98,6 @@
 
 ---
 "Get mixed cluster outlier_detection job stats":
-  - skip:
-      reason: "Waiting for backport of https://github.com/elastic/elasticsearch/pull/47935"
-      version: all
   - do:
       ml.get_data_frame_analytics_stats:
         id: "mixed_cluster_outlier_detection_job"

From 4a5feab88d7eca0c57d6a8f4eed8aee8d7c83b7a Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Mon, 16 Mar 2020 12:14:01 -0700
Subject: [PATCH 062/285] [DOCS] Add anchors for scripted metric aggregations
 (#53618)

---
 .../metrics/scripted-metric-aggregation.asciidoc      | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index f73dec1eb8be4..1f753be25d172 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -102,6 +102,7 @@ Verify this response as well but in a hidden block.
 
 For more details on specifying scripts see <<modules-scripting, script documentation>>.
 
+[[scripted-metric-aggregation-return-types]]
 ==== Allowed return types
 
 Whilst any valid script object can be used within a single script, the scripts must return or store in the `state` object only the following types:
@@ -111,6 +112,7 @@ Whilst any valid script object can be used within a single script, the scripts m
 * Map (containing only keys and values of the types listed here)
 * Array (containing elements of only the types listed here)
 
+[[scripted-metric-aggregation-scope]]
 ==== Scope of scripts
 
 The scripted metric aggregation uses scripts at 4 stages of its execution:
@@ -139,7 +141,8 @@ reduce_script::     Executed once on the coordinating node after all shards have
 In the above example, the `reduce_script` iterates through the `profit` returned by each shard summing the values before returning the
 final combined profit which will be returned in the response of the aggregation.
 
-==== Worked Example
+[[scripted-metric-aggregation-example]]
+==== Worked example
 
 Imagine a situation where you index the following documents into an index with 2 shards:
 
@@ -255,7 +258,8 @@ produce the response:
 --------------------------------------------------
 // NOTCONSOLE
 
-==== Other Parameters
+[[scripted-metric-aggregation-parameters]]
+==== Other parameters
 
 [horizontal]
 params::           Optional. An object whose contents will be passed as variables to the  `init_script`, `map_script` and `combine_script`. This can be
@@ -268,7 +272,8 @@ params::           Optional. An object whose contents will be passed as variable
 --------------------------------------------------
 // NOTCONSOLE
 
-==== Empty Buckets
+[[scripted-metric-aggregation-empty-buckets]]
+==== Empty buckets
 
 If a parent bucket of the scripted metric aggregation does not collect any documents an empty aggregation response will be returned from the
 shard with a `null` value. In this case the `reduce_script`'s `states` variable will contain `null` as a response from that shard.

From 84a59f8447081fb81d1953796ba99600c4a90542 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Mon, 16 Mar 2020 15:15:25 -0400
Subject: [PATCH 063/285] Add scripting, supported-type tests to ValueCount
 (#53500)

Also adds a few small notes to the documentation regarding potentially
unintuitive behavior
---
 .../metrics/valuecount-aggregation.asciidoc   |   7 +
 .../metrics/ValueCountAggregatorTests.java    | 151 ++++++++++++++++++
 2 files changed, 158 insertions(+)

diff --git a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
index 74cc0ae76447b..36539eef1257c 100644
--- a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
@@ -6,6 +6,9 @@ These values can be extracted either from specific fields in the documents, or b
 this aggregator will be used in conjunction with other single-value aggregations. For example, when computing the `avg`
 one might be interested in the number of values the average is computed over.
 
+`value_count` does not de-duplicate values, so even if a field has duplicates (or a script generates multiple
+identical values for a single document), each value will be counted individually.
+
 [source,console]
 --------------------------------------------------
 POST /sales/_search?size=0
@@ -77,3 +80,7 @@ POST /sales/_search?size=0
 }
 --------------------------------------------------
 // TEST[setup:sales,stored_example_script]
+
+NOTE:: Because `value_count` is designed to work with any field it internally treats all values as simple bytes.
+Due to this implementation, if `_value` script variable is used to fetch a value instead of accessing the field
+directly (e.g. a "value script"), the field value will be returned as a string instead of it's native format.
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java
index e3060d4c78ef0..559aec4d231dc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
@@ -35,6 +37,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
@@ -44,14 +47,28 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.mapper.RangeType;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 import static java.util.Collections.singleton;
 
@@ -59,6 +76,41 @@ public class ValueCountAggregatorTests extends AggregatorTestCase {
 
     private static final String FIELD_NAME = "field";
 
+    /** Script to return the {@code _value} provided by aggs framework. */
+    private static final String VALUE_SCRIPT = "_value";
+    private static final String SINGLE_SCRIPT = "single";
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new ValueCountAggregationBuilder("foo", null).field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.BYTES,
+            CoreValuesSourceType.GEOPOINT,
+            CoreValuesSourceType.RANGE,
+            CoreValuesSourceType.HISTOGRAM
+        );
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+        scripts.put(VALUE_SCRIPT, vars -> (Double.valueOf((String) vars.get("_value")) + 1));
+        scripts.put(SINGLE_SCRIPT, vars -> 1);
+
+        MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME,
+            scripts,
+            Collections.emptyMap());
+        Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
+
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
+
     public void testNoDocs() throws IOException {
         for (ValueType valueType : ValueType.values()) {
             testCase(new MatchAllDocsQuery(), valueType, iw -> {
@@ -185,6 +237,105 @@ public void testRangeFieldValues() throws IOException {
         }, fieldType);
     }
 
+    public void testValueScriptNumber() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field(FIELD_NAME)
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
+
+        MappedFieldType fieldType = createMappedFieldType(ValueType.NUMERIC);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 8)));
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 9)));
+        }, card -> {
+            assertEquals(3, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, fieldType);
+    }
+
+    public void testSingleScriptNumber() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field(FIELD_NAME);
+
+        MappedFieldType fieldType = createMappedFieldType(ValueType.NUMERIC);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            Document doc = new Document();
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, 7));
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, 7));
+            iw.addDocument(doc);
+
+            doc = new Document();
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, 8));
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, 8));
+            iw.addDocument(doc);
+
+            doc = new Document();
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, 1));
+            doc.add(new SortedNumericDocValuesField(FIELD_NAME, 1));
+            iw.addDocument(doc);
+        }, card -> {
+            // note: this is 6, even though the script returns a single value.  ValueCount does not de-dedupe
+            assertEquals(6, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, fieldType);
+    }
+
+    public void testValueScriptString() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field(FIELD_NAME)
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
+
+        MappedFieldType fieldType = createMappedFieldType(ValueType.STRING);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("1"))));
+            iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("2"))));
+            iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("3"))));
+        }, card -> {
+            assertEquals(3, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, fieldType);
+    }
+
+    public void testSingleScriptString() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field(FIELD_NAME);
+
+        MappedFieldType fieldType = createMappedFieldType(ValueType.STRING);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            Document doc = new Document();
+            // Note: unlike numerics, lucene de-dupes strings so we increment here
+            doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("1")));
+            doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("2")));
+            iw.addDocument(doc);
+
+            doc = new Document();
+            doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("3")));
+            doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("4")));
+            iw.addDocument(doc);
+
+            doc = new Document();
+            doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("5")));
+            doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("6")));
+            iw.addDocument(doc);
+        }, card -> {
+            // note: this is 6, even though the script returns a single value.  ValueCount does not de-dedupe
+            assertEquals(6, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, fieldType);
+    }
+
     private void testCase(Query query,
                           ValueType valueType,
                           CheckedConsumer<RandomIndexWriter, IOException> indexer,

From f2f0fc03de085beb8dc2dc850048178c9273d8b9 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 16 Mar 2020 15:51:29 -0400
Subject: [PATCH 064/285] Update skip before backport

PR #53617 has yet to be backported so we should skip its integration
test on nodes earlier than 8.0.0
---
 .../rest-api-spec/test/search.aggregation/50_filter.yml       | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml
index b059e0892be08..a94c28fa034f5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml
@@ -107,6 +107,10 @@ setup:
 
 ---
 "Sorting terms":
+  - skip:
+      version: " - 7.99.99"
+      reason: fixed in 8.0.0 to be backported to 7.7.0
+
   - do:
       bulk:
         refresh: true

From 12e24852af3f49e6ebb31ac8d6c58baaadfa8169 Mon Sep 17 00:00:00 2001
From: Ryan Earle <ryan.earle@elastic.co>
Date: Mon, 16 Mar 2020 16:41:05 -0400
Subject: [PATCH 065/285] [DOCS] Remove `force` as valid value for
 `version_type` (#53428)

The `force` value for the `version_type` parameter was deprecated in 6.8.

This removes the value from the parameter definition.
---
 docs/reference/rest-api/common-parms.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index 34b418a157888..baaea29f7e7e6 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -922,7 +922,7 @@ end::segment-version[]
 tag::version_type[]
 `version_type`::
 (Optional, enum) Specific version type: `internal`, `external`,
-`external_gte`, `force`.
+`external_gte`.
 end::version_type[]
 
 tag::wait_for_active_shards[]

From 2abf40a6b61b17995ddb013e74736c98e489d568 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 16 Mar 2020 21:07:55 -0400
Subject: [PATCH 066/285] Add server name to remote info API (#53634)

This commit adds the configured server_name to the proxy mode info so
that it can be exposed in the remote info API.
---
 .../client/cluster/ProxyModeInfo.java         | 14 ++++++++---
 .../client/cluster/RemoteConnectionInfo.java  |  5 ++--
 .../cluster/RemoteInfoResponseTests.java      |  4 +++-
 .../transport/ProxyConnectionStrategy.java    | 24 +++++++++++++++----
 .../RemoteClusterConnectionTests.java         | 12 ++++++----
 5 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
index 79758ec709c63..3be10ce41a0fe 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
@@ -24,14 +24,17 @@
 public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
     static final String NAME = "proxy";
     static final String PROXY_ADDRESS = "proxy_address";
+    static final String SERVER_NAME = "server_name";
     static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
     static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
     private final String address;
+    private final String serverName;
     private final int maxSocketConnections;
     private final int numSocketsConnected;
 
-    ProxyModeInfo(String address, int maxSocketConnections, int numSocketsConnected) {
+    ProxyModeInfo(String address, String serverName, int maxSocketConnections, int numSocketsConnected) {
         this.address = address;
+        this.serverName = serverName;
         this.maxSocketConnections = maxSocketConnections;
         this.numSocketsConnected = numSocketsConnected;
     }
@@ -50,6 +53,10 @@ public String getAddress() {
         return address;
     }
 
+    public String getServerName() {
+        return serverName;
+    }
+
     public int getMaxSocketConnections() {
         return maxSocketConnections;
     }
@@ -65,11 +72,12 @@ public boolean equals(Object o) {
         ProxyModeInfo otherProxy = (ProxyModeInfo) o;
         return maxSocketConnections == otherProxy.maxSocketConnections &&
                 numSocketsConnected == otherProxy.numSocketsConnected &&
-                Objects.equals(address, otherProxy.address);
+                Objects.equals(address, otherProxy.address) &&
+                Objects.equals(serverName, otherProxy.serverName);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(address, maxSocketConnections, numSocketsConnected);
+        return Objects.hash(address, serverName, maxSocketConnections, numSocketsConnected);
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
index 474b991cd434a..cb0fb726297a1 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
@@ -48,9 +48,9 @@ public final class RemoteConnectionInfo {
                 String mode = (String) args[1];
                 ModeInfo modeInfo;
                 if (mode.equals(ProxyModeInfo.NAME)) {
-                    modeInfo = new ProxyModeInfo((String) args[4], (int) args[5], (int) args[6]);
+                    modeInfo = new ProxyModeInfo((String) args[4], (String) args[5], (int) args[6], (int) args[7]);
                 } else if (mode.equals(SniffModeInfo.NAME)) {
-                    modeInfo = new SniffModeInfo((List<String>) args[7], (int) args[8], (int) args[9]);
+                    modeInfo = new SniffModeInfo((List<String>) args[8], (int) args[9], (int) args[10]);
                 } else {
                     throw new IllegalArgumentException("mode cannot be " + mode);
                 }
@@ -67,6 +67,7 @@ public final class RemoteConnectionInfo {
         PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
 
         PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.SERVER_NAME));
         PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
         PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/cluster/RemoteInfoResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/cluster/RemoteInfoResponseTests.java
index 88f8f6f533e1d..0d5171eb310bb 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/cluster/RemoteInfoResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/cluster/RemoteInfoResponseTests.java
@@ -83,6 +83,7 @@ protected void assertInstances(org.elasticsearch.action.admin.cluster.remote.Rem
                 ProxyConnectionStrategy.ProxyModeInfo serverModeInfo =
                         (ProxyConnectionStrategy.ProxyModeInfo) serverRemoteInfo.getModeInfo();
                 assertThat(clientModeInfo.getAddress(), equalTo(serverModeInfo.getAddress()));
+                assertThat(clientModeInfo.getServerName(), equalTo(serverModeInfo.getServerName()));
                 assertThat(clientModeInfo.getMaxSocketConnections(), equalTo(serverModeInfo.getMaxSocketConnections()));
                 assertThat(clientModeInfo.getNumSocketsConnected(), equalTo(serverModeInfo.getNumSocketsConnected()));
             } else {
@@ -95,9 +96,10 @@ private static RemoteConnectionInfo createRandomRemoteConnectionInfo() {
         RemoteConnectionInfo.ModeInfo modeInfo;
         if (randomBoolean()) {
             String address = randomAlphaOfLength(8);
+            String serverName = randomAlphaOfLength(8);
             int maxSocketConnections = randomInt(5);
             int numSocketsConnected = randomInt(5);
-            modeInfo = new ProxyConnectionStrategy.ProxyModeInfo(address, maxSocketConnections, numSocketsConnected);
+            modeInfo = new ProxyConnectionStrategy.ProxyModeInfo(address, serverName, maxSocketConnections, numSocketsConnected);
         } else {
             List<String> seedNodes = randomList(randomInt(8), () -> randomAlphaOfLength(8));
             int maxConnectionsPerCluster = randomInt(5);
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index 2d314a591b57a..fa4d227ece76f 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -172,7 +172,7 @@ protected void connectImpl(ActionListener<Void> listener) {
 
     @Override
     public RemoteConnectionInfo.ModeInfo getModeInfo() {
-        return new ProxyModeInfo(configuredAddress, maxNumConnections, connectionManager.size());
+        return new ProxyModeInfo(configuredAddress, configuredServerName, maxNumConnections, connectionManager.size());
     }
 
     private void performProxyConnectionProcess(ActionListener<Void> listener) {
@@ -255,17 +255,24 @@ private static TransportAddress resolveAddress(String address) {
     public static class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
 
         private final String address;
+        private final String serverName;
         private final int maxSocketConnections;
         private final int numSocketsConnected;
 
-        public ProxyModeInfo(String address, int maxSocketConnections, int numSocketsConnected) {
+        public ProxyModeInfo(String address, String serverName, int maxSocketConnections, int numSocketsConnected) {
             this.address = address;
+            this.serverName = serverName;
             this.maxSocketConnections = maxSocketConnections;
             this.numSocketsConnected = numSocketsConnected;
         }
 
         private ProxyModeInfo(StreamInput input) throws IOException {
             address = input.readString();
+            if (input.getVersion().onOrAfter(Version.V_8_0_0)) {
+                serverName = input.readString();
+            } else {
+                serverName = null;
+            }
             maxSocketConnections = input.readVInt();
             numSocketsConnected = input.readVInt();
         }
@@ -273,6 +280,7 @@ private ProxyModeInfo(StreamInput input) throws IOException {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field("proxy_address", address);
+            builder.field("server_name", serverName);
             builder.field("num_proxy_sockets_connected", numSocketsConnected);
             builder.field("max_proxy_socket_connections", maxSocketConnections);
             return builder;
@@ -281,6 +289,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(address);
+            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                out.writeString(serverName);
+            }
             out.writeVInt(maxSocketConnections);
             out.writeVInt(numSocketsConnected);
         }
@@ -299,6 +310,10 @@ public String getAddress() {
             return address;
         }
 
+        public String getServerName() {
+            return serverName;
+        }
+
         public int getMaxSocketConnections() {
             return maxSocketConnections;
         }
@@ -319,12 +334,13 @@ public boolean equals(Object o) {
             ProxyModeInfo otherProxy = (ProxyModeInfo) o;
             return maxSocketConnections == otherProxy.maxSocketConnections &&
                 numSocketsConnected == otherProxy.numSocketsConnected &&
-                Objects.equals(address, otherProxy.address);
+                Objects.equals(address, otherProxy.address) &&
+                Objects.equals(serverName, otherProxy.serverName);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(address, maxSocketConnections, numSocketsConnected);
+            return Objects.hash(address, serverName, maxSocketConnections, numSocketsConnected);
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 2e3102cb7e926..088f14ef36c1a 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -346,6 +346,7 @@ public void testGetConnectionInfo() throws Exception {
 
     public void testRemoteConnectionInfo() throws IOException {
         List<String> remoteAddresses = Collections.singletonList("seed:1");
+        String serverName = "the_server_name";
 
         RemoteConnectionInfo.ModeInfo modeInfo1;
         RemoteConnectionInfo.ModeInfo modeInfo2;
@@ -354,8 +355,8 @@ public void testRemoteConnectionInfo() throws IOException {
             modeInfo1 = new SniffConnectionStrategy.SniffModeInfo(remoteAddresses, 4, 4);
             modeInfo2 = new SniffConnectionStrategy.SniffModeInfo(remoteAddresses, 4, 3);
         } else {
-            modeInfo1 = new ProxyConnectionStrategy.ProxyModeInfo(remoteAddresses.get(0), 18, 18);
-            modeInfo2 = new ProxyConnectionStrategy.ProxyModeInfo(remoteAddresses.get(0), 18, 17);
+            modeInfo1 = new ProxyConnectionStrategy.ProxyModeInfo(remoteAddresses.get(0), serverName, 18, 18);
+            modeInfo2 = new ProxyConnectionStrategy.ProxyModeInfo(remoteAddresses.get(0), serverName,18, 17);
         }
 
         RemoteConnectionInfo stats =
@@ -395,6 +396,7 @@ private static RemoteConnectionInfo assertSerialization(RemoteConnectionInfo inf
 
     public void testRenderConnectionInfoXContent() throws IOException {
         List<String> remoteAddresses = Arrays.asList("seed:1", "seed:2");
+        String serverName = "the_server_name";
 
         RemoteConnectionInfo.ModeInfo modeInfo;
 
@@ -402,7 +404,7 @@ public void testRenderConnectionInfoXContent() throws IOException {
         if (sniff) {
             modeInfo = new SniffConnectionStrategy.SniffModeInfo(remoteAddresses, 3, 2);
         } else {
-            modeInfo = new ProxyConnectionStrategy.ProxyModeInfo(remoteAddresses.get(0), 18, 16);
+            modeInfo = new ProxyConnectionStrategy.ProxyModeInfo(remoteAddresses.get(0), serverName,18, 16);
         }
 
         RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster", modeInfo, TimeValue.timeValueMinutes(30), true);
@@ -418,8 +420,8 @@ public void testRenderConnectionInfoXContent() throws IOException {
                 "\"skip_unavailable\":true}}", Strings.toString(builder));
         } else {
             assertEquals("{\"test_cluster\":{\"connected\":true,\"mode\":\"proxy\",\"proxy_address\":\"seed:1\"," +
-                "\"num_proxy_sockets_connected\":16,\"max_proxy_socket_connections\":18,\"initial_connect_timeout\":\"30m\"," +
-                "\"skip_unavailable\":true}}", Strings.toString(builder));
+                "\"server_name\":\"the_server_name\",\"num_proxy_sockets_connected\":16,\"max_proxy_socket_connections\":18,"+
+                "\"initial_connect_timeout\":\"30m\",\"skip_unavailable\":true}}", Strings.toString(builder));
         }
     }
 

From 87dc720dacaa5c1f388018c0dec0929bf881e5fe Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 16 Mar 2020 21:21:31 -0400
Subject: [PATCH 067/285] Update server name serialization version

This commit updates the serialization version for the server name on the
proxy mode info, used in the remote info API.
---
 .../org/elasticsearch/transport/ProxyConnectionStrategy.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index fa4d227ece76f..dc721bf6cde26 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -268,7 +268,7 @@ public ProxyModeInfo(String address, String serverName, int maxSocketConnections
 
         private ProxyModeInfo(StreamInput input) throws IOException {
             address = input.readString();
-            if (input.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (input.getVersion().onOrAfter(Version.V_7_7_0)) {
                 serverName = input.readString();
             } else {
                 serverName = null;
@@ -289,7 +289,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(address);
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                 out.writeString(serverName);
             }
             out.writeVInt(maxSocketConnections);

From 41e3b4aa905b053c80d52618ca23fa5919ae818e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 16 Mar 2020 21:27:02 -0400
Subject: [PATCH 068/285] Invoke response handler on failure to send (#53631)

Today it can happen that a transport message fails to send (for example,
because a transport interceptor rejects the request). In this case, the
response handler is never invoked, which can lead to necessary cleanups
not being performed. There are two ways to handle this. One is to expect
every callsite that sends a message to try/catch these exceptions and
handle them appropriately. The other is merely to invoke the response
handler to handle the exception, which is already equipped to handle
transport exceptions.
---
 .../transport/TransportService.java           |  56 ++++++----
 .../test/transport/MockTransportService.java  |   8 +-
 .../AbstractSimpleTransportTestCase.java      | 102 +++++++++++++++++-
 3 files changed, 141 insertions(+), 25 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index 0f9b25cc9c617..dcf9b3595c80e 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -46,7 +46,6 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -519,37 +518,57 @@ public void removeConnectionListener(TransportConnectionListener listener) {
     public <T extends TransportResponse> void sendRequest(final DiscoveryNode node, final String action,
                                                                 final TransportRequest request,
                                                                 final TransportResponseHandler<T> handler) {
+        final Transport.Connection connection;
         try {
-            Transport.Connection connection = getConnection(node);
-            sendRequest(connection, action, request, TransportRequestOptions.EMPTY, handler);
-        } catch (NodeNotConnectedException ex) {
+            connection = getConnection(node);
+        } catch (final NodeNotConnectedException ex) {
             // the caller might not handle this so we invoke the handler
             handler.handleException(ex);
+            return;
         }
+        sendRequest(connection, action, request, TransportRequestOptions.EMPTY, handler);
     }
 
     public final <T extends TransportResponse> void sendRequest(final DiscoveryNode node, final String action,
                                                                 final TransportRequest request,
                                                                 final TransportRequestOptions options,
                                                                 TransportResponseHandler<T> handler) {
+        final Transport.Connection connection;
         try {
-            Transport.Connection connection = getConnection(node);
-            sendRequest(connection, action, request, options, handler);
-        } catch (NodeNotConnectedException ex) {
+            connection = getConnection(node);
+        } catch (final NodeNotConnectedException ex) {
             // the caller might not handle this so we invoke the handler
             handler.handleException(ex);
+            return;
         }
+        sendRequest(connection, action, request, options, handler);
     }
 
+    /**
+     * Sends a request on the specified connection. If there is a failure sending the request, the specified handler is invoked.
+     *
+     * @param connection the connection to send the request on
+     * @param action     the name of the action
+     * @param request    the request
+     * @param options    the options for this request
+     * @param handler    the response handler
+     * @param <T>        the type of the transport response
+     */
     public final <T extends TransportResponse> void sendRequest(final Transport.Connection connection, final String action,
                                                                 final TransportRequest request,
                                                                 final TransportRequestOptions options,
                                                                 TransportResponseHandler<T> handler) {
         try {
             asyncSender.sendRequest(connection, action, request, options, handler);
-        } catch (NodeNotConnectedException ex) {
+        } catch (final Exception ex) {
             // the caller might not handle this so we invoke the handler
-            handler.handleException(ex);
+            final TransportException te;
+            if (ex instanceof TransportException) {
+                te = (TransportException) ex;
+            } else {
+                te = new TransportException("failure to send", ex);
+            }
+            handler.handleException(te);
         }
     }
 
@@ -569,13 +588,15 @@ public final <T extends TransportResponse> void sendChildRequest(final Discovery
                                                                      final TransportRequest request, final Task parentTask,
                                                                      final TransportRequestOptions options,
                                                                      final TransportResponseHandler<T> handler) {
+        final Transport.Connection connection;
         try {
-            Transport.Connection connection = getConnection(node);
-            sendChildRequest(connection, action, request, parentTask, options, handler);
-        } catch (NodeNotConnectedException ex) {
+            connection = getConnection(node);
+        } catch (final NodeNotConnectedException ex) {
             // the caller might not handle this so we invoke the handler
             handler.handleException(ex);
+            return;
         }
+        sendChildRequest(connection, action, request, parentTask, options, handler);
     }
 
     public <T extends TransportResponse> void sendChildRequest(final Transport.Connection connection, final String action,
@@ -589,16 +610,7 @@ public <T extends TransportResponse> void sendChildRequest(final Transport.Conne
                                                                final TransportRequestOptions options,
                                                                final TransportResponseHandler<T> handler) {
         request.setParentTask(localNode.getId(), parentTask.getId());
-        try {
-            sendRequest(connection, action, request, options, handler);
-        } catch (TaskCancelledException ex) {
-            // The parent task is already cancelled - just fail the request
-            handler.handleException(new TransportException(ex));
-        } catch (NodeNotConnectedException ex) {
-            // the caller might not handle this so we invoke the handler
-            handler.handleException(ex);
-        }
-
+        sendRequest(connection, action, request, options, handler);
     }
 
     private <T extends TransportResponse> void sendRequestInternal(final Transport.Connection connection, final String action,
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index f90727c936e00..0867bb4abec1d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -120,7 +120,13 @@ public static MockNioTransport newMockTransport(Settings settings, Version versi
 
     public static MockTransportService createNewService(Settings settings, Transport transport, Version version, ThreadPool threadPool,
                                                         @Nullable ClusterSettings clusterSettings, Set<String> taskHeaders) {
-        return new MockTransportService(settings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+        return createNewService(settings, transport, version, threadPool, clusterSettings, taskHeaders, NOOP_TRANSPORT_INTERCEPTOR);
+    }
+
+    public static MockTransportService createNewService(Settings settings, Transport transport, Version version, ThreadPool threadPool,
+                                                        @Nullable ClusterSettings clusterSettings, Set<String> taskHeaders,
+                                                        TransportInterceptor interceptor) {
+        return new MockTransportService(settings, transport, threadPool, interceptor,
             boundAddress ->
                 new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), UUIDs.randomBase64UUID(), boundAddress.publishAddress(),
                     Node.NODE_ATTRIBUTES.getAsMap(settings), DiscoveryNode.getRolesFromSettings(settings), version),
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index c0fd76d7f753b..0a89b2811b1a6 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -94,12 +94,15 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
 public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
@@ -187,7 +190,8 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
     }
 
     private MockTransportService buildService(final String name, final Version version, @Nullable ClusterSettings clusterSettings,
-                                              Settings settings, boolean acceptRequests, boolean doHandshake) {
+                                              Settings settings, boolean acceptRequests, boolean doHandshake,
+                                              TransportInterceptor interceptor) {
         Settings updatedSettings = Settings.builder()
             .put(TransportSettings.PORT.getKey(), getPortRange())
             .put(settings)
@@ -198,7 +202,7 @@ private MockTransportService buildService(final String name, final Version versi
         }
         Transport transport = build(updatedSettings, version, clusterSettings, doHandshake);
         MockTransportService service = MockTransportService.createNewService(updatedSettings, transport, version, threadPool,
-            clusterSettings, Collections.emptySet());
+            clusterSettings, Collections.emptySet(), interceptor);
         service.start();
         if (acceptRequests) {
             service.acceptIncomingRequests();
@@ -206,6 +210,11 @@ private MockTransportService buildService(final String name, final Version versi
         return service;
     }
 
+    private MockTransportService buildService(final String name, final Version version, @Nullable ClusterSettings clusterSettings,
+                                              Settings settings, boolean acceptRequests, boolean doHandshake) {
+        return buildService(name, version, clusterSettings, settings, acceptRequests, doHandshake, NOOP_TRANSPORT_INTERCEPTOR);
+    }
+
     protected MockTransportService buildService(final String name, final Version version, Settings settings) {
         return buildService(name, version, null, settings);
     }
@@ -2744,6 +2753,95 @@ public void onConnectionClosed(Transport.Connection connection) {
         }
     }
 
+    // test that the response handler is invoked on a failure to send
+    public void testFailToSend() throws InterruptedException {
+        final RuntimeException failToSendException;
+        if (randomBoolean()) {
+            failToSendException = new IllegalStateException("fail to send");
+        } else {
+            failToSendException = new TransportException("fail to send");
+        }
+        final TransportInterceptor interceptor = new TransportInterceptor() {
+            @Override
+            public AsyncSender interceptSender(final AsyncSender sender) {
+                return new AsyncSender() {
+                    @Override
+                    public <T extends TransportResponse> void sendRequest(
+                        final Transport.Connection connection,
+                        final String action,
+                        final TransportRequest request,
+                        final TransportRequestOptions options,
+                        final TransportResponseHandler<T> handler) {
+                        if ("fail-to-send-action".equals(action)) {
+                            throw failToSendException;
+                        } else {
+                            sender.sendRequest(connection, action, request, options, handler);
+                        }
+                    }
+                };
+            }
+        };
+        try (MockTransportService serviceC = buildService("TS_C", CURRENT_VERSION, null, Settings.EMPTY, true, true, interceptor)) {
+            serviceC.start();
+            serviceC.acceptIncomingRequests();
+            final CountDownLatch latch = new CountDownLatch(1);
+            serviceC.connectToNode(
+                serviceA.getLocalDiscoNode(),
+                ConnectionProfile.buildDefaultConnectionProfile(Settings.EMPTY),
+                new ActionListener<>() {
+                    @Override
+                    public void onResponse(final Void v) {
+                        latch.countDown();
+                    }
+
+                    @Override
+                    public void onFailure(final Exception e) {
+                        fail(e.getMessage());
+                    }
+            });
+            latch.await();
+            final AtomicReference<TransportException> te = new AtomicReference<>();
+            final Transport.Connection connection = serviceC.getConnection(nodeA);
+            serviceC.sendRequest(
+                connection,
+                "fail-to-send-action",
+                TransportRequest.Empty.INSTANCE,
+                TransportRequestOptions.EMPTY,
+                new TransportResponseHandler<TransportResponse>() {
+                    @Override
+                    public void handleResponse(final TransportResponse response) {
+                        fail("handle response should not be invoked");
+                    }
+
+                    @Override
+                    public void handleException(final TransportException exp) {
+                        te.set(exp);
+                    }
+
+                    @Override
+                    public String executor() {
+                        return ThreadPool.Names.SAME;
+                    }
+
+                    @Override
+                    public TransportResponse read(final StreamInput in) {
+                        return TransportResponse.Empty.INSTANCE;
+                    }
+                });
+            assertThat(te.get(), not(nullValue()));
+
+            if (failToSendException instanceof IllegalStateException) {
+                assertThat(te.get().getMessage(), equalTo("failure to send"));
+                assertThat(te.get().getCause(), instanceOf(IllegalStateException.class));
+                assertThat(te.get().getCause().getMessage(), equalTo("fail to send"));
+            } else {
+                assertThat(te.get().getMessage(), equalTo("fail to send"));
+                assertThat(te.get().getCause(), nullValue());
+            }
+        }
+
+    }
+
     private void closeConnectionChannel(Transport.Connection connection) {
         StubbableTransport.WrappedConnection wrappedConnection = (StubbableTransport.WrappedConnection) connection;
         TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) wrappedConnection.getConnection();

From 3608be62f53e1c8258d437e3354478f7e006e423 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 16 Mar 2020 18:54:40 -0700
Subject: [PATCH 069/285] Add internalClusterTest to check task (#53444)

This commit adds internalClusterTest in xpack core to run as part of
check. This was accidentally removed in a refactoring. Other xpack
modules already do this, but core was left out. This commit also mutes 2
tests that currently fail.

closes #53407
---
 x-pack/plugin/core/build.gradle                                 | 1 +
 .../xpack/core/rest/action/ReloadSynonymAnalyzerIT.java         | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index 6858df1c199ab..307fd1f1a499c 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -153,3 +153,4 @@ integTest.enabled = false
 task internalClusterTest(type: Test) {
   include "**/*IT.class"
 }
+check.dependsOn internalClusterTest
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
index 6f3c21d1d74fc..37cd500bf31b2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.rest.action;
 
+import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.Response;
 import org.elasticsearch.action.search.SearchResponse;
@@ -39,6 +40,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53443")
 public class ReloadSynonymAnalyzerIT extends ESIntegTestCase {
 
     @Override

From b67863e67a3be7a7ee761672ecb102fbecbaa28c Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Tue, 17 Mar 2020 16:34:00 +1100
Subject: [PATCH 070/285] Explicitly require that derived API keys have no
 privileges (#53647)

The current implicit behaviour is that when an API keys is used to create another API key,
the child key is created without any privilege. This implicit behaviour is surprising and is
a source of confusion for users.

This change makes that behaviour explicit.
---
 .../core/security/authz/RoleDescriptor.java   | 10 +++
 .../action/TransportCreateApiKeyAction.java   | 12 ++++
 .../security/authc/ApiKeyIntegTests.java      | 71 +++++++++++++++++++
 .../security/authz/RoleDescriptorTests.java   | 53 ++++++++++++++
 4 files changed, 146 insertions(+)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
index 832e9c9365e94..fed249544f9ac 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
@@ -212,6 +212,16 @@ public int hashCode() {
         return result;
     }
 
+
+    public boolean isEmpty() {
+        return clusterPrivileges.length == 0
+            && configurableClusterPrivileges.length == 0
+            && indicesPrivileges.length == 0
+            && applicationPrivileges.length == 0
+            && runAs.length == 0
+            && metadata.size() == 0;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         return toXContent(builder, params, false);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
index 730affc7ea953..2f5de387b6b73 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
 import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
@@ -44,7 +45,18 @@ protected void doExecute(Task task, CreateApiKeyRequest request, ActionListener<
         if (authentication == null) {
             listener.onFailure(new IllegalStateException("authentication is required"));
         } else {
+            if (Authentication.AuthenticationType.API_KEY == authentication.getAuthenticationType() && grantsAnyPrivileges(request)) {
+                listener.onFailure(new IllegalArgumentException(
+                    "creating derived api keys requires an explicit role descriptor that is empty (has no privileges)"));
+                return;
+            }
             generator.generateApiKey(authentication, request, listener);
         }
     }
+
+    private boolean grantsAnyPrivileges(CreateApiKeyRequest request) {
+        return request.getRoleDescriptors() == null
+            || request.getRoleDescriptors().isEmpty()
+            || false == request.getRoleDescriptors().stream().allMatch(RoleDescriptor::isEmpty);
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
index 9c695ac1ae194..4587395eabd6b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
@@ -9,6 +9,8 @@
 import com.google.common.collect.Sets;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
@@ -758,6 +760,75 @@ public void testApiKeyWithManageOwnPrivilegeIsAbleToInvalidateItselfButNotAnyOth
         assertThat(invalidateResponse.getErrors().size(), equalTo(0));
     }
 
+    public void testDerivedKeys() throws ExecutionException, InterruptedException {
+        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
+                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
+        final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client)
+            .setName("key-1")
+            .setRoleDescriptors(Collections.singletonList(
+                new RoleDescriptor("role", new String[] { "manage_api_key" }, null, null)))
+            .get();
+
+        assertEquals("key-1", response.getName());
+        assertNotNull(response.getId());
+        assertNotNull(response.getKey());
+
+        // use the first ApiKey for authorized action
+        final String base64ApiKeyKeyValue = Base64.getEncoder().encodeToString(
+            (response.getId() + ":" + response.getKey().toString()).getBytes(StandardCharsets.UTF_8));
+        final Client clientKey1 = client().filterWithHeader(Collections.singletonMap("Authorization", "ApiKey " + base64ApiKeyKeyValue));
+
+        final String expectedMessage = "creating derived api keys requires an explicit role descriptor that is empty";
+
+        final IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class,
+                () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-2").get());
+        assertThat(e1.getMessage(), containsString(expectedMessage));
+
+        final IllegalArgumentException e2 = expectThrows(IllegalArgumentException.class,
+            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-3")
+                .setRoleDescriptors(Collections.emptyList()).get());
+        assertThat(e2.getMessage(), containsString(expectedMessage));
+
+        final IllegalArgumentException e3 = expectThrows(IllegalArgumentException.class,
+            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-4")
+                .setRoleDescriptors(Collections.singletonList(
+                    new RoleDescriptor("role", new String[] {"manage_own_api_key"}, null, null)
+                )).get());
+        assertThat(e3.getMessage(), containsString(expectedMessage));
+
+        final List<RoleDescriptor> roleDescriptors = randomList(2, 10,
+            () -> new RoleDescriptor("role", null, null, null));
+        roleDescriptors.set(randomInt(roleDescriptors.size() - 1),
+            new RoleDescriptor("role", new String[] {"manage_own_api_key"}, null, null));
+
+        final IllegalArgumentException e4 = expectThrows(IllegalArgumentException.class,
+            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-5")
+                .setRoleDescriptors(roleDescriptors).get());
+        assertThat(e4.getMessage(), containsString(expectedMessage));
+
+        final CreateApiKeyResponse key100Response = new CreateApiKeyRequestBuilder(clientKey1).setName("key-100")
+            .setRoleDescriptors(Collections.singletonList(
+                new RoleDescriptor("role", null, null, null)
+            )).get();
+        assertEquals("key-100", key100Response.getName());
+        assertNotNull(key100Response.getId());
+        assertNotNull(key100Response.getKey());
+
+        // Check at the end to allow sometime for the operation to happen. Since an erroneous creation is
+        // asynchronous so that the document is not available immediately.
+        assertApiKeyNotCreated(client, "key-2");
+        assertApiKeyNotCreated(client, "key-3");
+        assertApiKeyNotCreated(client, "key-4");
+        assertApiKeyNotCreated(client, "key-5");
+    }
+
+    private void assertApiKeyNotCreated(Client client, String keyName) throws ExecutionException, InterruptedException {
+        new RefreshRequestBuilder(client, RefreshAction.INSTANCE).setIndices(SECURITY_MAIN_ALIAS).execute().get();
+        assertEquals(0, client.execute(GetApiKeyAction.INSTANCE,
+            GetApiKeyRequest.usingApiKeyName(keyName, false)).get().getApiKeyInfos().length);
+    }
+
     private void verifyGetResponse(int expectedNumberOfApiKeys, List<CreateApiKeyResponse> responses,
                                    GetApiKeyResponse response, Set<String> validApiKeyIds, List<String> invalidatedApiKeyIds) {
         verifyGetResponse(SecuritySettingsSource.TEST_SUPERUSER, expectedNumberOfApiKeys, responses, response, validApiKeyIds,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java
index 66161ee27ec19..b8661168c3232 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java
@@ -32,7 +32,9 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -325,4 +327,55 @@ public void testParseIndicesPrivilegesFailsWhenExceptFieldsAreNotSubsetOfGranted
         assertThat(epe, TestMatchers.throwableWithMessage(containsString("f2")));
         assertThat(epe, TestMatchers.throwableWithMessage(containsString("f3")));
     }
+
+    public void testIsEmpty() {
+        assertTrue(new RoleDescriptor(
+            randomAlphaOfLengthBetween(1, 10), null, null, null, null, null, null, null)
+            .isEmpty());
+
+        assertTrue(new RoleDescriptor(
+            randomAlphaOfLengthBetween(1, 10),
+            new String[0],
+            new RoleDescriptor.IndicesPrivileges[0],
+            new RoleDescriptor.ApplicationResourcePrivileges[0],
+            new ConfigurableClusterPrivilege[0],
+            new String[0],
+            new HashMap<>(),
+            new HashMap<>())
+            .isEmpty());
+
+        final List<Boolean> booleans = Arrays.asList(
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean());
+
+        final RoleDescriptor roleDescriptor = new RoleDescriptor(
+            randomAlphaOfLengthBetween(1, 10),
+            booleans.get(0) ? new String[0] : new String[] { "foo" },
+            booleans.get(1) ?
+                new RoleDescriptor.IndicesPrivileges[0] :
+                new RoleDescriptor.IndicesPrivileges[] {
+                    RoleDescriptor.IndicesPrivileges.builder().indices("idx").privileges("foo").build() },
+            booleans.get(2) ?
+                new RoleDescriptor.ApplicationResourcePrivileges[0] :
+                new RoleDescriptor.ApplicationResourcePrivileges[] {
+                    RoleDescriptor.ApplicationResourcePrivileges.builder()
+                        .application("app").privileges("foo").resources("res").build() },
+            booleans.get(3) ?
+                new ConfigurableClusterPrivilege[0] :
+                new ConfigurableClusterPrivilege[] {
+                    new ConfigurableClusterPrivileges.ManageApplicationPrivileges(Collections.singleton("foo")) },
+            booleans.get(4) ? new String[0] : new String[] { "foo" },
+            booleans.get(5) ? new HashMap<>() : Collections.singletonMap("foo", "bar"),
+            Collections.singletonMap("foo", "bar"));
+
+        if (booleans.stream().anyMatch(e -> e.equals(false))) {
+            assertFalse(roleDescriptor.isEmpty());
+        } else {
+            assertTrue(roleDescriptor.isEmpty());
+        }
+    }
 }

From f65e4d6ff7b2e00eb6f9c985fbe7cb24de00f045 Mon Sep 17 00:00:00 2001
From: Andrei Stefan <astefan@users.noreply.github.com>
Date: Tue, 17 Mar 2020 11:46:17 +0200
Subject: [PATCH 071/285] SQL: add support for index aliases for SYS COLUMNS
 command (#53525)

---
 .../xpack/ql/index/IndexResolver.java         | 236 +++++++++++-
 .../xpack/ql/type/InvalidMappedField.java     |   5 +
 .../sql/qa/single_node/SysColumnsIT.java      |  13 +
 .../xpack/sql/qa/jdbc/SysColumnsTestCase.java | 358 ++++++++++++++++++
 .../single-node-only/command-sys.csv-spec     |  96 +++--
 .../analysis/index/IndexResolverTests.java    |   6 +-
 6 files changed, 656 insertions(+), 58 deletions(-)
 create mode 100644 x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java
 create mode 100644 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java

diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
index 2b8cddfcf8b15..cfc125d492519 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ql.index;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
@@ -16,12 +17,14 @@
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.IndicesOptions.Option;
 import org.elasticsearch.action.support.IndicesOptions.WildcardStates;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
@@ -42,9 +45,11 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -164,7 +169,6 @@ public boolean equals(Object obj) {
     private final String clusterName;
     private final DataTypeRegistry typeRegistry;
 
-
     public IndexResolver(Client client, String clusterName, DataTypeRegistry typeRegistry) {
         this.client = client;
         this.clusterName = clusterName;
@@ -296,7 +300,7 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
         }
 
         // merge all indices onto the same one
-        List<EsIndex> indices = buildIndices(typeRegistry, indexNames, null, fieldCaps, i -> indexPattern, (n, types) -> {
+        List<EsIndex> indices = buildIndices(typeRegistry, indexNames, null, fieldCaps, null, i -> indexPattern, (n, types) -> {
             StringBuilder errorMessage = new StringBuilder();
 
             boolean hasUnmapped = types.containsKey(UNMAPPED);
@@ -473,17 +477,32 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boo
     public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen,
             ActionListener<List<EsIndex>> listener) {
         FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen);
-        client.fieldCaps(fieldRequest,
-                ActionListener.wrap(
-                        response -> listener.onResponse(
-                                separateMappings(typeRegistry, indexWildcard, javaRegex, response.getIndices(), response.get())),
-                        listener::onFailure));
+        client.fieldCaps(fieldRequest, wrap(response -> {
+            client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases ->
+                listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases())),
+                ex -> {
+                    if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) {
+                        listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null));
+                    } else {
+                        listener.onFailure(ex);
+                    }
+                }));
+            },
+            listener::onFailure));
 
     }
     
-    public static List<EsIndex> separateMappings(DataTypeRegistry typeRegistry, String indexPattern, String javaRegex, String[] indexNames,
-            Map<String, Map<String, FieldCapabilities>> fieldCaps) {
-        return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, Function.identity(), (s, cap) -> null);
+    private GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse response, boolean includeFrozen) {
+        return new GetAliasesRequest()
+                .local(true)
+                .aliases("*")
+                .indices(response.getIndices())
+                .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS);
+    }
+    
+    public static List<EsIndex> separateMappings(DataTypeRegistry typeRegistry, String javaRegex, String[] indexNames,
+            Map<String, Map<String, FieldCapabilities>> fieldCaps, ImmutableOpenMap<String, List<AliasMetaData>> aliases) {
+        return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null);
     }
     
     private static class Fields {
@@ -496,16 +515,27 @@ private static class Fields {
      * each field.
      */
     private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[] indexNames, String javaRegex,
-            Map<String, Map<String, FieldCapabilities>> fieldCaps,
+            Map<String, Map<String, FieldCapabilities>> fieldCaps, ImmutableOpenMap<String, List<AliasMetaData>> aliases,
             Function<String, String> indexNameProcessor,
             BiFunction<String, Map<String, FieldCapabilities>, InvalidMappedField> validityVerifier) {
 
-        if (indexNames == null || indexNames.length == 0) {
+        if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.isEmpty())) {
             return emptyList();
         }
 
-        final List<String> resolvedIndices = asList(indexNames);
-        Map<String, Fields> indices = new LinkedHashMap<>(resolvedIndices.size());
+        Set<String> resolvedAliases = new HashSet<>();
+        if (aliases != null) {
+            Iterator<ObjectObjectCursor<String, List<AliasMetaData>>> iterator = aliases.iterator();
+            while (iterator.hasNext()) {
+                for (AliasMetaData alias : iterator.next().value) {
+                    resolvedAliases.add(alias.getAlias());
+                }
+            }
+        }
+
+        List<String> resolvedIndices = new ArrayList<>(asList(indexNames));
+        int mapSize = CollectionUtils.mapSize(resolvedIndices.size() + resolvedAliases.size());
+        Map<String, Fields> indices = new LinkedHashMap<>(mapSize);
         Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null;
 
         // sort fields in reverse order to build the field hierarchy
@@ -525,6 +555,8 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
             Map<String, FieldCapabilities> types = new LinkedHashMap<>(entry.getValue());
             // apply verification and possibly remove the "duplicate" CONSTANT_KEYWORD field type
             final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types);
+            // apply verification for fields belonging to index aliases
+            Map<String, InvalidMappedField> invalidFieldsForAliases = getInvalidFieldsForAliases(fieldName, types, aliases);
 
             // filter meta fields and unmapped
             FieldCapabilities unmapped = types.get(UNMAPPED);
@@ -545,7 +577,7 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
                 List<String> concreteIndices = null;
                 if (capIndices != null) {
                     if (unmappedIndices.isEmpty()) {
-                        concreteIndices = asList(capIndices);
+                        concreteIndices = new ArrayList<>(asList(capIndices));
                     } else {
                         concreteIndices = new ArrayList<>(capIndices.length);
                         for (String capIndex : capIndices) {
@@ -559,30 +591,55 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
                     concreteIndices = resolvedIndices;
                 }
 
+                // add to the list of concrete indices the aliases associated with these indices
+                Set<String> uniqueAliases = new LinkedHashSet<>();
+                if (aliases != null) {
+                    for (String concreteIndex : concreteIndices) {
+                        if (aliases.containsKey(concreteIndex)) {
+                            List<AliasMetaData> concreteIndexAliases = aliases.get(concreteIndex);
+                            concreteIndexAliases.stream().forEach(e -> uniqueAliases.add(e.alias()));
+                        }
+                    }
+                    concreteIndices.addAll(uniqueAliases);
+                }
+
                 // put the field in their respective mappings
                 for (String index : concreteIndices) {
-                    if (pattern == null || pattern.matcher(index).matches()) {
-                        String indexName = indexNameProcessor.apply(index);
+                    boolean isIndexAlias = uniqueAliases.contains(index);
+                    if (pattern == null || pattern.matcher(index).matches() || isIndexAlias) {
+                        String indexName = isIndexAlias ? index : indexNameProcessor.apply(index);
                         Fields indexFields = indices.get(indexName);
                         if (indexFields == null) {
                             indexFields = new Fields();
                             indices.put(indexName, indexFields);
                         }
                         EsField field = indexFields.flattedMapping.get(fieldName);
-                        if (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false)) {
+                        boolean createField = false;
+                        if (isIndexAlias == false) {
+                            if (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false)) {
+                                createField = true;
+                            }
+                        }
+                        else {
+                            if (field == null && invalidFieldsForAliases.get(index) == null) {
+                                createField = true;
+                            }
+                        }
+                        
+                        if (createField) {
                             int dot = fieldName.lastIndexOf('.');
                             /*
                              * Looking up the "tree" at the parent fields here to see if the field is an alias.
                              * When the upper elements of the "tree" have no elements in fieldcaps, then this is an alias field. But not
                              * always: if there are two aliases - a.b.c.alias1 and a.b.c.alias2 - only one of them will be considered alias.
                              */
-                            Holder<Boolean> isAlias = new Holder<>(false);
+                            Holder<Boolean> isAliasFieldType = new Holder<>(false);
                             if (dot >= 0) {
                                 String parentName = fieldName.substring(0, dot);
                                 if (indexFields.flattedMapping.get(parentName) == null) {
                                     // lack of parent implies the field is an alias
                                     if (fieldCaps.get(parentName) == null) {
-                                        isAlias.set(true);
+                                        isAliasFieldType.set(true);
                                     }
                                 }
                             }
@@ -590,7 +647,7 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
                             createField(typeRegistry, fieldName, fieldCaps, indexFields.hierarchicalMapping, indexFields.flattedMapping,
                                     s -> invalidField != null ? invalidField :
                                         createField(typeRegistry, s, typeCap.getType(), emptyMap(), typeCap.isAggregatable(),
-                                                isAlias.get()));
+                                                isAliasFieldType.get()));
                         }
                     }
                 }
@@ -605,4 +662,141 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
         foundIndices.sort(Comparator.comparing(EsIndex::name));
         return foundIndices;
     }
+
+
+    /*
+     * Checks if the field is valid (same type and same capabilities - searchable/aggregatable) across indices belonging to a list
+     * of aliases.
+     * A field can look like the example below (generated by field_caps API).
+     *   "name": {
+     *       "text": {
+     *           "type": "text",
+     *           "searchable": false,
+     *           "aggregatable": false,
+     *           "indices": [
+     *               "bar",
+     *               "foo"
+     *           ],
+     *           "non_searchable_indices": [
+     *               "foo"
+     *           ]
+     *       },
+     *       "keyword": {
+     *           "type": "keyword",
+     *           "searchable": false,
+     *           "aggregatable": true,
+     *           "non_aggregatable_indices": [
+     *               "bar", "baz"
+     *           ]
+     *       }
+     *   }
+     */
+    private static Map<String, InvalidMappedField> getInvalidFieldsForAliases(String fieldName, Map<String, FieldCapabilities> types,
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases) {
+        if (aliases == null || aliases.isEmpty()) {
+            return emptyMap();
+        }
+        Map<String, InvalidMappedField> invalidFields = new HashMap<>();
+        Map<String, Set<String>> typesErrors = new HashMap<>(); // map holding aliases and a list of unique field types across its indices
+        Map<String, Set<String>> aliasToIndices = new HashMap<>(); // map with aliases and their list of indices
+        
+        Iterator<ObjectObjectCursor<String, List<AliasMetaData>>> iter = aliases.iterator();
+        while (iter.hasNext()) {
+            ObjectObjectCursor<String, List<AliasMetaData>> index = iter.next();
+            for (AliasMetaData aliasMetaData : index.value) {
+                String aliasName = aliasMetaData.alias();
+                aliasToIndices.putIfAbsent(aliasName, new HashSet<>());
+                aliasToIndices.get(aliasName).add(index.key);
+            }
+        }
+        
+        // iterate over each type
+        for (Entry<String, FieldCapabilities> type : types.entrySet()) {
+            String esFieldType = type.getKey();
+            if (esFieldType == UNMAPPED) {
+                continue;
+            }
+            String[] indices = type.getValue().indices();
+            // if there is a list of indices where this field type is defined
+            if (indices != null) {
+                // Look at all these indices' aliases and add the type of the field to a list (Set) with unique elements.
+                // A valid mapping for a field in an index alias should contain only one type. If it doesn't, this means that field
+                // is mapped as different types across the indices in this index alias.
+                for (String index : indices) {
+                    List<AliasMetaData> indexAliases = aliases.get(index);
+                    if (indexAliases == null) {
+                        continue;
+                    }
+                    for (AliasMetaData aliasMetaData : indexAliases) {
+                        String aliasName = aliasMetaData.alias();
+                        if (typesErrors.containsKey(aliasName)) {
+                            typesErrors.get(aliasName).add(esFieldType);
+                        } else {
+                            Set<String> fieldTypes = new HashSet<>();
+                            fieldTypes.add(esFieldType);
+                            typesErrors.put(aliasName, fieldTypes);
+                        }
+                    }
+                }
+            }
+        }
+        
+        for (String aliasName : aliasToIndices.keySet()) {
+            // if, for the same index alias, there are multiple field types for this fieldName ie the index alias has indices where the same
+            // field name is of different types
+            Set<String> esFieldTypes = typesErrors.get(aliasName);
+            if (esFieldTypes != null && esFieldTypes.size() > 1) {
+                // consider the field as invalid, for the currently checked index alias
+                // the error message doesn't actually matter
+                invalidFields.put(aliasName, new InvalidMappedField(fieldName));
+            } else {
+                // if the field type is the same across all this alias' indices, check the field's capabilities (searchable/aggregatable)
+                for (Entry<String, FieldCapabilities> type : types.entrySet()) {
+                    if (type.getKey() == UNMAPPED) {
+                        continue;
+                    }
+                    FieldCapabilities f = type.getValue();
+
+                    // the existence of a list of non_aggregatable_indices is an indication that not all indices have the same capabilities
+                    // but this list can contain indices belonging to other aliases, so we need to check only for this alias
+                    if (f.nonAggregatableIndices() != null) {
+                        Set<String> aliasIndices = aliasToIndices.get(aliasName);
+                        int nonAggregatableCount = 0;
+                        // either all or none of the non-aggregatable indices belonging to a certain alias should be in this list
+                        for (String nonAggIndex : f.nonAggregatableIndices()) {
+                            if (aliasIndices.contains(nonAggIndex)) {
+                                nonAggregatableCount++;
+                            }
+                        }
+                        if (nonAggregatableCount > 0 && nonAggregatableCount != aliasIndices.size()) {
+                            invalidFields.put(aliasName, new InvalidMappedField(fieldName));
+                            break;
+                        }
+                    }
+
+                    // perform the same check for non_searchable_indices list
+                    if (f.nonSearchableIndices() != null) {
+                        Set<String> aliasIndices = aliasToIndices.get(aliasName);
+                        int nonSearchableCount = 0;
+                        // either all or none of the non-searchable indices belonging to a certain alias should be in this list
+                        for (String nonSearchIndex : f.nonSearchableIndices()) {
+                            if (aliasIndices.contains(nonSearchIndex)) {
+                                nonSearchableCount++;
+                            }
+                        }
+                        if (nonSearchableCount > 0 && nonSearchableCount != aliasIndices.size()) {
+                            invalidFields.put(aliasName, new InvalidMappedField(fieldName));
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+
+        if (invalidFields.size() > 0) {
+            return invalidFields;
+        }
+        // everything checks
+        return emptyMap();
+    }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java
index c69407a25598b..29c633759ef26 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java
@@ -25,6 +25,11 @@ public InvalidMappedField(String name, String errorMessage) {
         this.errorMessage = errorMessage;
     }
 
+    public InvalidMappedField(String name) {
+        super(name, DataTypes.UNSUPPORTED, emptyMap(), false);
+        this.errorMessage = StringUtils.EMPTY;
+    }
+
     public String errorMessage() {
         return errorMessage;
     }
diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java
new file mode 100644
index 0000000000000..b2568d970bca1
--- /dev/null
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.qa.single_node;
+
+import org.elasticsearch.xpack.sql.qa.jdbc.SysColumnsTestCase;
+
+public class SysColumnsIT extends SysColumnsTestCase {
+
+}
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java
new file mode 100644
index 0000000000000..12a30834caf9a
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java
@@ -0,0 +1,358 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.elasticsearch.xpack.sql.qa.jdbc.ResultSetTestCase.updateMapping;
+
+public class SysColumnsTestCase extends JdbcIntegrationTestCase {
+    
+    public void testAliasWithIncompatibleTypes() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("value").field("type", "double").endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "text").endObject();
+            builder.startObject("value").field("type", "double").endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); 
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"      ,"id"      ,"KEYWORD"},
+            {"test1"      ,"value"   ,"DOUBLE"},
+            {"test2"      ,"id"      ,"TEXT"},
+            {"test2"      ,"value"   ,"DOUBLE"},
+            {"test_alias" ,"value"   ,"DOUBLE"}
+        });
+    }
+
+    public void testAliasWithIncompatibleSearchableProperty() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("value").field("type", "boolean").endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "keyword").field("index", false).endObject();
+            builder.startObject("value").field("type", "boolean").endObject();
+        });
+        
+        createIndexWithMapping("test3", builder -> {
+            builder.startObject("id").field("type", "keyword").field("index", false).endObject();
+            builder.startObject("value").field("type", "boolean").endObject();
+        });
+        
+        createIndexWithMapping("test4", builder -> {
+            builder.startObject("id").field("type", "keyword").field("index", false).endObject();
+            builder.startObject("value").field("type", "boolean").endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test3").field("alias", "test_alias2").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test4").field("alias", "test_alias2").endObject().endObject();
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"       ,"id"      ,"KEYWORD"},
+            {"test1"       ,"value"   ,"BOOLEAN"},
+            {"test2"       ,"id"      ,"KEYWORD"},
+            {"test2"       ,"value"   ,"BOOLEAN"},
+            {"test3"       ,"id"      ,"KEYWORD"},
+            {"test3"       ,"value"   ,"BOOLEAN"},
+            {"test4"       ,"id"      ,"KEYWORD"},
+            {"test4"       ,"value"   ,"BOOLEAN"},
+            {"test_alias"  ,"value"   ,"BOOLEAN"},
+            {"test_alias2" ,"id"      ,"KEYWORD"},
+            {"test_alias2" ,"value"   ,"BOOLEAN"}
+        });
+    }
+    
+    public void testAliasWithIncompatibleAggregatableProperty() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "text").field("fielddata", true).endObject();
+            builder.startObject("value").field("type", "date").endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "text").endObject();
+            builder.startObject("value").field("type", "date").endObject();
+        });
+        
+        createIndexWithMapping("test3", builder -> {
+            builder.startObject("id").field("type", "text").field("fielddata", true).endObject();
+            builder.startObject("value").field("type", "date").endObject();
+        });
+        
+        createIndexWithMapping("test4", builder -> {
+            builder.startObject("id").field("type", "text").field("fielddata", true).endObject();
+            builder.startObject("value").field("type", "date").endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test3").field("alias", "test_alias2").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test4").field("alias", "test_alias2").endObject().endObject();
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"      ,"id"      ,"TEXT"},
+            {"test1"      ,"value"   ,"DATETIME"},
+            {"test2"      ,"id"      ,"TEXT"},
+            {"test2"      ,"value"   ,"DATETIME"},
+            {"test3"      ,"id"      ,"TEXT"},
+            {"test3"      ,"value"   ,"DATETIME"},
+            {"test4"      ,"id"      ,"TEXT"},
+            {"test4"      ,"value"   ,"DATETIME"},
+            {"test_alias" ,"value"   ,"DATETIME"},
+            {"test_alias2","id"      ,"TEXT"},
+            {"test_alias2","value"   ,"DATETIME"},
+        });
+    }
+
+    public void testAliasWithIncompatibleTypesInSubfield() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "keyword")
+                .endObject().endObject().endObject();
+            builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long")
+                .endObject().endObject().endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer")
+                .endObject().endObject().endObject();
+            builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long")
+                .endObject().endObject().endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); 
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"      ,"id"       ,"TEXT"},
+            {"test1"      ,"id.raw"   ,"KEYWORD"},
+            {"test1"      ,"value"    ,"DATETIME"},
+            {"test1"      ,"value.raw","LONG"},
+            {"test2"      ,"id"       ,"TEXT"},
+            {"test2"      ,"id.raw"   ,"INTEGER"},
+            {"test2"      ,"value"    ,"DATETIME"},
+            {"test2"      ,"value.raw","LONG"},
+            {"test_alias" ,"id"       ,"TEXT"},
+            {"test_alias" ,"value"    ,"DATETIME"},
+            {"test_alias" ,"value.raw","LONG"},
+        });
+    }
+
+    public void testAliasWithIncompatibleSearchablePropertyInSubfield() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer")
+                .endObject().endObject().endObject();
+            builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long")
+                .endObject().endObject().endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer")
+                .field("index", false).endObject().endObject().endObject();
+            builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long")
+                .endObject().endObject().endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); 
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"      ,"id"       ,"TEXT"},
+            {"test1"      ,"id.raw"   ,"INTEGER"},
+            {"test1"      ,"value"    ,"DATETIME"},
+            {"test1"      ,"value.raw","LONG"},
+            {"test2"      ,"id"       ,"TEXT"},
+            {"test2"      ,"id.raw"   ,"INTEGER"},
+            {"test2"      ,"value"    ,"DATETIME"},
+            {"test2"      ,"value.raw","LONG"},
+            {"test_alias" ,"id"       ,"TEXT"},
+            {"test_alias" ,"value"    ,"DATETIME"},
+            {"test_alias" ,"value.raw","LONG"},
+        });
+    }
+    
+    public void testAliasWithIncompatibleAggregatablePropertyInSubfield() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer")
+                .field("doc_values", false).endObject().endObject().endObject();
+            builder.startObject("value").field("type", "ip").startObject("fields").startObject("raw").field("type", "text")
+                .endObject().endObject().endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer")
+                .endObject().endObject().endObject();
+            builder.startObject("value").field("type", "ip").startObject("fields").startObject("raw").field("type", "text")
+                .endObject().endObject().endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); 
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"      ,"id"       ,"TEXT"},
+            {"test1"      ,"id.raw"   ,"INTEGER"},
+            {"test1"      ,"value"    ,"IP"},
+            {"test1"      ,"value.raw","TEXT"},
+            {"test2"      ,"id"       ,"TEXT"},
+            {"test2"      ,"id.raw"   ,"INTEGER"},
+            {"test2"      ,"value"    ,"IP"},
+            {"test2"      ,"value.raw","TEXT"},
+            {"test_alias" ,"id"       ,"TEXT"},
+            {"test_alias" ,"value"    ,"IP"},
+            {"test_alias" ,"value.raw","TEXT"},
+        });
+    }
+    
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53445")
+    public void testAliasWithSubfieldsAndDifferentRootFields() throws Exception {
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("name").field("type", "text").startObject("fields").startObject("raw").field("type", "keyword")
+                .endObject().endObject().endObject();
+        });
+        
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("name").field("type", "keyword").field("index", false).startObject("fields").startObject("raw")
+                .field("type", "keyword").endObject().endObject().endObject();
+        });
+
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); 
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"test1"      ,"id"      ,"KEYWORD"},
+            {"test1"      ,"name"    ,"TEXT"},
+            {"test1"      ,"name.raw","KEYWORD"},
+            {"test2"      ,"id"      ,"KEYWORD"},
+            {"test2"      ,"name"    ,"KEYWORD"},
+            {"test2"      ,"name.raw","KEYWORD"},
+            {"test_alias" ,"id"      ,"KEYWORD"}
+        });
+    }
+    
+    public void testMultiIndicesMultiAlias() throws Exception {
+        createIndexWithMapping("test2", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("name").field("type", "text").endObject();
+        });
+        createIndexWithMapping("test4", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("name").field("type", "text").field("index", false).endObject();
+        });
+        createIndexWithMapping("test1", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("name").field("type", "keyword").endObject();
+            builder.startObject("number").field("type", "long").endObject();
+        });
+        createIndexWithMapping("test3", builder -> {
+            builder.startObject("id").field("type", "keyword").endObject();
+            builder.startObject("name").field("type", "keyword").endObject();
+            builder.startObject("number").field("type", "long").endObject();
+        });
+        
+        createAliases(builder -> {
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "alias1").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test1").field("alias", "alias2").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "alias2").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test2").field("alias", "alias3").endObject().endObject();
+            builder.startObject().startObject("add").field("index", "test4").field("alias", "alias3").endObject().endObject();
+        });
+        
+        assertResultsForQuery("SYS COLUMNS", new String[][] {
+            {"alias1","id"    ,"KEYWORD"},
+            {"alias1","name"  ,"KEYWORD"},
+            {"alias1","number","LONG"},
+            {"alias2","id"    ,"KEYWORD"},
+            {"alias2","number","LONG"},
+            {"alias3","id"    ,"KEYWORD"},
+            {"test1" ,"id"    ,"KEYWORD"},
+            {"test1" ,"name"  ,"KEYWORD"},
+            {"test1" ,"number","LONG"},
+            {"test2" ,"id"    ,"KEYWORD"},
+            {"test2" ,"name"  ,"TEXT"},
+            {"test3" ,"id"    ,"KEYWORD"},
+            {"test3" ,"name"  ,"KEYWORD"},
+            {"test3" ,"number","LONG"},
+            {"test4" ,"id"    ,"KEYWORD"},
+            {"test4" ,"name"  ,"TEXT"}
+        });
+    }
+    
+    private static void createIndexWithMapping(String indexName, CheckedConsumer<XContentBuilder, IOException> mapping) throws Exception {
+        ResultSetTestCase.createIndex(indexName);
+        updateMapping(indexName, mapping);
+    }
+    
+    private void doWithQuery(String query, CheckedConsumer<ResultSet, SQLException> consumer) throws SQLException {
+        try (Connection connection = esJdbc()) {
+            try (PreparedStatement statement = connection.prepareStatement(query)) {
+                try (ResultSet results = statement.executeQuery()) {
+                    consumer.accept(results);
+                }
+            }
+        }
+    }
+    
+    private static void createAliases(CheckedConsumer<XContentBuilder, IOException> definitions) throws Exception {
+        Request request = new Request("POST", "/_aliases");
+        XContentBuilder createAliases = JsonXContent.contentBuilder().startObject();
+        createAliases.startArray("actions");
+        {
+            definitions.accept(createAliases);
+        }
+        createAliases.endArray();
+        createAliases.endObject();
+        request.setJsonEntity(Strings.toString(createAliases));
+        client().performRequest(request);
+    }
+    
+    private void assertResultsForQuery(String query, String[][] rows) throws Exception {
+        doWithQuery(query, (results) -> {
+            for (String[] row : rows) {
+                results.next();
+                assertEquals(row[0], results.getString(3)); // table name
+                assertEquals(row[1], results.getString(4)); // column name
+                assertEquals(row[2], results.getString(6)); // type name
+            }
+            assertFalse(results.next());
+        });
+    }
+}
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
index dee74303b9bda..4bc099d882142 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
@@ -90,38 +90,66 @@ integTest   |null           |test_alias    |salary            |4              |I
 sysColumnsAllTables
 SYS COLUMNS TABLE LIKE '%';
 
-   TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s  |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
----------------+---------------+---------------+------------------+---------------+----------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------
-integTest      |null           |logs           |@timestamp        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_in          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |2                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |bytes_out         |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_ip         |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |client_port       |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |dest_ip           |12             |IP              |45             |45              |null            |null             |1          |null           |null           |12             |0                 |null               |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |id                |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |logs           |status            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp       |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |birth_date        |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |1                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |emp_no            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |3                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |7                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |extra_no          |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |8                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name        |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |9                 |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |10                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |gender            |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |11                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |hire_date         |93             |DATETIME        |29             |8               |null            |null             |1          |null           |null           |9              |3                 |null               |12                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |languages         |-6             |BYTE            |5              |1               |null            |10               |1          |null           |null           |-6             |0                 |null               |13                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name         |12             |TEXT            |2147483647     |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |14                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD         |32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |15                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647      |null            |null             |1          |null           |null           |12             |0                 |2147483647         |16                |YES          |null           |null           |null           |null            |NO                |NO
-integTest      |null           |test_emp_copy  |salary            |4              |INTEGER         |11             |4               |null            |10               |1          |null           |null           |4              |0                 |null               |17                |YES          |null           |null           |null           |null            |NO                |NO
+    TABLE_CAT:s  |  TABLE_SCHEM:s|  TABLE_NAME:s |  COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s  |  COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
+-----------------+---------------+---------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
+integTest        |null           |logs           |@timestamp        |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |bytes_in          |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |2               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |bytes_out         |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |client_ip         |12             |IP              |45             |45             |null            |null           |1              |null           |null           |12             |0               |null             |4               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |client_port       |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |5               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |dest_ip           |12             |IP              |45             |45             |null            |null           |1              |null           |null           |12             |0               |null             |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |id                |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |logs           |status            |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |birth_date        |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |emp_no            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |extra_no          |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |first_name        |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |gender            |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |hire_date         |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |languages         |-6             |BYTE            |5              |1              |null            |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |last_name         |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias     |salary            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |birth_date        |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |emp_no            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |extra_no          |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |first_name        |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |gender            |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |hire_date         |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |languages         |-6             |BYTE            |5              |1              |null            |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |last_name         |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_alias_emp |salary            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |birth_date        |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |emp_no            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |first_name        |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |4               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |5               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |gender            |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |hire_date         |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |languages         |-6             |BYTE            |5              |1              |null            |10             |1              |null           |null           |-6             |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |last_name         |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp       |salary            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |birth_date        |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |emp_no            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |extra_gender      |12             |CONSTANT_KEYWORD|32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |extra_no          |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |first_name        |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |gender            |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |hire_date         |93             |DATETIME        |29             |8              |null            |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |languages         |-6             |BYTE            |5              |1              |null            |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |last_name         |12             |TEXT            |2147483647     |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD         |32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |null_constant     |12             |CONSTANT_KEYWORD|32766          |2147483647     |null            |null           |1              |null           |null           |12             |0               |2147483647       |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest        |null           |test_emp_copy  |salary            |4              |INTEGER         |11             |4              |null            |10             |1              |null           |null           |4              |0               |null             |17              |YES            |null           |null           |null           |null            |NO              |NO                                
 ;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
index 614cc24ec44aa..d69384f32e53f 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
@@ -375,7 +375,7 @@ public static IndexResolution merge(EsIndex... indices) {
     }
 
     public static List<EsIndex> separate(EsIndex... indices) {
-        return separateMappings("*", null, Stream.of(indices).map(EsIndex::name).toArray(String[]::new),
+        return separateMappings(null, Stream.of(indices).map(EsIndex::name).toArray(String[]::new),
                 fromMappings(indices));
     }
 
@@ -484,8 +484,8 @@ private static IndexResolution mergedMappings(String indexPattern, String[] inde
         return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps);
     }
     
-    private static List<EsIndex> separateMappings(String indexPattern, String javaRegex, String[] indexNames,
+    private static List<EsIndex> separateMappings(String javaRegex, String[] indexNames,
             Map<String, Map<String, FieldCapabilities>> fieldCaps) {
-        return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, javaRegex, indexNames, fieldCaps);
+        return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null);
     }
 }

From ff94792e41689ecd075c7eae7b73529d1b548346 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 17 Mar 2020 10:54:44 +0100
Subject: [PATCH 072/285] Shortcut query phase using the results of other
 shards (#51852)

This commit, built on top of #51708, allows to modify shard search requests based on informations collected on other shards. It is intended to speed up sorted queries on time-based indices. For queries that are only interested in the top documents.

This change will rewrite the shard queries to match none if the bottom sort value computed in prior shards is better than all values in the shard.
For queries that mix top documents and aggregations this change will reset the size of the top documents to 0 instead of rewriting to match none.
This means that we don't need to keep a search context open for this shard since we know in advance that it doesn't contain any competitive hit.
---
 .../BulkByScrollParallelizationHelper.java    |   2 +-
 .../search/140_pre_filter_search_shards.yml   |  52 +++-
 .../search/AbstractSearchAsyncAction.java     |   4 +-
 .../search/BottomSortValuesCollector.java     | 111 ++++++++
 .../action/search/SearchPhaseController.java  |  14 +-
 .../SearchQueryThenFetchAsyncAction.java      |  54 +++-
 .../action/search/SearchRequest.java          |  13 +
 .../elasticsearch/search/DocValueFormat.java  |   4 +
 .../search/SearchSortValuesAndFormats.java    | 107 +++++++
 .../search/builder/SearchSourceBuilder.java   |   6 +-
 .../search/internal/ShardSearchRequest.java   |  50 ++++
 .../search/query/TopDocsCollectorContext.java |  20 +-
 .../search/sort/FieldSortBuilder.java         |  52 ++++
 .../BottomSortValuesCollectorTests.java       | 263 ++++++++++++++++++
 .../SearchQueryThenFetchAsyncActionTests.java | 157 +++++++++++
 .../SearchSortValuesAndFormatsTests.java      |  99 +++++++
 .../internal/ShardSearchRequestTests.java     |   4 +
 .../search/sort/FieldSortBuilderTests.java    |  54 ++++
 18 files changed, 1036 insertions(+), 30 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/SearchSortValuesAndFormats.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/SearchSortValuesAndFormatsTests.java

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
index fbcb3d51efac3..6f1948ad70167 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
@@ -203,7 +203,7 @@ static SearchRequest[] sliceIntoSubRequests(SearchRequest request, String field,
                 if (request.source().slice() != null) {
                     throw new IllegalStateException("Can't slice a request that already has a slice configuration");
                 }
-                slicedSource = request.source().copyWithNewSlice(sliceBuilder);
+                slicedSource = request.source().shallowCopy().slice(sliceBuilder);
             }
             SearchRequest searchRequest = new SearchRequest(request);
             searchRequest.source(slicedSource);
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml
index 636f3d88e10cf..15cea782c4679 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml
@@ -19,7 +19,7 @@ setup:
             mappings:
               properties:
                 created_at:
-                   type: date
+                   type: date_nanos
                    format: "yyyy-MM-dd"
   - do:
       indices.create:
@@ -154,10 +154,9 @@ setup:
   - match: { hits.total: 2 }
   - length: { aggregations.idx_terms.buckets: 2 }
 
-    # check that empty responses are correctly handled when rewriting to match_no_docs
+  # check that empty responses are correctly handled when rewriting to match_no_docs
   - do:
       search:
-        rest_total_hits_as_int: true
         # ensure that one shard can return empty response
         max_concurrent_shard_requests: 1
         body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"}}}, "aggs" : { "idx_terms" :  { "terms" : { "field" : "_index" } } } }
@@ -166,12 +165,11 @@ setup:
   - match: { _shards.successful: 3 }
   - match: { _shards.skipped : 0 }
   - match: { _shards.failed: 0 }
-  - match: { hits.total: 2 }
+  - match: { hits.total.value: 2 }
   - length: { aggregations.idx_terms.buckets: 2 }
 
   - do:
       search:
-        rest_total_hits_as_int: true
         # ensure that one shard can return empty response
         max_concurrent_shard_requests: 2
         body: { "size" : 0, "query" : { "range" : { "created_at" : { "gte" : "2019-02-01"}}}, "aggs" : { "idx_terms" :  { "terms" : { "field" : "_index" } } } }
@@ -180,5 +178,47 @@ setup:
   - match: { _shards.successful: 3 }
   - match: { _shards.skipped : 0 }
   - match: { _shards.failed: 0 }
-  - match: { hits.total: 0 }
+  - match: { hits.total.value: 0 }
   - length: { aggregations.idx_terms.buckets: 0 }
+
+  # check field sort is correct when skipping query phase
+  - do:
+      search:
+        # ensure that one shard can return empty response
+        max_concurrent_shard_requests: 1
+        pre_filter_shard_size: 1
+        body:
+          "size": 1
+          "track_total_hits": 1
+          "sort": [{ "created_at": { "order": "desc", "numeric_type": "date" } }]
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped: 0 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: "gte" }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "3" }
+
+  # same with aggs
+  - do:
+      search:
+        # ensure that one shard can return empty response
+        max_concurrent_shard_requests: 1
+        pre_filter_shard_size: 1
+        body:
+          "size": 1
+          "track_total_hits": 1
+          "sort": [{ "created_at": { "order": "desc", "numeric_type": "date" } }]
+          "aggs" : { "idx_terms" :  { "terms" : { "field" : "_index" } } }
+
+  - match: { _shards.total: 3 }
+  - match: { _shards.successful: 3 }
+  - match: { _shards.skipped: 0 }
+  - match: { _shards.failed: 0 }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: "gte" }
+  - length: { hits.hits: 1 }
+  - match: {hits.hits.0._id: "3" }
+  - length: { aggregations.idx_terms.buckets: 3 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 1d2339dd9e6fe..f6f90cc7ccbb3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -77,7 +77,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
      **/
     private final BiFunction<String, String, Transport.Connection> nodeIdToConnection;
     private final SearchTask task;
-    private final SearchPhaseResults<Result> results;
+    final SearchPhaseResults<Result> results;
     private final ClusterState clusterState;
     private final Map<String, AliasFilter> aliasFilter;
     private final Map<String, Float> concreteIndexBoosts;
@@ -467,7 +467,7 @@ public final void onShardFailure(final int shardIndex, @Nullable SearchShardTarg
      * @param result the result returned form the shard
      * @param shardIt the shard iterator
      */
-    private void onShardResult(Result result, SearchShardIterator shardIt) {
+    protected void onShardResult(Result result, SearchShardIterator shardIt) {
         assert result.getShardIndex() != -1 : "shard index is not set";
         assert result.getSearchShardTarget() != null : "search shard target must not be null";
         successfulOps.incrementAndGet();
diff --git a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java
new file mode 100644
index 0000000000000..06df5cd6123b7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchSortValuesAndFormats;
+
+/**
+ * Utility class to keep track of the bottom doc's sort values in a distributed search.
+ */
+class BottomSortValuesCollector {
+    private final int topNSize;
+    private final SortField[] sortFields;
+    private final FieldComparator[] comparators;
+    private final int[] reverseMuls;
+
+    private volatile long totalHits;
+    private volatile SearchSortValuesAndFormats bottomSortValues;
+
+    BottomSortValuesCollector(int topNSize, SortField[] sortFields) {
+        this.topNSize = topNSize;
+        this.comparators = new FieldComparator[sortFields.length];
+        this.reverseMuls = new int[sortFields.length];
+        this.sortFields = sortFields;
+        for (int i = 0; i < sortFields.length; i++) {
+            comparators[i] = sortFields[i].getComparator(1, i);
+            reverseMuls[i] = sortFields[i].getReverse() ? -1 : 1;
+        }
+    }
+
+    long getTotalHits() {
+        return totalHits;
+    }
+
+    /**
+     * @return The best bottom sort values consumed so far.
+     */
+    SearchSortValuesAndFormats getBottomSortValues() {
+        return bottomSortValues;
+    }
+
+    synchronized void consumeTopDocs(TopFieldDocs topDocs, DocValueFormat[] sortValuesFormat) {
+        totalHits += topDocs.totalHits.value;
+        if (validateShardSortFields(topDocs.fields) == false) {
+            return;
+        }
+
+        FieldDoc shardBottomDoc = extractBottom(topDocs);
+        if (shardBottomDoc == null) {
+            return;
+        }
+        if (bottomSortValues == null
+                || compareValues(shardBottomDoc.fields, bottomSortValues.getRawSortValues()) < 0) {
+            bottomSortValues = new SearchSortValuesAndFormats(shardBottomDoc.fields, sortValuesFormat);
+        }
+    }
+
+    /**
+     * @return <code>false</code> if the provided {@link SortField} array differs
+     * from the initial {@link BottomSortValuesCollector#sortFields}.
+     */
+    private boolean validateShardSortFields(SortField[] shardSortFields) {
+        for (int i = 0; i  < shardSortFields.length; i++) {
+            if (shardSortFields[i].equals(sortFields[i]) == false) {
+                // ignore shards response that would make the sort incompatible
+                // (e.g.: mixing keyword/numeric or long/double).
+                // TODO: we should fail the entire request because the topdocs
+                //  merge will likely fail later but this is not possible with
+                //  the current async logic that only allows shard failures here.
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private FieldDoc extractBottom(TopFieldDocs topDocs) {
+        return topNSize > 0 && topDocs.scoreDocs.length == topNSize ?
+            (FieldDoc) topDocs.scoreDocs[topNSize-1] : null;
+    }
+
+    private int compareValues(Object[] v1, Object[] v2) {
+        for (int i = 0; i < v1.length; i++) {
+            int cmp = reverseMuls[i] * comparators[i].compareValues(v1[i], v2[i]);
+            if (cmp != 0) {
+                return cmp;
+            }
+        }
+        return 0;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index d8a3e26b8d82b..f2cf6b199dcbe 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -478,7 +478,8 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
         for (SearchPhaseResult entry : queryResults) {
             QuerySearchResult result = entry.queryResult();
             from = result.from();
-            size = result.size();
+            // sorted queries can set the size to 0 if they have enough competitive hits.
+            size = Math.max(result.size(), size);
             if (hasSuggest) {
                 assert result.suggest() != null;
                 for (Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> suggestion : result.suggest()) {
@@ -724,15 +725,6 @@ int getNumBuffered() {
         int getNumReducePhases() { return numReducePhases; }
     }
 
-    private int resolveTrackTotalHits(SearchRequest request) {
-        if (request.scroll() != null) {
-            // no matter what the value of track_total_hits is
-            return SearchContext.TRACK_TOTAL_HITS_ACCURATE;
-        }
-        return request.source() == null ? SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : request.source().trackTotalHitsUpTo() == null ?
-                SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : request.source().trackTotalHitsUpTo();
-    }
-
     /**
      * Returns a new ArraySearchPhaseResults instance. This might return an instance that reduces search responses incrementally.
      */
@@ -743,7 +735,7 @@ ArraySearchPhaseResults<SearchPhaseResult> newSearchPhaseResults(SearchProgressL
         boolean isScrollRequest = request.scroll() != null;
         final boolean hasAggs = source != null && source.aggregations() != null;
         final boolean hasTopDocs = source == null || source.size() != 0;
-        final int trackTotalHitsUpTo = resolveTrackTotalHits(request);
+        final int trackTotalHitsUpTo = request.resolveTrackTotalHitsUpTo();
         InternalAggregation.ReduceContextBuilder aggReduceContextBuilder = requestToAggReduceContextBuilder.apply(request);
         if (isScrollRequest == false && (hasAggs || hasTopDocs)) {
             // no incremental reduce if scroll is used - we only hit a single shard or sometimes more...
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 24345c606003a..e42d8405da5b0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.search;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.search.TopFieldDocs;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
@@ -28,6 +29,9 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
 
 import java.util.Map;
@@ -35,11 +39,18 @@
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
 
-final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<SearchPhaseResult> {
+import static org.elasticsearch.action.search.SearchPhaseController.getTopDocsSize;
+
+class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<SearchPhaseResult> {
 
     private final SearchPhaseController searchPhaseController;
     private final SearchProgressListener progressListener;
 
+    // informations to track the best bottom top doc globally.
+    private final int topDocsSize;
+    private final int trackTotalHitsUpTo;
+    private volatile BottomSortValuesCollector bottomSortCollector;
+
     SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService,
                                     final BiFunction<String, String, Transport.Connection> nodeIdToConnection,
                                     final Map<String, AliasFilter> aliasFilter,
@@ -53,9 +64,10 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<Se
                 executor, request, listener, shardsIts, timeProvider, clusterState, task,
                 searchPhaseController.newSearchPhaseResults(task.getProgressListener(), request, shardsIts.size()),
                 request.getMaxConcurrentShardRequests(), clusters);
+        this.topDocsSize = getTopDocsSize(request);
+        this.trackTotalHitsUpTo = request.resolveTrackTotalHitsUpTo();
         this.searchPhaseController = searchPhaseController;
         this.progressListener = task.getProgressListener();
-        final SearchProgressListener progressListener = task.getProgressListener();
         final SearchSourceBuilder sourceBuilder = request.source();
         progressListener.notifyListShards(SearchProgressListener.buildSearchShards(this.shardsIts),
             SearchProgressListener.buildSearchShards(toSkipShardsIts), clusters, sourceBuilder == null || sourceBuilder.size() != 0);
@@ -63,8 +75,9 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<Se
 
     protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard,
                                        final SearchActionListener<SearchPhaseResult> listener) {
+        ShardSearchRequest request = rewriteShardSearchRequest(super.buildShardSearchRequest(shardIt));
         getSearchTransport().sendExecuteQuery(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()),
-            buildShardSearchRequest(shardIt), getTask(), listener);
+            request, getTask(), listener);
     }
 
     @Override
@@ -72,8 +85,43 @@ protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget
         progressListener.notifyQueryFailure(shardIndex, shardTarget, exc);
     }
 
+    @Override
+    protected void onShardResult(SearchPhaseResult result, SearchShardIterator shardIt) {
+        QuerySearchResult queryResult = result.queryResult();
+        if (queryResult.isNull() == false && queryResult.topDocs().topDocs instanceof TopFieldDocs) {
+            TopFieldDocs topDocs = (TopFieldDocs) queryResult.topDocs().topDocs;
+            if (bottomSortCollector == null) {
+                synchronized (this) {
+                    if (bottomSortCollector == null) {
+                        bottomSortCollector = new BottomSortValuesCollector(topDocsSize, topDocs.fields);
+                    }
+                }
+            }
+            bottomSortCollector.consumeTopDocs(topDocs, queryResult.sortValueFormats());
+        }
+        super.onShardResult(result, shardIt);
+    }
+
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
         return new FetchSearchPhase(results, searchPhaseController, context, clusterState());
     }
+
+    private ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) {
+        if (bottomSortCollector == null) {
+            return request;
+        }
+
+        // disable tracking total hits if we already reached the required estimation.
+        if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_ACCURATE
+                && bottomSortCollector.getTotalHits() > trackTotalHitsUpTo) {
+            request.source(request.source().shallowCopy().trackTotalHits(false));
+        }
+
+        // set the current best bottom field doc
+        if (bottomSortCollector.getBottomSortValues() != null) {
+            request.setBottomSortValues(bottomSortCollector.getBottomSortValues());
+        }
+        return request;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 96206aa4bcd58..761a50bfa4168 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -558,6 +558,19 @@ public boolean isSuggestOnly() {
         return source != null && source.isSuggestOnly();
     }
 
+    public int resolveTrackTotalHitsUpTo() {
+        return resolveTrackTotalHitsUpTo(scroll, source);
+    }
+
+    public static int resolveTrackTotalHitsUpTo(Scroll scroll, SearchSourceBuilder source) {
+        if (scroll != null) {
+            // no matter what the value of track_total_hits is
+            return SearchContext.TRACK_TOTAL_HITS_ACCURATE;
+        }
+        return source == null ? SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : source.trackTotalHitsUpTo() == null ?
+            SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : source.trackTotalHitsUpTo();
+    }
+
     @Override
     public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
         // generating description in a lazy way since source can be quite big
diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 7f681bfa47c92..5b45cd660697a 100644
--- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -230,6 +230,10 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
 
+        public DateMathParser getDateMathParser() {
+            return parser;
+        }
+
         @Override
         public String format(long value) {
             return formatter.format(resolution.toInstant(value).atZone(timeZone));
diff --git a/server/src/main/java/org/elasticsearch/search/SearchSortValuesAndFormats.java b/server/src/main/java/org/elasticsearch/search/SearchSortValuesAndFormats.java
new file mode 100644
index 0000000000000..f5f80836cde8f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/SearchSortValuesAndFormats.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.Lucene;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class SearchSortValuesAndFormats implements Writeable {
+    private final Object[] rawSortValues;
+    private final Object[] formattedSortValues;
+    private final DocValueFormat[] sortValueFormats;
+
+    public SearchSortValuesAndFormats(Object[] rawSortValues, DocValueFormat[] sortValueFormats) {
+        assert rawSortValues.length == sortValueFormats.length;
+        this.rawSortValues = rawSortValues;
+        this.sortValueFormats = sortValueFormats;
+        this.formattedSortValues = Arrays.copyOf(rawSortValues, rawSortValues.length);
+        for (int i = 0; i < rawSortValues.length; ++i) {
+            Object sortValue = rawSortValues[i];
+            if (sortValue instanceof BytesRef) {
+                this.formattedSortValues[i] = sortValueFormats[i].format((BytesRef) sortValue);
+            } else if (sortValue instanceof Long) {
+                this.formattedSortValues[i] = sortValueFormats[i].format((long) sortValue);
+            } else if (sortValue instanceof Double) {
+                this.formattedSortValues[i] = sortValueFormats[i].format((double) sortValue);
+            } else if (sortValue instanceof Float || sortValue instanceof Integer) {
+                // sort by _score or _doc
+                this.formattedSortValues[i] = sortValue;
+            } else {
+                assert sortValue == null : "Sort values must be a BytesRef, Long, Integer, Double or Float, but got "
+                    + sortValue.getClass() + ": " + sortValue;
+                this.formattedSortValues[i] = sortValue;
+            }
+        }
+    }
+
+    public SearchSortValuesAndFormats(StreamInput in) throws IOException {
+        this.rawSortValues = in.readArray(Lucene::readSortValue, Object[]::new);
+        this.formattedSortValues = in.readArray(Lucene::readSortValue, Object[]::new);
+        this.sortValueFormats = new DocValueFormat[formattedSortValues.length];
+        for (int i = 0; i < sortValueFormats.length; ++i) {
+            sortValueFormats[i] = in.readNamedWriteable(DocValueFormat.class);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeArray(Lucene::writeSortValue, rawSortValues);
+        out.writeArray(Lucene::writeSortValue, formattedSortValues);
+        for (int i = 0; i < sortValueFormats.length; i++) {
+            out.writeNamedWriteable(sortValueFormats[i]);
+        }
+    }
+
+    public Object[] getRawSortValues() {
+        return rawSortValues;
+    }
+
+    public Object[] getFormattedSortValues() {
+        return formattedSortValues;
+    }
+
+    public DocValueFormat[] getSortValueFormats() {
+        return sortValueFormats;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SearchSortValuesAndFormats that = (SearchSortValuesAndFormats) o;
+        return Arrays.equals(rawSortValues, that.rawSortValues) &&
+            Arrays.equals(formattedSortValues, that.formattedSortValues) &&
+            Arrays.equals(sortValueFormats, that.sortValueFormats);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Arrays.hashCode(rawSortValues);
+        result = 31 * result + Arrays.hashCode(formattedSortValues);
+        result = 31 * result + Arrays.hashCode(sortValueFormats);
+        return result;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index e95845b225164..14c61311bddf6 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -959,13 +959,13 @@ public SearchSourceBuilder rewrite(QueryRewriteContext context) throws IOExcepti
     /**
      * Create a shallow copy of this builder with a new slice configuration.
      */
-    public SearchSourceBuilder copyWithNewSlice(SliceBuilder slice) {
-        return shallowCopy(queryBuilder, postQueryBuilder, aggregations, slice, sorts, rescoreBuilders, highlightBuilder);
+    public SearchSourceBuilder shallowCopy() {
+        return shallowCopy(queryBuilder, postQueryBuilder, aggregations, sliceBuilder, sorts, rescoreBuilders, highlightBuilder);
     }
 
     /**
      * Create a shallow copy of this source replaced {@link #queryBuilder}, {@link #postQueryBuilder}, and {@link #sliceBuilder}. Used by
-     * {@link #rewrite(QueryRewriteContext)} and {@link #copyWithNewSlice(SliceBuilder)}.
+     * {@link #rewrite(QueryRewriteContext)}}.
      */
     @SuppressWarnings("rawtypes")
     private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder postQueryBuilder,
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 412b53f2d2e3e..6c1fc7e959b40 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -39,15 +39,19 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.AliasFilterParsingException;
 import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.search.SearchSortValuesAndFormats;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -57,6 +61,8 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED;
+
 /**
  * Shard level request that represents a search.
  * It provides all the methods that the {@link SearchContext} needs.
@@ -77,6 +83,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque
     private final OriginalIndices originalIndices;
 
     private boolean canReturnNullResponseIfMatchNoDocs;
+    private SearchSortValuesAndFormats bottomSortValues;
 
     //these are the only mutable fields, as they are subject to rewriting
     private AliasFilter aliasFilter;
@@ -175,6 +182,11 @@ public ShardSearchRequest(StreamInput in) throws IOException {
         } else {
             canReturnNullResponseIfMatchNoDocs = false;
         }
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            bottomSortValues = in.readOptionalWriteable(SearchSortValuesAndFormats::new);
+        } else {
+            bottomSortValues = null;
+        }
         originalIndices = OriginalIndices.readOriginalIndices(in);
     }
 
@@ -212,6 +224,9 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
         if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
             out.writeBoolean(canReturnNullResponseIfMatchNoDocs);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(bottomSortValues);
+        }
     }
 
     @Override
@@ -286,6 +301,20 @@ public String preference() {
         return preference;
     }
 
+    /**
+     * Sets the bottom sort values that can be used by the searcher to filter documents
+     * that are after it. This value is computed by coordinating nodes that throttles the
+     * query phase. After a partial merge of successful shards the sort values of the
+     * bottom top document are passed as an hint on subsequent shard requests.
+     */
+    public void setBottomSortValues(SearchSortValuesAndFormats values) {
+        this.bottomSortValues = values;
+    }
+
+    public SearchSortValuesAndFormats getBottomSortValues() {
+        return bottomSortValues;
+    }
+
     /**
      * Returns true if the caller can handle null response {@link QuerySearchResult#nullInstance()}.
      * Defaults to false since the coordinator node needs at least one shard response to build the global
@@ -343,6 +372,27 @@ static class RequestRewritable implements Rewriteable<Rewriteable> {
         public Rewriteable rewrite(QueryRewriteContext ctx) throws IOException {
             SearchSourceBuilder newSource = request.source() == null ? null : Rewriteable.rewrite(request.source(), ctx);
             AliasFilter newAliasFilter = Rewriteable.rewrite(request.getAliasFilter(), ctx);
+
+            QueryShardContext shardContext = ctx.convertToShardContext();
+
+            FieldSortBuilder primarySort = FieldSortBuilder.getPrimaryFieldSortOrNull(newSource);
+            if (shardContext != null
+                    && primarySort != null
+                    && primarySort.isBottomSortShardDisjoint(shardContext, request.getBottomSortValues())) {
+                assert newSource != null : "source should contain a primary sort field";
+                newSource = newSource.shallowCopy();
+                int trackTotalHitsUpTo = SearchRequest.resolveTrackTotalHitsUpTo(request.scroll, request.source);
+                if (trackTotalHitsUpTo == TRACK_TOTAL_HITS_DISABLED
+                        && newSource.suggest() == null
+                        && newSource.aggregations() == null) {
+                    newSource.query(new MatchNoneQueryBuilder());
+                } else {
+                    newSource.size(0);
+                }
+                request.source(newSource);
+                request.setBottomSortValues(null);
+            }
+
             if (newSource == request.source() && newAliasFilter == request.getAliasFilter()) {
                 return this;
             } else {
diff --git a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
index f912d1e99129a..3f8ef30863584 100644
--- a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
+++ b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
@@ -99,6 +99,7 @@ boolean shouldRescore() {
     }
 
     static class EmptyTopDocsCollectorContext extends TopDocsCollectorContext {
+        private final Sort sort;
         private final Collector collector;
         private final Supplier<TotalHits> hitCountSupplier;
 
@@ -109,9 +110,13 @@ static class EmptyTopDocsCollectorContext extends TopDocsCollectorContext {
          * @param trackTotalHitsUpTo True if the total number of hits should be tracked
          * @param hasFilterCollector True if the collector chain contains a filter
          */
-        private EmptyTopDocsCollectorContext(IndexReader reader, Query query,
-                                             int trackTotalHitsUpTo, boolean hasFilterCollector) throws IOException {
+        private EmptyTopDocsCollectorContext(IndexReader reader,
+                                             Query query,
+                                             @Nullable SortAndFormats sortAndFormats,
+                                             int trackTotalHitsUpTo,
+                                             boolean hasFilterCollector) throws IOException {
             super(REASON_SEARCH_COUNT, 0);
+            this.sort = sortAndFormats == null ? null : sortAndFormats.sort;
             if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                 this.collector = new EarlyTerminatingCollector(new TotalHitCountCollector(), 0, false);
                 // for bwc hit count is set to 0, it will be converted to -1 by the coordinating node
@@ -147,7 +152,13 @@ Collector create(Collector in) {
         @Override
         void postProcess(QuerySearchResult result) {
             final TotalHits totalHitCount = hitCountSupplier.get();
-            result.topDocs(new TopDocsAndMaxScore(new TopDocs(totalHitCount, Lucene.EMPTY_SCORE_DOCS), Float.NaN), null);
+            final TopDocs topDocs;
+            if (sort != null) {
+                topDocs = new TopFieldDocs(totalHitCount, Lucene.EMPTY_SCORE_DOCS, sort.getSort());
+            } else {
+                topDocs = new TopDocs(totalHitCount, Lucene.EMPTY_SCORE_DOCS);
+            }
+            result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), null);
         }
     }
 
@@ -421,7 +432,8 @@ static TopDocsCollectorContext createTopDocsCollectorContext(SearchContext searc
         final int totalNumDocs = Math.max(1, reader.numDocs());
         if (searchContext.size() == 0) {
             // no matter what the value of from is
-            return new EmptyTopDocsCollectorContext(reader, query, searchContext.trackTotalHitsUpTo(), hasFilterCollector);
+            return new EmptyTopDocsCollectorContext(reader, query, searchContext.sort(),
+                searchContext.trackTotalHitsUpTo(), hasFilterCollector);
         } else if (searchContext.scrollContext() != null) {
             // we can disable the tracking of total hits after the initial scroll query
             // since the total hits is preserved in the scroll context.
diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 65a3a4137ed59..044c59d24a809 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -26,10 +26,12 @@
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.SortField;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
@@ -50,6 +52,7 @@
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.search.SearchSortValuesAndFormats;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -351,6 +354,55 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
     }
 
+    public boolean canRewriteToMatchNone() {
+        return nestedSort == null && (missing == null || "_last".equals(missing));
+    }
+
+    /**
+     * Returns whether some values of the given {@link QueryShardContext#getIndexReader()} are within the
+     * primary sort value provided in the <code>bottomSortValues</code>.
+     */
+    public boolean isBottomSortShardDisjoint(QueryShardContext context, SearchSortValuesAndFormats bottomSortValues) throws IOException {
+        if (bottomSortValues == null || bottomSortValues.getRawSortValues().length == 0) {
+            return false;
+        }
+
+        if (canRewriteToMatchNone() == false) {
+            return false;
+        }
+        MappedFieldType fieldType = context.fieldMapper(fieldName);
+        if (fieldType == null) {
+            // unmapped
+            return false;
+        }
+        if (fieldType.indexOptions() == IndexOptions.NONE) {
+            return false;
+        }
+        DocValueFormat docValueFormat = bottomSortValues.getSortValueFormats()[0];
+        final DateMathParser dateMathParser;
+        if (docValueFormat instanceof DocValueFormat.DateTime) {
+            if (fieldType instanceof DateFieldType && ((DateFieldType) fieldType).resolution() == NANOSECONDS) {
+                // we parse the formatted value with the resolution of the local field because
+                // the provided format can use a different one (date vs date_nanos).
+                docValueFormat = DocValueFormat.withNanosecondResolution(docValueFormat);
+            }
+            dateMathParser = ((DocValueFormat.DateTime) docValueFormat).getDateMathParser();
+        } else {
+            dateMathParser = null;
+        }
+        Object bottomSortValue =  bottomSortValues.getFormattedSortValues()[0];
+        Object minValue = order() == SortOrder.DESC ? bottomSortValue : null;
+        Object maxValue = order() == SortOrder.DESC ? null : bottomSortValue;
+        try {
+            MappedFieldType.Relation relation = fieldType.isFieldWithinQuery(context.getIndexReader(), minValue, maxValue,
+                true, true, null, dateMathParser, context);
+            return relation == MappedFieldType.Relation.DISJOINT;
+        } catch (ElasticsearchParseException exc) {
+            // can happen if the sort field is mapped differently in another search index
+            return false;
+        }
+    }
+
     @Override
     public BucketedSort buildBucketedSort(QueryShardContext context, int bucketSize, BucketedSort.ExtraData extra) throws IOException {
         if (DOC_FIELD_NAME.equals(fieldName)) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java
new file mode 100644
index 0000000000000..7c626f33e9c40
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.test.ESTestCase;
+
+import java.time.ZoneId;
+import java.util.Arrays;
+
+import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
+import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class BottomSortValuesCollectorTests extends ESTestCase {
+    public void testWithStrings() {
+        for (boolean reverse : new boolean[] { true, false }) {
+            SortField[] sortFields = new SortField[] { new SortField("foo", SortField.Type.STRING_VAL, reverse) };
+            DocValueFormat[] sortFormats = new DocValueFormat[] { DocValueFormat.RAW };
+            BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newBytesArray("foo", "goo", "hoo")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newBytesArray("bar", "car", "zar")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newBytesArray()), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newBytesArray("tar", "zar", "zzz")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newBytesArray(null, null, "zzz")), sortFormats);
+            assertThat(collector.getTotalHits(), equalTo(350L));
+            assertNotNull(collector.getBottomSortValues());
+            assertThat(collector.getBottomSortValues().getSortValueFormats().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getSortValueFormats()[0], instanceOf(DocValueFormat.RAW.getClass()));
+            assertThat(collector.getBottomSortValues().getRawSortValues().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getFormattedSortValues().length, equalTo(1));
+            if (reverse) {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(new BytesRef("tar")));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo("tar"));
+            } else {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(new BytesRef("hoo")));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo("hoo"));
+            }
+        }
+    }
+
+    public void testWithLongs() {
+        for (boolean reverse : new boolean[] { true, false }) {
+            SortField[] sortFields = new SortField[] { new SortField("foo", SortField.Type.LONG, reverse) };
+            DocValueFormat[] sortFormats = new DocValueFormat[]{ DocValueFormat.RAW };
+            BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newLongArray(5L, 10L, 15L)), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newLongArray(25L, 350L, 3500L)), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newLongArray(1L, 2L, 3L)), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newLongArray()), sortFormats);
+            // ignore bottom if we have less top docs than the requested size
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 1,
+                newLongArray(-100L)), sortFormats);
+            assertNotNull(collector.getBottomSortValues());
+            assertThat(collector.getTotalHits(), equalTo(301L));
+            assertThat(collector.getBottomSortValues().getSortValueFormats().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getSortValueFormats()[0], instanceOf(DocValueFormat.RAW.getClass()));
+            assertThat(collector.getBottomSortValues().getRawSortValues().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getFormattedSortValues().length, equalTo(1));
+            if (reverse) {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(25L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo(25L));
+            } else {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(3L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo(3L));
+            }
+        }
+    }
+
+    public void testWithDoubles() {
+        for (boolean reverse : new boolean[] { true, false }) {
+            SortField[] sortFields = new SortField[]{ new SortField("foo", SortField.Type.LONG, reverse) };
+            DocValueFormat[] sortFormats = new DocValueFormat[] { DocValueFormat.RAW };
+            BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newDoubleArray(500d, 5000d, 6755d)), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newDoubleArray(0.1d, 1.5d, 3.5d)), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newDoubleArray()), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newDoubleArray(100d, 101d, 102d)), sortFormats);
+            // ignore bottom if we have less top docs than the requested size
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 2,
+                newDoubleArray(0d, 1d)), sortFormats);
+            assertThat(collector.getTotalHits(), equalTo(302L));
+            assertNotNull(collector.getBottomSortValues());
+            assertThat(collector.getBottomSortValues().getSortValueFormats().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getSortValueFormats()[0], instanceOf(DocValueFormat.RAW.getClass()));
+            assertThat(collector.getBottomSortValues().getRawSortValues().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getFormattedSortValues().length, equalTo(1));
+            if (reverse) {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(500d));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo(500d));
+            } else {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(3.5d));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo(3.5d));
+            }
+        }
+    }
+
+    public void testWithDates() {
+        for (boolean reverse : new boolean[] { true, false }) {
+            SortField[] sortFields = new SortField[]{ new SortField("foo", SortField.Type.LONG, reverse) };
+            DocValueFormat[] sortFormats = new DocValueFormat[] {
+                new DocValueFormat.DateTime(DEFAULT_DATE_TIME_FORMATTER, ZoneId.of("UTC"), DateFieldMapper.Resolution.MILLISECONDS)};
+            BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newDateArray("2017-06-01T12:18:20Z", "2018-04-03T15:10:27Z", "2013-06-01T13:10:20Z")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newDateArray("2018-05-21T08:10:10Z", "2015-02-08T15:12:34Z", "2015-01-01T13:10:30Z")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newDateArray()), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newDateArray("2019-12-30T07:34:20Z", "2017-03-01T12:10:30Z", "2015-07-09T14:00:30Z")), sortFormats);
+            assertThat(collector.getTotalHits(), equalTo(300L));
+            assertNotNull(collector.getBottomSortValues());
+            assertThat(collector.getBottomSortValues().getSortValueFormats().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getSortValueFormats()[0], instanceOf(DocValueFormat.DateTime.class));
+            assertThat(collector.getBottomSortValues().getRawSortValues().length, equalTo(1));
+            if (reverse) {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(1436450430000L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo("2015-07-09T14:00:30.000Z"));
+            } else {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(1522768227000L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo("2018-04-03T15:10:27.000Z"));
+            }
+        }
+    }
+
+    public void testWithDateNanos() {
+        for (boolean reverse : new boolean[] { true, false }) {
+            SortField[] sortFields = new SortField[]{ new SortField("foo", SortField.Type.LONG, reverse) };
+            DocValueFormat[] sortFormats = new DocValueFormat[] {
+                new DocValueFormat.DateTime(DEFAULT_DATE_TIME_FORMATTER, ZoneId.of("UTC"), DateFieldMapper.Resolution.NANOSECONDS)};
+            BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newDateNanoArray("2017-06-01T12:18:20Z", "2018-04-03T15:10:27Z", "2013-06-01T13:10:20Z")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newDateNanoArray("2018-05-21T08:10:10Z", "2015-02-08T15:12:34Z", "2015-01-01T13:10:30Z")), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newDateNanoArray()), sortFormats);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 50,
+                newDateNanoArray("2019-12-30T07:34:20Z", "2017-03-01T12:10:30Z", "2015-07-09T14:00:30Z")), sortFormats);
+            assertThat(collector.getTotalHits(), equalTo(300L));
+            assertNotNull(collector.getBottomSortValues());
+            assertThat(collector.getBottomSortValues().getSortValueFormats().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getSortValueFormats()[0], instanceOf(DocValueFormat.DateTime.class));
+            assertThat(collector.getBottomSortValues().getRawSortValues().length, equalTo(1));
+            if (reverse) {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(1436450430000000000L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo("2015-07-09T14:00:30.000Z"));
+            } else {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(1522768227000000000L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo("2018-04-03T15:10:27.000Z"));
+            }
+        }
+    }
+
+    public void testWithMixedTypes() {
+        for (boolean reverse : new boolean[] { true, false }) {
+            SortField[] sortFields = new SortField[] { new SortField("foo", SortField.Type.LONG, reverse) };
+            SortField[] otherSortFields = new SortField[] { new SortField("foo", SortField.Type.STRING_VAL, reverse) };
+            DocValueFormat[] sortFormats = new DocValueFormat[] { DocValueFormat.RAW };
+            BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields);
+            collector.consumeTopDocs(createTopDocs(sortFields[0], 100,
+                newLongArray(1000L, 100L, 10L)), sortFormats);
+            collector.consumeTopDocs(createTopDocs(otherSortFields[0], 50,
+                newBytesArray("foo", "bar", "zoo")), sortFormats);
+            assertThat(collector.getTotalHits(), equalTo(150L));
+            assertNotNull(collector.getBottomSortValues());
+            assertThat(collector.getBottomSortValues().getSortValueFormats().length, equalTo(1));
+            assertThat(collector.getBottomSortValues().getSortValueFormats()[0], instanceOf(DocValueFormat.RAW.getClass()));
+            assertThat(collector.getBottomSortValues().getRawSortValues().length, equalTo(1));
+            if (reverse) {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(10L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo(10L));
+            } else {
+                assertThat(collector.getBottomSortValues().getRawSortValues()[0], equalTo(1000L));
+                assertThat(collector.getBottomSortValues().getFormattedSortValues()[0], equalTo(1000L));
+            }
+        }
+    }
+
+    private Object[] newDoubleArray(Double... values) {
+        return values;
+    }
+
+    private Object[] newLongArray(Long... values) {
+        return values;
+    }
+
+    private Object[] newBytesArray(String... values) {
+        BytesRef[] bytesRefs = new BytesRef[values.length];
+        for (int i = 0; i < bytesRefs.length; i++) {
+            bytesRefs[i] = values[i] == null ? null : new BytesRef(values[i]);
+        }
+        return bytesRefs;
+    }
+
+    private Object[] newDateArray(String... values) {
+        Long[] longs = new Long[values.length];
+        for (int i = 0; i < values.length; i++) {
+            longs[i] = DEFAULT_DATE_TIME_FORMATTER.parseMillis(values[i]);
+        }
+        return longs;
+    }
+
+    private Object[] newDateNanoArray(String... values) {
+        Long[] longs = new Long[values.length];
+        for (int i = 0; i < values.length; i++) {
+            longs[i] = DateUtils.toNanoSeconds(DEFAULT_DATE_TIME_FORMATTER.parseMillis(values[i]));
+        }
+        return longs;
+    }
+
+    private TopFieldDocs createTopDocs(SortField sortField, int totalHits, Object[] values) {
+        FieldDoc[] fieldDocs = new FieldDoc[values.length];
+        FieldComparator cmp = sortField.getComparator(1, 0);
+        for (int i = 0; i < values.length; i++) {
+            fieldDocs[i] = new FieldDoc(i, Float.NaN, new Object[] { values[i] });
+        }
+        int reverseMul = sortField.getReverse() ? -1 : 1;
+        Arrays.sort(fieldDocs, (o1, o2) -> reverseMul * cmp.compareValues(o1.fields[0], o2.fields[0]));
+        return new TopFieldDocs(new TotalHits(totalHits, GREATER_THAN_OR_EQUAL_TO),
+            fieldDocs, new SortField[] { sortField });
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
new file mode 100644
index 0000000000000..0d4e0ede61167
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class SearchQueryThenFetchAsyncActionTests extends ESTestCase {
+    public void testBottomFieldSort() throws InterruptedException {
+        final TransportSearchAction.SearchTimeProvider timeProvider =
+            new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), System::nanoTime);
+
+        Map<String, Transport.Connection> lookup = new ConcurrentHashMap<>();
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
+        lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
+        lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
+
+        int numShards = randomIntBetween(10, 20);
+        int numConcurrent = randomIntBetween(1, 4);
+        AtomicInteger numWithTopDocs = new AtomicInteger();
+        AtomicInteger successfulOps = new AtomicInteger();
+        AtomicBoolean canReturnNullResponse = new AtomicBoolean(false);
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+            @Override
+            public void sendExecuteQuery(Transport.Connection connection, ShardSearchRequest request,
+                                         SearchTask task, SearchActionListener<SearchPhaseResult> listener) {
+                int shardId = request.shardId().id();
+                if (request.canReturnNullResponseIfMatchNoDocs()) {
+                    canReturnNullResponse.set(true);
+                }
+                if (request.getBottomSortValues() != null) {
+                    assertNotEquals(shardId, (int) request.getBottomSortValues().getFormattedSortValues()[0]);
+                    numWithTopDocs.incrementAndGet();
+                }
+                QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("N/A", 123),
+                    new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null, OriginalIndices.NONE));
+                SortField sortField = new SortField("timestamp", SortField.Type.LONG);
+                queryResult.topDocs(new TopDocsAndMaxScore(new TopFieldDocs(
+                    new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
+                    new FieldDoc[] {
+                        new FieldDoc(randomInt(1000), Float.NaN, new Object[] { request.shardId().id() })
+                    }, new SortField[] { sortField }), Float.NaN),
+                    new DocValueFormat[] { DocValueFormat.RAW });
+                queryResult.from(0);
+                queryResult.size(1);
+                successfulOps.incrementAndGet();
+                new Thread(() -> listener.onResponse(queryResult)).start();
+            }
+        };
+        CountDownLatch latch = new CountDownLatch(1);
+        GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("idx",
+            new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
+            numShards, randomBoolean(), primaryNode, replicaNode);
+        final SearchRequest searchRequest = new SearchRequest();
+        searchRequest.setMaxConcurrentShardRequests(numConcurrent);
+        searchRequest.setBatchedReduceSize(2);
+        searchRequest.source(new SearchSourceBuilder()
+            .size(1)
+            .trackTotalHitsUpTo(2)
+            .sort(SortBuilders.fieldSort("timestamp")));
+        searchRequest.allowPartialSearchResults(false);
+        SearchPhaseController controller = new SearchPhaseController((b) -> new InternalAggregation.ReduceContextBuilder() {
+            @Override
+            public InternalAggregation.ReduceContext forPartialReduction() {
+                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null);
+            }
+
+            public InternalAggregation.ReduceContext forFinalReduction() {
+                return InternalAggregation.ReduceContext.forFinalReduction(
+                    BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineAggregator.PipelineTree.EMPTY);
+            };
+        });
+        SearchTask task = new SearchTask(0, "n/a", "n/a", "test", null, Collections.emptyMap());
+        SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction(logger,
+            searchTransportService, (clusterAlias, node) -> lookup.get(node),
+            Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)),
+            Collections.emptyMap(), Collections.emptyMap(), controller, EsExecutors.newDirectExecutorService(), searchRequest,
+            null, shardsIter, timeProvider, null, task,
+            SearchResponse.Clusters.EMPTY) {
+            @Override
+            protected SearchPhase getNextPhase(SearchPhaseResults<SearchPhaseResult> results, SearchPhaseContext context) {
+                return new SearchPhase("test") {
+                    @Override
+                    public void run() {
+                        latch.countDown();
+                    }
+                };
+            }
+        };
+        action.start();
+        latch.await();
+        assertThat(successfulOps.get(), equalTo(numShards));
+        assertTrue(canReturnNullResponse.get());
+        assertThat(numWithTopDocs.get(), greaterThanOrEqualTo(1));
+        SearchPhaseController.ReducedQueryPhase phase = action.results.reduce();
+        assertThat(phase.numReducePhases, greaterThanOrEqualTo(1));
+        assertThat(phase.totalHits.value, equalTo(2L));
+        assertThat(phase.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+        assertThat(phase.sortedTopDocs.scoreDocs.length, equalTo(1));
+        assertThat(phase.sortedTopDocs.scoreDocs[0], instanceOf(FieldDoc.class));
+        assertThat(((FieldDoc) phase.sortedTopDocs.scoreDocs[0]).fields.length, equalTo(1));
+        assertThat(((FieldDoc) phase.sortedTopDocs.scoreDocs[0]).fields[0], equalTo(0));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/SearchSortValuesAndFormatsTests.java b/server/src/test/java/org/elasticsearch/search/SearchSortValuesAndFormatsTests.java
new file mode 100644
index 0000000000000..2273ab3af0db7
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/SearchSortValuesAndFormatsTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class SearchSortValuesAndFormatsTests extends AbstractWireSerializingTestCase<SearchSortValuesAndFormats> {
+    private NamedWriteableRegistry namedWriteableRegistry;
+
+    @Before
+    public void initRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(searchModule.getNamedWriteables());
+        namedWriteableRegistry = new NamedWriteableRegistry(entries);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
+
+    @Override
+    protected Writeable.Reader<SearchSortValuesAndFormats> instanceReader() {
+        return SearchSortValuesAndFormats::new;
+    }
+
+    @Override
+    protected SearchSortValuesAndFormats createTestInstance() {
+        return randomInstance();
+    }
+
+    @Override
+    protected SearchSortValuesAndFormats mutateInstance(SearchSortValuesAndFormats instance) {
+        Object[] sortValues = instance.getRawSortValues();
+        Object[] newValues = Arrays.copyOf(sortValues, sortValues.length + 1);
+        DocValueFormat[] newFormats = Arrays.copyOf(instance.getSortValueFormats(), sortValues.length + 1);
+        newValues[sortValues.length] =  randomSortValue();
+        newFormats[sortValues.length] = DocValueFormat.RAW;
+        return new SearchSortValuesAndFormats(newValues, newFormats);
+    }
+
+    private static Object randomSortValue() {
+        switch(randomIntBetween(0, 5)) {
+            case 0:
+                return null;
+            case 1:
+                return new BytesRef(randomAlphaOfLengthBetween(3, 10));
+            case 2:
+                return randomInt();
+            case 3:
+                return randomLong();
+            case 4:
+                return randomFloat();
+            case 5:
+                return randomDouble();
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
+
+    public static SearchSortValuesAndFormats randomInstance()  {
+        int size = randomIntBetween(1, 20);
+        Object[] values = new Object[size];
+        DocValueFormat[] sortValueFormats = new DocValueFormat[size];
+        for (int i = 0; i < size; i++) {
+            values[i] = randomSortValue();
+            sortValueFormats[i] = DocValueFormat.RAW;
+        }
+        return new SearchSortValuesAndFormats(values, sortValueFormats);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
index 2d0aa2f591d8d..a9759c6298e83 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.search.AbstractSearchTestCase;
+import org.elasticsearch.search.SearchSortValuesAndFormatsTests;
 
 import java.io.IOException;
 
@@ -94,6 +95,9 @@ private ShardSearchRequest createShardSearchRequest() throws IOException {
             randomIntBetween(1, 100), filteringAliases, randomBoolean() ? 1.0f : randomFloat(),
             Math.abs(randomLong()), randomAlphaOfLengthBetween(3, 10), routings);
         req.canReturnNullResponseIfMatchNoDocs(randomBoolean());
+        if (randomBoolean()) {
+            req.setBottomSortValues(SearchSortValuesAndFormatsTests.randomInstance());
+        }
         return req;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index b4482e5545503..b457e560c65d4 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -27,12 +27,14 @@
 import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.AssertingIndexSearcher;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSelector;
 import org.apache.lucene.search.SortedNumericSortField;
@@ -60,6 +62,7 @@
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.SearchSortValuesAndFormats;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
@@ -91,6 +94,8 @@ protected FieldSortBuilder createTestItem() {
             randomInt());
 
 
+
+
     public FieldSortBuilder randomFieldSortBuilder() {
         String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
         FieldSortBuilder builder = new FieldSortBuilder(fieldName);
@@ -565,6 +570,55 @@ public void testGetMaxKeywordValue() throws IOException {
         }
     }
 
+    public void testIsBottomSortShardDisjoint() throws Exception {
+        try (Directory dir = newDirectory()) {
+            int numDocs = randomIntBetween(5, 10);
+            long maxValue = -1;
+            long minValue = Integer.MAX_VALUE;
+            try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new KeywordAnalyzer())) {
+                FieldSortBuilder fieldSort = SortBuilders.fieldSort("custom-date");
+                try (DirectoryReader reader = writer.getReader()) {
+                    QueryShardContext context = createMockShardContext(new IndexSearcher(reader));
+                    assertTrue(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { 0L }, new DocValueFormat[] { DocValueFormat.RAW })));
+                }
+                for (int i = 0; i < numDocs; i++) {
+                    Document doc = new Document();
+                    long value = randomLongBetween(1, Integer.MAX_VALUE);
+                    doc.add(new LongPoint("custom-date", value));
+                    doc.add(new SortedNumericDocValuesField("custom-date", value));
+                    writer.addDocument(doc);
+                    maxValue = Math.max(maxValue, value);
+                    minValue = Math.min(minValue, value);
+                }
+                try (DirectoryReader reader = writer.getReader()) {
+                    QueryShardContext context = createMockShardContext(new IndexSearcher(reader));
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context, null));
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { minValue }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    assertTrue(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { minValue-1 }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { minValue+1 }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    fieldSort.order(SortOrder.DESC);
+                    assertTrue(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { maxValue+1 }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { maxValue }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { minValue }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    fieldSort.setNestedSort(new NestedSortBuilder("empty"));
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { minValue-1 }, new DocValueFormat[] { DocValueFormat.RAW })));
+                    fieldSort.setNestedSort(null);
+                    fieldSort.missing("100");
+                    assertFalse(fieldSort.isBottomSortShardDisjoint(context,
+                        new SearchSortValuesAndFormats(new Object[] { maxValue+1 }, new DocValueFormat[] { DocValueFormat.RAW })));
+                }
+            }
+        }
+    }
+
     @Override
     protected FieldSortBuilder fromXContent(XContentParser parser, String fieldName) throws IOException {
         return FieldSortBuilder.fromXContent(parser, fieldName);

From 68a698f9ae317a4465098817fd493bad46b68c95 Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Tue, 17 Mar 2020 11:34:22 +0100
Subject: [PATCH 073/285] [Transform] add transform discovery node role
 (#53616)

Enhancement of #52712: Add a discovery node role using the letter t for transform.

Fixes #53156
---
 .../resources/rest-api-spec/test/11_nodes.yml    |  4 ++--
 .../rest-api-spec/test/cat.nodes/10_basic.yml    |  4 ++--
 .../elasticsearch/xpack/transform/Transform.java | 16 ++++++++++++++++
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml
index 9c46c1c0ccd77..c5b355e9b83f3 100644
--- a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml
+++ b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml
@@ -7,7 +7,7 @@
   - match:
       $body: |
         /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role           master          name
-        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
         /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role      \s+  master   \s+   name  \n
-           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml
index bde9e1d26826f..dbe18fbd83829 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml
@@ -7,7 +7,7 @@
   - match:
       $body: |
                /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role           master          name
-               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
                /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role      \s+  master   \s+   name  \n
-                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
index 2faa5c527fb22..e92bc80f338f1 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -112,6 +113,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 
@@ -157,6 +159,15 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa
         Property.NodeScope
     );
 
+    public static final DiscoveryNodeRole TRANSFORM_ROLE = new DiscoveryNodeRole("transform", "t") {
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return TRANSFORM_ENABLED_NODE;
+        }
+
+    };
+
     public Transform(Settings settings) {
         this.settings = settings;
         this.enabled = XPackSettings.TRANSFORM_ENABLED.get(settings);
@@ -362,6 +373,11 @@ public Settings additionalSettings() {
         return additionalSettings.build();
     }
 
+    @Override
+    public Set<DiscoveryNodeRole> getRoles() {
+        return Collections.singleton(TRANSFORM_ROLE);
+    }
+
     @Override
     public void close() {
         if (transformServices.get() != null) {

From 3e607d9e93c6319fcd606714aabca4af91f133d3 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Tue, 17 Mar 2020 12:25:51 +0000
Subject: [PATCH 074/285] Rename AtomicFieldData to LeafFieldData (#53554)

This conforms with lucene's LeafReader naming convention, and
matches other per-segment structures in elasticsearch.
---
 .../expression/CountMethodValueSource.java    |  4 +-
 .../expression/DateMethodValueSource.java     |  4 +-
 .../expression/DateObjectValueSource.java     |  6 +--
 .../expression/EmptyMemberValueSource.java    |  4 +-
 .../expression/FieldDataValueSource.java      |  4 +-
 .../expression/GeoEmptyValueSource.java       |  4 +-
 .../expression/GeoLatitudeValueSource.java    |  4 +-
 .../expression/GeoLongitudeValueSource.java   |  4 +-
 .../ExpressionFieldScriptTests.java           |  4 +-
 .../ExpressionNumberSortScriptTests.java      |  4 +-
 .../ExpressionTermsSetQueryTests.java         |  4 +-
 .../index/mapper/ScaledFloatFieldMapper.java  | 12 ++---
 .../mapper/ScaledFloatFieldTypeTests.java     |  4 +-
 .../rescore/ExampleRescoreBuilder.java        | 12 ++---
 .../search/function/RandomScoreFunction.java  |  4 +-
 .../index/fielddata/IndexFieldData.java       |  4 +-
 .../index/fielddata/IndexFieldDataCache.java  | 10 ++---
 .../fielddata/IndexGeoPointFieldData.java     |  2 +-
 .../fielddata/IndexHistogramFieldData.java    |  2 +-
 .../fielddata/IndexNumericFieldData.java      |  2 +-
 .../fielddata/IndexOrdinalsFieldData.java     |  2 +-
 ...tomicFieldData.java => LeafFieldData.java} |  2 +-
 ...ldData.java => LeafGeoPointFieldData.java} |  4 +-
 ...dData.java => LeafHistogramFieldData.java} |  4 +-
 ...eldData.java => LeafNumericFieldData.java} |  4 +-
 ...ldData.java => LeafOrdinalsFieldData.java} |  4 +-
 .../index/fielddata/ScriptDocValues.java      |  2 +-
 .../LongValuesComparatorSource.java           |  6 +--
 .../ordinals/GlobalOrdinalsBuilder.java       | 12 ++---
 .../GlobalOrdinalsIndexFieldData.java         | 18 ++++----
 .../plain/AbstractIndexFieldData.java         |  4 +-
 .../plain/AbstractIndexOrdinalsFieldData.java | 10 ++---
 .../AbstractLatLonPointDVIndexFieldData.java  |  8 ++--
 ...ava => AbstractLeafGeoPointFieldData.java} | 10 ++---
 ...ava => AbstractLeafOrdinalsFieldData.java} | 12 ++---
 .../plain/BinaryDVIndexFieldData.java         |  8 ++--
 ...ldData.java => BinaryDVLeafFieldData.java} | 10 ++---
 .../plain/BytesBinaryDVIndexFieldData.java    |  8 ++--
 ...a.java => BytesBinaryDVLeafFieldData.java} |  6 +--
 .../plain/ConstantIndexFieldData.java         | 14 +++---
 .../plain/DocValuesIndexFieldData.java        |  2 +-
 ...a.java => LatLonPointDVLeafFieldData.java} |  4 +-
 ...ieldData.java => LeafDoubleFieldData.java} | 12 ++---
 ...gFieldData.java => LeafLongFieldData.java} |  8 ++--
 .../plain/PagedBytesIndexFieldData.java       | 10 ++---
 ...Data.java => PagedBytesLeafFieldData.java} |  4 +-
 .../plain/SortedNumericDVIndexFieldData.java  | 16 +++----
 ...ava => SortedSetDVBytesLeafFieldData.java} | 10 ++---
 .../SortedSetDVOrdinalsIndexFieldData.java    |  8 ++--
 .../index/mapper/IdFieldMapper.java           | 12 ++---
 .../cache/IndicesFieldDataCache.java          | 12 ++---
 .../DateHistogramAggregationBuilder.java      |  6 +--
 .../aggregations/support/ValuesSource.java    | 10 ++---
 .../fetch/subphase/FetchDocValuesPhase.java   | 10 ++---
 .../AbstractFieldDataImplTestCase.java        | 12 ++---
 .../fielddata/AbstractFieldDataTestCase.java  |  4 +-
 .../AbstractStringFieldDataTestCase.java      |  4 +-
 .../fielddata/BinaryDVFieldDataTests.java     |  2 +-
 .../index/fielddata/FieldDataCacheTests.java  | 10 ++---
 .../index/fielddata/FilterFieldDataTests.java |  8 ++--
 .../index/fielddata/GeoFieldDataTests.java    | 18 ++++----
 .../fielddata/IndexFieldDataServiceTests.java |  6 +--
 .../NoOrdinalsStringFieldDataTests.java       | 10 ++---
 .../index/mapper/DateFieldTypeTests.java      |  4 +-
 .../index/mapper/TypeFieldMapperTests.java    |  4 +-
 .../index/query/QueryShardContextTests.java   |  4 +-
 .../functionscore/FunctionScoreTests.java     | 18 ++++----
 .../search/lookup/LeafDocLookupTests.java     |  8 ++--
 .../mapper/HistogramFieldMapper.java          |  8 ++--
 .../mapper/FlatObjectFieldMapper.java         | 18 ++++----
 ...java => KeyedFlatObjectLeafFieldData.java} | 14 +++---
 .../mapper/FlatObjectFieldLookupTests.java    |  8 ++--
 ...=> KeyedFlatObjectLeafFieldDataTests.java} | 44 +++++++++----------
 ...ldDataCacheWithFieldSubsetReaderTests.java | 26 +++++------
 .../vectors/query/VectorDVIndexFieldData.java |  8 ++--
 ...ldData.java => VectorDVLeafFieldData.java} |  6 +--
 76 files changed, 307 insertions(+), 307 deletions(-)
 rename server/src/main/java/org/elasticsearch/index/fielddata/{AtomicFieldData.java => LeafFieldData.java} (94%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/{AtomicGeoPointFieldData.java => LeafGeoPointFieldData.java} (88%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/{AtomicHistogramFieldData.java => LeafHistogramFieldData.java} (88%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/{AtomicNumericFieldData.java => LeafNumericFieldData.java} (91%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/{AtomicOrdinalsFieldData.java => LeafOrdinalsFieldData.java} (87%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{AbstractAtomicGeoPointFieldData.java => AbstractLeafGeoPointFieldData.java} (87%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{AbstractAtomicOrdinalsFieldData.java => AbstractLeafOrdinalsFieldData.java} (85%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{BinaryDVAtomicFieldData.java => BinaryDVLeafFieldData.java} (88%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{BytesBinaryDVAtomicFieldData.java => BytesBinaryDVLeafFieldData.java} (94%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{LatLonPointDVAtomicFieldData.java => LatLonPointDVLeafFieldData.java} (94%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{AtomicDoubleFieldData.java => LeafDoubleFieldData.java} (85%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{AtomicLongFieldData.java => LeafLongFieldData.java} (90%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{PagedBytesAtomicFieldData.java => PagedBytesLeafFieldData.java} (93%)
 rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{SortedSetDVBytesAtomicFieldData.java => SortedSetDVBytesLeafFieldData.java} (84%)
 rename x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/{KeyedFlatObjectAtomicFieldData.java => KeyedFlatObjectLeafFieldData.java} (94%)
 rename x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/{KeyedFlatObjectAtomicFieldDataTests.java => KeyedFlatObjectLeafFieldDataTests.java} (75%)
 rename x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/{VectorDVAtomicFieldData.java => VectorDVLeafFieldData.java} (90%)

diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
index 6a36ff7a53943..d70c6790317f9 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
@@ -46,7 +46,7 @@ final class CountMethodValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final SortedNumericDoubleValues values = leafData.getDoubleValues();
 
         return new DoubleDocValues(this) {
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
index 13845a88fe3f0..ee8c604a39ba2 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
@@ -29,7 +29,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
@@ -52,7 +52,7 @@ class DateMethodValueSource extends FieldDataValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
         return new DoubleDocValues(this) {
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
index ee59892a7024f..89f991c174d30 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
@@ -41,7 +41,7 @@ class DateObjectValueSource extends FieldDataValueSource {
     final String methodName;
     final ToIntFunction<ReadableDateTime> function;
 
-    DateObjectValueSource(IndexFieldData<?> indexFieldData, MultiValueMode multiValueMode, 
+    DateObjectValueSource(IndexFieldData<?> indexFieldData, MultiValueMode multiValueMode,
                           String methodName, ToIntFunction<ReadableDateTime> function) {
         super(indexFieldData, multiValueMode);
 
@@ -54,7 +54,7 @@ class DateObjectValueSource extends FieldDataValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         MutableDateTime joda = new MutableDateTime(0, DateTimeZone.UTC);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
         return new DoubleDocValues(this) {
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
index aa12560e1fce0..750b03762a7f9 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
@@ -46,7 +46,7 @@ final class EmptyMemberValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final SortedNumericDoubleValues values = leafData.getDoubleValues();
         return new DoubleDocValues(this) {
             @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
index 3e49797bbacc8..63050a07da8bf 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
@@ -67,7 +67,7 @@ public int hashCode() {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
         return new DoubleDocValues(this) {
           @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
index 0b16aaf9dcde0..2949e1dd9287a 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
@@ -44,7 +44,7 @@ final class GeoEmptyValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf);
+        LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
         return new DoubleDocValues(this) {
             @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
index fd812dac5a3a8..3c04de1658669 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
@@ -44,7 +44,7 @@ final class GeoLatitudeValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf);
+        LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
         return new DoubleDocValues(this) {
             @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
index fd05d92d62350..8b2c5d83c0257 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
@@ -44,7 +44,7 @@ final class GeoLongitudeValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf);
+        LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
         return new DoubleDocValues(this) {
             @Override
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
index 073d6d19aed3a..03aeca782b53a 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.script.expression;
 
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MapperService;
@@ -56,7 +56,7 @@ public void setUp() throws Exception {
         when(doubleValues.advanceExact(anyInt())).thenReturn(true);
         when(doubleValues.nextValue()).thenReturn(2.718);
 
-        AtomicNumericFieldData atomicFieldData = mock(AtomicNumericFieldData.class);
+        LeafNumericFieldData atomicFieldData = mock(LeafNumericFieldData.class);
         when(atomicFieldData.getDoubleValues()).thenReturn(doubleValues);
 
         IndexNumericFieldData fieldData = mock(IndexNumericFieldData.class);
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
index 4842b7b03c03a..057ebdb3af5a1 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.script.expression;
 
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MapperService;
@@ -56,7 +56,7 @@ public void setUp() throws Exception {
         when(doubleValues.advanceExact(anyInt())).thenReturn(true);
         when(doubleValues.nextValue()).thenReturn(2.718);
 
-        AtomicNumericFieldData atomicFieldData = mock(AtomicNumericFieldData.class);
+        LeafNumericFieldData atomicFieldData = mock(LeafNumericFieldData.class);
         when(atomicFieldData.getDoubleValues()).thenReturn(doubleValues);
 
         IndexNumericFieldData fieldData = mock(IndexNumericFieldData.class);
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
index 93b8effae51e5..43a3e26259b97 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.script.expression;
 
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MapperService;
@@ -56,7 +56,7 @@ public void setUp() throws Exception {
         when(doubleValues.advanceExact(anyInt())).thenReturn(true);
         when(doubleValues.nextValue()).thenReturn(2.718);
 
-        AtomicNumericFieldData atomicFieldData = mock(AtomicNumericFieldData.class);
+        LeafNumericFieldData atomicFieldData = mock(LeafNumericFieldData.class);
         when(atomicFieldData.getDoubleValues()).thenReturn(doubleValues);
 
         IndexNumericFieldData fieldData = mock(IndexNumericFieldData.class);
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
index d92cb195141b3..81c092f3a2de6 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
@@ -43,7 +43,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -517,12 +517,12 @@ public String getFieldName() {
         }
 
         @Override
-        public AtomicNumericFieldData load(LeafReaderContext context) {
+        public LeafNumericFieldData load(LeafReaderContext context) {
             return new ScaledFloatLeafFieldData(scaledFieldData.load(context), scalingFactor);
         }
 
         @Override
-        public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception {
+        public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception {
             return new ScaledFloatLeafFieldData(scaledFieldData.loadDirect(context), scalingFactor);
         }
 
@@ -559,12 +559,12 @@ public NumericType getNumericType() {
 
     }
 
-    private static class ScaledFloatLeafFieldData implements AtomicNumericFieldData {
+    private static class ScaledFloatLeafFieldData implements LeafNumericFieldData {
 
-        private final AtomicNumericFieldData scaledFieldData;
+        private final LeafNumericFieldData scaledFieldData;
         private final double scalingFactorInverse;
 
-        ScaledFloatLeafFieldData(AtomicNumericFieldData scaledFieldData, double scalingFactor) {
+        ScaledFloatLeafFieldData(LeafNumericFieldData scaledFieldData, double scalingFactor) {
             this.scaledFieldData = scaledFieldData;
             this.scalingFactorInverse = 1d / scalingFactor;
         }
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
index 4389e809bfb2f..1e4ab0b99a20c 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.junit.Before;
@@ -193,7 +193,7 @@ public void testFieldData() throws IOException {
             IndexNumericFieldData fielddata = (IndexNumericFieldData) ft.fielddataBuilder("index")
                 .build(indexSettings, ft, null, null, null);
             assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE);
-            AtomicNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0));
+            LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0));
             SortedNumericDoubleValues values = leafFieldData.getDoubleValues();
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
diff --git a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java
index b459c91283585..cf16500378583 100644
--- a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java
+++ b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -170,7 +170,7 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
                  * them in (reader, field, docId) order because that is the
                  * order they are on disk.
                  */
-                ScoreDoc[] sortedByDocId = new ScoreDoc[topDocs.scoreDocs.length]; 
+                ScoreDoc[] sortedByDocId = new ScoreDoc[topDocs.scoreDocs.length];
                 System.arraycopy(topDocs.scoreDocs, 0, sortedByDocId, 0, topDocs.scoreDocs.length);
                 Arrays.sort(sortedByDocId, (a, b) -> a.doc - b.doc); // Safe because doc ids >= 0
                 Iterator<LeafReaderContext> leaves = searcher.getIndexReader().leaves().iterator();
@@ -183,11 +183,11 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
                             leaf = leaves.next();
                             endDoc = leaf.docBase + leaf.reader().maxDoc();
                         } while (topDocs.scoreDocs[i].doc >= endDoc);
-                        AtomicFieldData fd = context.factorField.load(leaf);
-                        if (false == (fd instanceof AtomicNumericFieldData)) {
+                        LeafFieldData fd = context.factorField.load(leaf);
+                        if (false == (fd instanceof LeafNumericFieldData)) {
                             throw new IllegalArgumentException("[" + context.factorField.getFieldName() + "] is not a number");
                         }
-                        data = ((AtomicNumericFieldData) fd).getDoubleValues();
+                        data = ((LeafNumericFieldData) fd).getDoubleValues();
                     }
                     if (false == data.advanceExact(topDocs.scoreDocs[i].doc - leaf.docBase)) {
                         throw new IllegalArgumentException("document [" + topDocs.scoreDocs[i].doc
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
index b55730f514ac9..6b168ce21c033 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.util.StringHelper;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 
@@ -57,7 +57,7 @@ public RandomScoreFunction(int seed, int salt, IndexFieldData<?> uidFieldData) {
     public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) {
         final SortedBinaryDocValues values;
         if (fieldData != null) {
-            AtomicFieldData leafData = fieldData.load(ctx);
+            LeafFieldData leafData = fieldData.load(ctx);
             values = leafData.getBytesValues();
             if (values == null) throw new NullPointerException("failed to get fielddata");
         } else {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index 5002639f09121..82634dffa432f 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -54,7 +54,7 @@
  * Thread-safe utility class that allows to get per-segment values via the
  * {@link #load(LeafReaderContext)} method.
  */
-public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
+public interface IndexFieldData<FD extends LeafFieldData> extends IndexComponent {
 
     /**
      * The field name.
@@ -251,7 +251,7 @@ IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType,
                              CircuitBreakerService breakerService, MapperService mapperService);
     }
 
-    interface Global<FD extends AtomicFieldData> extends IndexFieldData<FD> {
+    interface Global<FD extends LeafFieldData> extends IndexFieldData<FD> {
 
         IndexFieldData<FD> loadGlobal(DirectoryReader indexReader);
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
index 83108ad571190..40db7592a545f 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -29,9 +29,9 @@
  */
 public interface IndexFieldDataCache {
 
-    <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData) throws Exception;
+    <FD extends LeafFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData) throws Exception;
 
-    <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader, IFD indexFieldData)
+    <FD extends LeafFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader, IFD indexFieldData)
         throws Exception;
 
     /**
@@ -60,15 +60,15 @@ default void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lo
     class None implements IndexFieldDataCache {
 
         @Override
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData)
+        public <FD extends LeafFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData)
             throws Exception {
             return indexFieldData.loadDirect(context);
         }
 
         @Override
         @SuppressWarnings("unchecked")
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader,
-                IFD indexFieldData) throws Exception {
+        public <FD extends LeafFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader,
+                                                                                          IFD indexFieldData) throws Exception {
             return (IFD) indexFieldData.localGlobalDirect(indexReader);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
index 53466e9b4ec61..4c9391caf52ca 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
@@ -23,5 +23,5 @@
 /**
  * Specialization of {@link IndexFieldData} for geo points.
  */
-public interface IndexGeoPointFieldData extends IndexFieldData<AtomicGeoPointFieldData> {
+public interface IndexGeoPointFieldData extends IndexFieldData<LeafGeoPointFieldData> {
 }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexHistogramFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexHistogramFieldData.java
index 0a8160f427f43..a944e44cd9f6b 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexHistogramFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexHistogramFieldData.java
@@ -26,7 +26,7 @@
 /**
  * Specialization of {@link IndexFieldData} for histograms.
  */
-public abstract class IndexHistogramFieldData extends DocValuesIndexFieldData implements IndexFieldData<AtomicHistogramFieldData> {
+public abstract class IndexHistogramFieldData extends DocValuesIndexFieldData implements IndexFieldData<LeafHistogramFieldData> {
 
     public IndexHistogramFieldData(Index index, String fieldName) {
         super(index, fieldName);
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
index cdab0a1493cb5..b881c09c94984 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.fielddata;
 
-public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericFieldData> {
+public interface IndexNumericFieldData extends IndexFieldData<LeafNumericFieldData> {
 
     enum NumericType {
         BOOLEAN(false),
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
index 9d4dbaef21125..b81832f1385bf 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
@@ -27,7 +27,7 @@
 /**
  * Specialization of {@link IndexFieldData} for data that is indexed with ordinals.
  */
-public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdinalsFieldData> {
+public interface IndexOrdinalsFieldData extends IndexFieldData.Global<LeafOrdinalsFieldData> {
 
     /**
      * Load a global view of the ordinals for the given {@link IndexReader},
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java
index 20c9c3c70c10c..34d3cc05aa22b 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java
@@ -25,7 +25,7 @@
 /**
  * The thread safe {@link org.apache.lucene.index.LeafReader} level cache of the data.
  */
-public interface AtomicFieldData extends Accountable, Releasable {
+public interface LeafFieldData extends Accountable, Releasable {
 
     /**
      * Returns field values for use in scripting.
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafGeoPointFieldData.java
similarity index 88%
rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/LeafGeoPointFieldData.java
index a496e1d93ecd3..d30e98ec249b7 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafGeoPointFieldData.java
@@ -20,9 +20,9 @@
 
 
 /**
- * {@link AtomicFieldData} specialization for geo points.
+ * {@link LeafFieldData} specialization for geo points.
  */
-public interface AtomicGeoPointFieldData extends AtomicFieldData {
+public interface LeafGeoPointFieldData extends LeafFieldData {
 
     /**
      * Return geo-point values.
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicHistogramFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafHistogramFieldData.java
similarity index 88%
rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicHistogramFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/LeafHistogramFieldData.java
index 1678ca0df7783..6f4f6e9bf0f65 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicHistogramFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafHistogramFieldData.java
@@ -22,9 +22,9 @@
 import java.io.IOException;
 
 /**
- * {@link AtomicFieldData} specialization for histogram data.
+ * {@link LeafFieldData} specialization for histogram data.
  */
-public interface AtomicHistogramFieldData extends AtomicFieldData {
+public interface LeafHistogramFieldData extends LeafFieldData {
 
     /**
      * Return Histogram values.
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java
index 32f6ddddc7c24..1f4ab23fad758 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java
@@ -22,9 +22,9 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 
 /**
- * Specialization of {@link AtomicFieldData} for numeric data.
+ * Specialization of {@link LeafFieldData} for numeric data.
  */
-public interface AtomicNumericFieldData extends AtomicFieldData {
+public interface LeafNumericFieldData extends LeafFieldData {
 
     /**
      * Get an integer view of the values of this segment. If the implementation
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafOrdinalsFieldData.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/LeafOrdinalsFieldData.java
index 14ed16ecb3ec9..88a42ccf83f36 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafOrdinalsFieldData.java
@@ -22,10 +22,10 @@
 import org.apache.lucene.index.SortedSetDocValues;
 
 /**
- * Specialization of {@link AtomicFieldData} for data that is indexed with
+ * Specialization of {@link LeafFieldData} for data that is indexed with
  * ordinals.
  */
-public interface AtomicOrdinalsFieldData extends AtomicFieldData {
+public interface LeafOrdinalsFieldData extends LeafFieldData {
 
     /**
      * Return the ordinals values for the current atomic reader.
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index a5349b5137996..080ef97d5f86d 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -452,7 +452,7 @@ public void setNextDocId(int docId) throws IOException {
             if (in.advanceExact(docId)) {
                 resize(in.docValueCount());
                 for (int i = 0; i < count; i++) {
-                    // We need to make a copy here, because BytesBinaryDVAtomicFieldData's SortedBinaryDocValues
+                    // We need to make a copy here, because BytesBinaryDVLeafFieldData's SortedBinaryDocValues
                     // implementation reuses the returned BytesRef. Otherwise we would end up with the same BytesRef
                     // instance for all slots in the values array.
                     values[i].copyBytes(in.nextValue());
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
index 6de7ddaf869ad..56b0cc5fde80d 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.util.BitSet;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -67,7 +67,7 @@ public SortField.Type reducedType() {
     }
 
     private SortedNumericDocValues loadDocValues(LeafReaderContext context) {
-        final AtomicNumericFieldData data = indexFieldData.load(context);
+        final LeafNumericFieldData data = indexFieldData.load(context);
         SortedNumericDocValues values;
         if (data instanceof SortedNumericDVIndexFieldData.NanoSecondFieldData) {
             values = ((SortedNumericDVIndexFieldData.NanoSecondFieldData) data).getLongValuesAsNanos();
@@ -76,7 +76,7 @@ private SortedNumericDocValues loadDocValues(LeafReaderContext context) {
         }
         return converter != null ? converter.apply(values) : values;
     }
-    
+
     private NumericDocValues getNumericDocValues(LeafReaderContext context, long missingValue) throws IOException {
         final SortedNumericDocValues values = loadDocValues(context);
         if (nested == null) {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
index 3c8f100a17db5..d5fd31cfd5be9 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
@@ -29,10 +29,10 @@
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.io.IOException;
@@ -56,7 +56,7 @@ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexO
         assert indexReader.leaves().size() > 1;
         long startTimeNS = System.nanoTime();
 
-        final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()];
+        final LeafOrdinalsFieldData[] atomicFD = new LeafOrdinalsFieldData[indexReader.leaves().size()];
         final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()];
         for (int i = 0; i < indexReader.leaves().size(); ++i) {
             atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i));
@@ -83,10 +83,10 @@ public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, fin
             IndexOrdinalsFieldData indexFieldData) throws IOException {
         assert indexReader.leaves().size() > 1;
 
-        final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()];
+        final LeafOrdinalsFieldData[] atomicFD = new LeafOrdinalsFieldData[indexReader.leaves().size()];
         final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()];
         for (int i = 0; i < indexReader.leaves().size(); ++i) {
-            atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) {
+            atomicFD[i] = new AbstractLeafOrdinalsFieldData(AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) {
                 @Override
                 public SortedSetDocValues getOrdinalsValues() {
                     return DocValues.emptySortedSet();
@@ -110,7 +110,7 @@ public void close() {
         }
         final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT);
         return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
-                atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
+                atomicFD, ordinalMap, 0, AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
index b2cd392a070b3..ec7414c258d04 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
@@ -29,11 +29,11 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.sort.BucketedSort;
@@ -60,12 +60,12 @@ public final class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent i
     private final long memorySizeInBytes;
 
     private final OrdinalMap ordinalMap;
-    private final AtomicOrdinalsFieldData[] segmentAfd;
+    private final LeafOrdinalsFieldData[] segmentAfd;
     private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction;
 
     protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings,
                                            String fieldName,
-                                           AtomicOrdinalsFieldData[] segmentAfd,
+                                           LeafOrdinalsFieldData[] segmentAfd,
                                            OrdinalMap ordinalMap,
                                            long memorySizeInBytes,
                                            Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) {
@@ -82,7 +82,7 @@ public IndexOrdinalsFieldData newConsumer(DirectoryReader source) {
     }
 
     @Override
-    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
         throw new IllegalStateException("loadDirect(LeafReaderContext) should not be called in this context");
     }
 
@@ -127,7 +127,7 @@ public Collection<Accountable> getChildResources() {
     }
 
     @Override
-    public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+    public LeafOrdinalsFieldData load(LeafReaderContext context) {
         throw new IllegalStateException("load(LeafReaderContext) should not be called in this context");
     }
 
@@ -172,7 +172,7 @@ private TermsEnum[] getOrLoadTermsEnums() {
         }
 
         @Override
-        public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+        public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
             return load(context);
         }
 
@@ -216,9 +216,9 @@ public Collection<Accountable> getChildResources() {
         }
 
         @Override
-        public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+        public LeafOrdinalsFieldData load(LeafReaderContext context) {
             assert source.getReaderCacheHelper().getKey() == context.parent.reader().getReaderCacheHelper().getKey();
-            return new AbstractAtomicOrdinalsFieldData(scriptFunction) {
+            return new AbstractLeafOrdinalsFieldData(scriptFunction) {
                 @Override
                 public SortedSetDocValues getOrdinalsValues() {
                     final SortedSetDocValues values = segmentAfd[context.ord].getOrdinalsValues();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
index 5eaaf14f8bcb5..023c0a248a05c 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
@@ -26,14 +26,14 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.RamAccountingTermsEnum;
 
 import java.io.IOException;
 
-public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
+public abstract class AbstractIndexFieldData<FD extends LeafFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
 
     private final String fieldName;
     protected final IndexFieldDataCache cache;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 081875ae1e233..3d549c82b8239 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -28,7 +28,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
@@ -37,7 +37,7 @@
 
 import java.io.IOException;
 
-public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData<AtomicOrdinalsFieldData>
+public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData<LeafOrdinalsFieldData>
         implements IndexOrdinalsFieldData {
 
     private final double minFrequency, maxFrequency;
@@ -108,12 +108,12 @@ private IndexOrdinalsFieldData loadGlobalInternal(DirectoryReader indexReader) {
     @Override
     public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
         return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger,
-                AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+                AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
     }
 
     @Override
-    protected AtomicOrdinalsFieldData empty(int maxDoc) {
-        return AbstractAtomicOrdinalsFieldData.empty();
+    protected LeafOrdinalsFieldData empty(int maxDoc) {
+        return AbstractLeafOrdinalsFieldData.empty();
     }
 
     protected TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java
index 93e57e5bcda08..5cbd20029d40c 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
@@ -65,17 +65,17 @@ public LatLonPointDVIndexFieldData(Index index, String fieldName) {
         }
 
         @Override
-        public AtomicGeoPointFieldData load(LeafReaderContext context) {
+        public LeafGeoPointFieldData load(LeafReaderContext context) {
             LeafReader reader = context.reader();
             FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName);
             if (info != null) {
                 checkCompatible(info);
             }
-            return new LatLonPointDVAtomicFieldData(reader, fieldName);
+            return new LatLonPointDVLeafFieldData(reader, fieldName);
         }
 
         @Override
-        public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception {
+        public LeafGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception {
             return load(context);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java
index 5d6575e43783c..7dede7fe2c6c1 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.util.Accountable;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -28,7 +28,7 @@
 import java.util.Collection;
 import java.util.Collections;
 
-public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData {
+public abstract class AbstractLeafGeoPointFieldData implements LeafGeoPointFieldData {
 
     @Override
     public final SortedBinaryDocValues getBytesValues() {
@@ -40,14 +40,14 @@ public final ScriptDocValues.GeoPoints getScriptValues() {
         return new ScriptDocValues.GeoPoints(getGeoPointValues());
     }
 
-    public static AtomicGeoPointFieldData empty(final int maxDoc) {
-        return new AbstractAtomicGeoPointFieldData() {
+    public static LeafGeoPointFieldData empty(final int maxDoc) {
+        return new AbstractLeafGeoPointFieldData() {
 
             @Override
             public long ramBytesUsed() {
                 return 0;
             }
-            
+
             @Override
             public Collection<Accountable> getChildResources() {
                 return Collections.emptyList();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java
similarity index 85%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java
index 9130e19fb26e1..dabbad1323c21 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java
@@ -22,7 +22,7 @@
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.Accountable;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -32,7 +32,7 @@
 import java.util.function.Function;
 
 
-public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsFieldData {
+public abstract class AbstractLeafOrdinalsFieldData implements LeafOrdinalsFieldData {
 
     public static final Function<SortedSetDocValues, ScriptDocValues<?>> DEFAULT_SCRIPT_FUNCTION =
             ((Function<SortedSetDocValues, SortedBinaryDocValues>) FieldData::toString)
@@ -40,7 +40,7 @@ public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsF
 
     private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction;
 
-    protected AbstractAtomicOrdinalsFieldData(Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) {
+    protected AbstractLeafOrdinalsFieldData(Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) {
         this.scriptFunction = scriptFunction;
     }
 
@@ -54,14 +54,14 @@ public final SortedBinaryDocValues getBytesValues() {
         return FieldData.toString(getOrdinalsValues());
     }
 
-    public static AtomicOrdinalsFieldData empty() {
-        return new AbstractAtomicOrdinalsFieldData(DEFAULT_SCRIPT_FUNCTION) {
+    public static LeafOrdinalsFieldData empty() {
+        return new AbstractLeafOrdinalsFieldData(DEFAULT_SCRIPT_FUNCTION) {
 
             @Override
             public long ramBytesUsed() {
                 return 0;
             }
-            
+
             @Override
             public Collection<Accountable> getChildResources() {
                 return Collections.emptyList();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
index 06352640dc162..26732b01d3000 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
@@ -32,19 +32,19 @@
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.SortOrder;
 
-public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BinaryDVAtomicFieldData> {
+public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BinaryDVLeafFieldData> {
 
     public BinaryDVIndexFieldData(Index index, String fieldName) {
         super(index, fieldName);
     }
 
     @Override
-    public BinaryDVAtomicFieldData load(LeafReaderContext context) {
-        return new BinaryDVAtomicFieldData(context.reader(), fieldName);
+    public BinaryDVLeafFieldData load(LeafReaderContext context) {
+        return new BinaryDVLeafFieldData(context.reader(), fieldName);
     }
 
     @Override
-    public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public BinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
         return load(context);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java
similarity index 88%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java
index 014220304d603..2a431b26ad739 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.util.Accountable;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
@@ -33,13 +33,13 @@
 import java.util.Collection;
 import java.util.Collections;
 
-/** {@link AtomicFieldData} impl on top of Lucene's binary doc values. */
-public class BinaryDVAtomicFieldData implements AtomicFieldData {
+/** {@link LeafFieldData} impl on top of Lucene's binary doc values. */
+public class BinaryDVLeafFieldData implements LeafFieldData {
 
     private final LeafReader reader;
     private final String field;
 
-    public BinaryDVAtomicFieldData(LeafReader reader, String field) {
+    public BinaryDVLeafFieldData(LeafReader reader, String field) {
         this.reader = reader;
         this.field = field;
     }
@@ -68,7 +68,7 @@ public void close() {
     public long ramBytesUsed() {
         return 0; // unknown
     }
-    
+
     @Override
     public Collection<Accountable> getChildResources() {
         return Collections.emptyList();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
index 2a9e797084b23..2cdb75f82eba3 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
@@ -39,7 +39,7 @@
 
 import java.io.IOException;
 
-public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BytesBinaryDVAtomicFieldData> {
+public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BytesBinaryDVLeafFieldData> {
 
     public BytesBinaryDVIndexFieldData(Index index, String fieldName) {
         super(index, fieldName);
@@ -57,16 +57,16 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu
     }
 
     @Override
-    public BytesBinaryDVAtomicFieldData load(LeafReaderContext context) {
+    public BytesBinaryDVLeafFieldData load(LeafReaderContext context) {
         try {
-            return new BytesBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldName));
+            return new BytesBinaryDVLeafFieldData(DocValues.getBinary(context.reader(), fieldName));
         } catch (IOException e) {
             throw new IllegalStateException("Cannot load doc values", e);
         }
     }
 
     @Override
-    public BytesBinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public BytesBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
         return load(context);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java
index af4c7fd0da6b2..209c2ba427a49 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 
@@ -31,11 +31,11 @@
 import java.util.Collection;
 import java.util.Collections;
 
-final class BytesBinaryDVAtomicFieldData implements AtomicFieldData {
+final class BytesBinaryDVLeafFieldData implements LeafFieldData {
 
     private final BinaryDocValues values;
 
-    BytesBinaryDVAtomicFieldData(BinaryDocValues values) {
+    BytesBinaryDVLeafFieldData(BinaryDocValues values) {
         super();
         this.values = values;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java
index 98ad1d5c59283..81a87ad8465b3 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AbstractSortedDocValues;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
@@ -69,11 +69,11 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
 
     }
 
-    private static class ConstantAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
+    private static class ConstantLeafFieldData extends AbstractLeafOrdinalsFieldData {
 
         private final String value;
 
-        ConstantAtomicFieldData(String value) {
+        ConstantLeafFieldData(String value) {
             super(DEFAULT_SCRIPT_FUNCTION);
             this.value = value;
         }
@@ -134,14 +134,14 @@ public void close() {
 
     }
 
-    private final ConstantAtomicFieldData atomicFieldData;
+    private final ConstantLeafFieldData atomicFieldData;
 
     private ConstantIndexFieldData(IndexSettings indexSettings, String name, String value) {
         super(indexSettings, name, null, null,
                 TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
                 TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
                 TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
-        atomicFieldData = new ConstantAtomicFieldData(value);
+        atomicFieldData = new ConstantLeafFieldData(value);
     }
 
     @Override
@@ -149,12 +149,12 @@ public void clear() {
     }
 
     @Override
-    public final AtomicOrdinalsFieldData load(LeafReaderContext context) {
+    public final LeafOrdinalsFieldData load(LeafReaderContext context) {
         return atomicFieldData;
     }
 
     @Override
-    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context)
+    public LeafOrdinalsFieldData loadDirect(LeafReaderContext context)
             throws Exception {
         return atomicFieldData;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
index 529bdb84b12ac..8e33d2deec611 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
@@ -71,7 +71,7 @@ public static class Builder implements IndexFieldData.Builder {
         private static final Set<String> BINARY_INDEX_FIELD_NAMES = unmodifiableSet(newHashSet(IdFieldMapper.NAME));
 
         private NumericType numericType;
-        private Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction = AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION;
+        private Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction = AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION;
         private RangeType rangeType;
 
         public Builder numericType(NumericType type) {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java
index dba7dfb0c9e99..0a0af57b116ca 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java
@@ -30,11 +30,11 @@
 import java.util.Collection;
 import java.util.Collections;
 
-final class LatLonPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData {
+final class LatLonPointDVLeafFieldData extends AbstractLeafGeoPointFieldData {
     private final LeafReader reader;
     private final String fieldName;
 
-    LatLonPointDVAtomicFieldData(LeafReader reader, String fieldName) {
+    LatLonPointDVLeafFieldData(LeafReader reader, String fieldName) {
         super();
         this.reader = reader;
         this.fieldName = fieldName;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java
similarity index 85%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java
index 6fd08b82668f6..46cd3a5ccaef2 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java
@@ -21,7 +21,7 @@
 
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.util.Accountable;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -32,13 +32,13 @@
 
 
 /**
- * Specialization of {@link AtomicNumericFieldData} for floating-point numerics.
+ * Specialization of {@link LeafNumericFieldData} for floating-point numerics.
  */
-abstract class AtomicDoubleFieldData implements AtomicNumericFieldData {
+abstract class LeafDoubleFieldData implements LeafNumericFieldData {
 
     private final long ramBytesUsed;
 
-    AtomicDoubleFieldData(long ramBytesUsed) {
+    LeafDoubleFieldData(long ramBytesUsed) {
         this.ramBytesUsed = ramBytesUsed;
     }
 
@@ -62,8 +62,8 @@ public final SortedNumericDocValues getLongValues() {
         return FieldData.castToLong(getDoubleValues());
     }
 
-    public static AtomicNumericFieldData empty(final int maxDoc) {
-        return new AtomicDoubleFieldData(0) {
+    public static LeafNumericFieldData empty(final int maxDoc) {
+        return new LeafDoubleFieldData(0) {
 
             @Override
             public SortedNumericDoubleValues getDoubleValues() {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java
index 7b5b6998308bd..1f519257fddc5 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.fielddata.plain;
 
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -27,9 +27,9 @@
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
 /**
- * Specialization of {@link AtomicNumericFieldData} for integers.
+ * Specialization of {@link LeafNumericFieldData} for integers.
  */
-abstract class AtomicLongFieldData implements AtomicNumericFieldData {
+abstract class LeafLongFieldData implements LeafNumericFieldData {
 
     private final long ramBytesUsed;
     /**
@@ -37,7 +37,7 @@ abstract class AtomicLongFieldData implements AtomicNumericFieldData {
      */
     private final NumericType numericType;
 
-    AtomicLongFieldData(long ramBytesUsed, NumericType numericType) {
+    LeafLongFieldData(long ramBytesUsed, NumericType numericType) {
         this.ramBytesUsed = ramBytesUsed;
         this.numericType = numericType;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
index 9e99ddc985ac8..5017745cf958a 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
@@ -96,15 +96,15 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu
     }
 
     @Override
-    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
         LeafReader reader = context.reader();
-        AtomicOrdinalsFieldData data = null;
+        LeafOrdinalsFieldData data = null;
 
         PagedBytesEstimator estimator =
             new PagedBytesEstimator(context, breakerService.getBreaker(CircuitBreaker.FIELDDATA), getFieldName());
         Terms terms = reader.terms(getFieldName());
         if (terms == null) {
-            data = AbstractAtomicOrdinalsFieldData.empty();
+            data = AbstractLeafOrdinalsFieldData.empty();
             estimator.afterLoad(null, data.ramBytesUsed());
             return data;
         }
@@ -135,7 +135,7 @@ public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exce
             PagedBytes.Reader bytesReader = bytes.freeze(true);
             final Ordinals ordinals = builder.build();
 
-            data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals);
+            data = new PagedBytesLeafFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals);
             success = true;
             return data;
         } finally {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java
similarity index 93%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java
index e90ba987b62ec..7c76e1677e4e6 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java
@@ -29,13 +29,13 @@
 import java.util.Collection;
 import java.util.List;
 
-public class PagedBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
+public class PagedBytesLeafFieldData extends AbstractLeafOrdinalsFieldData {
 
     private final PagedBytes.Reader bytes;
     private final PackedLongValues termOrdToBytesOffset;
     protected final Ordinals ordinals;
 
-    public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedLongValues termOrdToBytesOffset, Ordinals ordinals) {
+    public PagedBytesLeafFieldData(PagedBytes.Reader bytes, PackedLongValues termOrdToBytesOffset, Ordinals ordinals) {
         super(DEFAULT_SCRIPT_FUNCTION);
         this.bytes = bytes;
         this.termOrdToBytesOffset = termOrdToBytesOffset;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
index 952a7d0358dbd..f159288dec8cd 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -167,12 +167,12 @@ public NumericType getNumericType() {
     }
 
     @Override
-    public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception {
         return load(context);
     }
 
     @Override
-    public AtomicNumericFieldData load(LeafReaderContext context) {
+    public LeafNumericFieldData load(LeafReaderContext context) {
         final LeafReader reader = context.reader();
         final String field = fieldName;
 
@@ -194,7 +194,7 @@ public AtomicNumericFieldData load(LeafReaderContext context) {
      * A small helper class that can be configured to load nanosecond field data either in nanosecond resolution retaining the original
      * values or in millisecond resolution converting the nanosecond values to milliseconds
      */
-    public final class NanoSecondFieldData extends AtomicLongFieldData {
+    public final class NanoSecondFieldData extends LeafLongFieldData {
 
         private final LeafReader reader;
         private final String fieldName;
@@ -230,7 +230,7 @@ public SortedNumericDocValues getLongValuesAsNanos() {
      * {@link DocValues#unwrapSingleton(SortedNumericDocValues)} will return
      * the underlying single-valued NumericDocValues representation.
      */
-    static final class SortedNumericLongFieldData extends AtomicLongFieldData {
+    static final class SortedNumericLongFieldData extends LeafLongFieldData {
         final LeafReader reader;
         final String field;
 
@@ -268,7 +268,7 @@ public Collection<Accountable> getChildResources() {
      * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return
      * the underlying single-valued NumericDoubleValues representation.
      */
-    static final class SortedNumericHalfFloatFieldData extends AtomicDoubleFieldData {
+    static final class SortedNumericHalfFloatFieldData extends LeafDoubleFieldData {
         final LeafReader reader;
         final String field;
 
@@ -360,7 +360,7 @@ public int docValueCount() {
      * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return
      * the underlying single-valued NumericDoubleValues representation.
      */
-    static final class SortedNumericFloatFieldData extends AtomicDoubleFieldData {
+    static final class SortedNumericFloatFieldData extends LeafDoubleFieldData {
         final LeafReader reader;
         final String field;
 
@@ -452,7 +452,7 @@ public int docValueCount() {
      * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return
      * the underlying single-valued NumericDoubleValues representation.
      */
-    static final class SortedNumericDoubleFieldData extends AtomicDoubleFieldData {
+    static final class SortedNumericDoubleFieldData extends LeafDoubleFieldData {
         final LeafReader reader;
         final String field;
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesLeafFieldData.java
similarity index 84%
rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesLeafFieldData.java
index 5e8a30ab6e4ea..1412f9ab2ec2c 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesLeafFieldData.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.Accountable;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 
 import java.io.IOException;
@@ -32,14 +32,14 @@
 import java.util.function.Function;
 
 /**
- * An {@link AtomicFieldData} implementation that uses Lucene {@link SortedSetDocValues}.
+ * An {@link LeafFieldData} implementation that uses Lucene {@link SortedSetDocValues}.
  */
-public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
+public final class SortedSetDVBytesLeafFieldData extends AbstractLeafOrdinalsFieldData {
 
     private final LeafReader reader;
     private final String field;
 
-    SortedSetDVBytesAtomicFieldData(LeafReader reader, String field, Function<SortedSetDocValues,
+    SortedSetDVBytesLeafFieldData(LeafReader reader, String field, Function<SortedSetDocValues,
             ScriptDocValues<?>> scriptFunction) {
         super(scriptFunction);
         this.reader = reader;
@@ -63,7 +63,7 @@ public void close() {
     public long ramBytesUsed() {
         return 0; // unknown
     }
-    
+
     @Override
     public Collection<Accountable> getChildResources() {
         return Collections.emptyList();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
index 611f05f4af060..1a670a4df587f 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
@@ -92,12 +92,12 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu
     }
 
     @Override
-    public AtomicOrdinalsFieldData load(LeafReaderContext context) {
-        return new SortedSetDVBytesAtomicFieldData(context.reader(), fieldName, scriptFunction);
+    public LeafOrdinalsFieldData load(LeafReaderContext context) {
+        return new SortedSetDVBytesLeafFieldData(context.reader(), fieldName, scriptFunction);
     }
 
     @Override
-    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
         return load(context);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
index 16db2778e5754..789147775fe5a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -186,7 +186,7 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
                     deprecationLogger.deprecatedAndMaybeLog("id_field_data", ID_FIELD_DATA_DEPRECATION_MESSAGE);
                     final IndexFieldData<?> fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache,
                         breakerService, mapperService);
-                    return new IndexFieldData<AtomicFieldData>() {
+                    return new IndexFieldData<LeafFieldData>() {
 
                         @Override
                         public Index index() {
@@ -199,12 +199,12 @@ public String getFieldName() {
                         }
 
                         @Override
-                        public AtomicFieldData load(LeafReaderContext context) {
+                        public LeafFieldData load(LeafReaderContext context) {
                             return wrap(fieldData.load(context));
                         }
 
                         @Override
-                        public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
+                        public LeafFieldData loadDirect(LeafReaderContext context) throws Exception {
                             return wrap(fieldData.loadDirect(context));
                         }
 
@@ -232,8 +232,8 @@ public void clear() {
         }
     }
 
-    private static AtomicFieldData wrap(AtomicFieldData in) {
-        return new AtomicFieldData() {
+    private static LeafFieldData wrap(LeafFieldData in) {
+        return new LeafFieldData() {
 
             @Override
             public void close() {
diff --git a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
index 848902b39692d..978f97f2ec877 100644
--- a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
@@ -38,7 +38,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.shard.ShardId;
@@ -129,8 +129,8 @@ static class IndexFieldCache implements IndexFieldDataCache, IndexReader.ClosedL
 
         @Override
         @SuppressWarnings("unchecked")
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final LeafReaderContext context,
-                final IFD indexFieldData) throws Exception {
+        public <FD extends LeafFieldData, IFD extends IndexFieldData<FD>> FD load(final LeafReaderContext context,
+                                                                                  final IFD indexFieldData) throws Exception {
             final ShardId shardId = ShardUtils.extractShardId(context.reader());
             final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper();
             if (cacheHelper == null) {
@@ -140,7 +140,7 @@ public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(fina
             final Accountable accountable = cache.computeIfAbsent(key, k -> {
                 cacheHelper.addClosedListener(IndexFieldCache.this);
                 Collections.addAll(k.listeners, this.listeners);
-                final AtomicFieldData fieldData = indexFieldData.loadDirect(context);
+                final LeafFieldData fieldData = indexFieldData.loadDirect(context);
                 for (Listener listener : k.listeners) {
                     try {
                         listener.onCache(shardId, fieldName, fieldData);
@@ -156,8 +156,8 @@ public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(fina
 
         @Override
         @SuppressWarnings("unchecked")
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader,
-                final IFD indexFieldData) throws Exception {
+        public <FD extends LeafFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader,
+                                                                                          final IFD indexFieldData) throws Exception {
             final ShardId shardId = ShardUtils.extractShardId(indexReader);
             final IndexReader.CacheHelper cacheHelper = indexReader.getReaderCacheHelper();
             if (cacheHelper == null) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index 0de9d9cce3257..c60f9c7f52cef 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -287,7 +287,7 @@ public DateHistogramAggregationBuilder offset(String offset) {
     }
 
     /**
-     * Parse the string specification of an offset. 
+     * Parse the string specification of an offset.
      */
     public static long parseStringOffset(String offset) {
         if (offset.charAt(0) == '-') {
@@ -458,7 +458,7 @@ ZoneId rewriteTimeZone(QueryShardContext context) throws IOException {
         Instant instant = null;
         final IndexNumericFieldData fieldData = context.getForField(ft);
         for (LeafReaderContext ctx : reader.leaves()) {
-            AtomicNumericFieldData leafFD = fieldData.load(ctx);
+            LeafNumericFieldData leafFD = fieldData.load(ctx);
             SortedNumericDocValues values = leafFD.getLongValues();
             if (values.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
                 instant = Instant.ofEpochMilli(values.nextValue());
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index 3f77744b5a56e..483c3847c38d7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.DocValueBits;
 import org.elasticsearch.index.fielddata.HistogramValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -170,20 +170,20 @@ public FieldData(IndexOrdinalsFieldData indexFieldData) {
 
                 @Override
                 public SortedBinaryDocValues bytesValues(LeafReaderContext context) {
-                    final AtomicOrdinalsFieldData atomicFieldData = indexFieldData.load(context);
+                    final LeafOrdinalsFieldData atomicFieldData = indexFieldData.load(context);
                     return atomicFieldData.getBytesValues();
                 }
 
                 @Override
                 public SortedSetDocValues ordinalsValues(LeafReaderContext context) {
-                    final AtomicOrdinalsFieldData atomicFieldData = indexFieldData.load(context);
+                    final LeafOrdinalsFieldData atomicFieldData = indexFieldData.load(context);
                     return atomicFieldData.getOrdinalsValues();
                 }
 
                 @Override
                 public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) {
                     final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader());
-                    final AtomicOrdinalsFieldData atomicFieldData = global.load(context);
+                    final LeafOrdinalsFieldData atomicFieldData = global.load(context);
                     return atomicFieldData.getOrdinalsValues();
                 }
 
@@ -565,7 +565,7 @@ public org.elasticsearch.index.fielddata.MultiGeoPointValues geoPointValues(Leaf
             }
         }
     }
-    
+
     public abstract static class Histogram extends ValuesSource {
 
         public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException;
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java
index 7d1d4b9dfeb97..d03e5cde7faf8 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java
@@ -24,8 +24,8 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -114,7 +114,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                     format = fieldType.docValueFormat(formatDesc, null);
                 }
                 LeafReaderContext subReaderContext = null;
-                AtomicFieldData data = null;
+                LeafFieldData data = null;
                 SortedBinaryDocValues binaryValues = null; // binary / string / ip fields
                 SortedNumericDocValues longValues = null; // int / date fields
                 SortedNumericDoubleValues doubleValues = null; // floating-point fields
@@ -127,14 +127,14 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                         if (indexFieldData instanceof IndexNumericFieldData) {
                             NumericType numericType = ((IndexNumericFieldData) indexFieldData).getNumericType();
                             if (numericType.isFloatingPoint()) {
-                                doubleValues = ((AtomicNumericFieldData) data).getDoubleValues();
+                                doubleValues = ((LeafNumericFieldData) data).getDoubleValues();
                             } else {
                                 // by default nanoseconds are cut to milliseconds within aggregations
                                 // however for doc value fields we need the original nanosecond longs
                                 if (isNanosecond) {
                                     longValues = ((SortedNumericDVIndexFieldData.NanoSecondFieldData) data).getLongValuesAsNanos();
                                 } else {
-                                    longValues = ((AtomicNumericFieldData) data).getLongValues();
+                                    longValues = ((LeafNumericFieldData) data).getLongValues();
                                 }
                             }
                         } else {
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
index 048455ccb41e2..498884b6cffaa 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java
@@ -76,7 +76,7 @@ public void testDeletedDocs() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             SortedBinaryDocValues values = fieldData.getBytesValues();
             for (int i = 0; i < readerContext.reader().maxDoc(); ++i) {
                 assertTrue(values.advanceExact(i));
@@ -90,7 +90,7 @@ public void testSingleValueAllSet() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
             SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
@@ -160,7 +160,7 @@ public void testSingleValueWithMissing() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
             SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
@@ -181,7 +181,7 @@ public void testMultiValueAllSet() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
             SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
@@ -215,7 +215,7 @@ public void testMultiValueWithMissing() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
             SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
@@ -231,7 +231,7 @@ public void testMissingValueForAll() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             // Some impls (FST) return size 0 and some (PagedBytes) do take size in the case no actual data is loaded
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(0L));
 
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
index d20bf83d7d0f2..c7883c8cd7603 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
@@ -175,9 +175,9 @@ public void testEmpty() throws Exception {
         IndexFieldData<?> fieldData = getForField("non_existing_field");
         int max = randomInt(7);
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData previous = null;
+            LeafFieldData previous = null;
             for (int i = 0; i < max; i++) {
-                AtomicFieldData current = fieldData.load(readerContext);
+                LeafFieldData current = fieldData.load(readerContext);
                 assertThat(current.ramBytesUsed(), equalTo(0L));
                 if (previous != null) {
                     assertThat(current, not(sameInstance(previous)));
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
index 1539a800f2ae3..e1069cab4bfd0 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
@@ -467,7 +467,7 @@ public void testGlobalOrdinals() throws Exception {
         // First segment
         assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.Consumer.class));
         LeafReaderContext leaf = topLevelReader.leaves().get(0);
-        AtomicOrdinalsFieldData afd = globalOrdinals.load(leaf);
+        LeafOrdinalsFieldData afd = globalOrdinals.load(leaf);
         SortedSetDocValues values = afd.getOrdinalsValues();
         assertTrue(values.advanceExact(0));
         long ord = values.nextOrd();
@@ -553,7 +553,7 @@ public void testTermsEnum() throws Exception {
 
         IndexOrdinalsFieldData ifd = getForField("value");
         for (LeafReaderContext atomicReaderContext : atomicReaderContexts) {
-            AtomicOrdinalsFieldData afd = ifd.load(atomicReaderContext);
+            LeafOrdinalsFieldData afd = ifd.load(atomicReaderContext);
 
             TermsEnum termsEnum = afd.getOrdinalsValues().termsEnum();
             int size = 0;
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
index 9c92b74bdd023..277a5f9e700c8 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
@@ -100,7 +100,7 @@ public void testDocValue() throws Exception {
         bytesList2.sort(null);
 
         // Test SortedBinaryDocValues's decoding:
-        AtomicFieldData fieldData = indexFieldData.load(reader);
+        LeafFieldData fieldData = indexFieldData.load(reader);
         SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
 
         assertTrue(bytesValues.advanceExact(0));
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
index 2590dc2660333..ee1c11aa8f071 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -89,7 +89,7 @@ public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception {
 
     private SortedSetDVOrdinalsIndexFieldData createSortedDV(String fieldName, IndexFieldDataCache indexFieldDataCache) {
         return new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(), indexFieldDataCache, fieldName, new NoneCircuitBreakerService(),
-                AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+                AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
     }
 
     private PagedBytesIndexFieldData createPagedBytes(String fieldName, IndexFieldDataCache indexFieldDataCache) {
@@ -115,14 +115,14 @@ private class DummyAccountingFieldDataCache implements IndexFieldDataCache {
         private int cachedGlobally = 0;
 
         @Override
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData)
+        public <FD extends LeafFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData)
             throws Exception {
             return indexFieldData.loadDirect(context);
         }
 
         @Override
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader,
-            IFD indexFieldData) throws Exception {
+        public <FD extends LeafFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader,
+                                                                                          IFD indexFieldData) throws Exception {
             cachedGlobally++;
             return (IFD) indexFieldData.localGlobalDirect(indexReader);
         }
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
index 1f168b58b1153..2311d051a28d8 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
@@ -70,7 +70,7 @@ public void testFilterByFrequency() throws Exception {
                     .build(builderCtx).fieldType();
             IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
             for (LeafReaderContext context : contexts) {
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+                LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
                 SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
                 assertThat(2L, equalTo(bytesValues.getValueCount()));
                 assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
@@ -85,7 +85,7 @@ public void testFilterByFrequency() throws Exception {
                     .build(builderCtx).fieldType();
             IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
             for (LeafReaderContext context : contexts) {
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+                LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
                 SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
                 assertThat(1L, equalTo(bytesValues.getValueCount()));
                 assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5"));
@@ -100,7 +100,7 @@ public void testFilterByFrequency() throws Exception {
                     .build(builderCtx).fieldType();
             IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
             for (LeafReaderContext context : contexts) {
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+                LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
                 SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
                 assertThat(2L, equalTo(bytesValues.getValueCount()));
                 assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
@@ -116,7 +116,7 @@ public void testFilterByFrequency() throws Exception {
                     .build(builderCtx).fieldType();
             IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
             for (LeafReaderContext context : contexts) {
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+                LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
                 SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
                 assertThat(2L, equalTo(bytesValues.getValueCount()));
                 assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java
index a2d2474886381..68a91402f33d9 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java
@@ -23,7 +23,7 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafGeoPointFieldData;
 
 import java.util.List;
 
@@ -157,10 +157,10 @@ public void testSingleValueAllSet() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
-            MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues();
+            MultiGeoPointValues fieldValues = ((AbstractLeafGeoPointFieldData)fieldData).getGeoPointValues();
             assertValues(fieldValues, 0);
             assertValues(fieldValues, 1);
             assertValues(fieldValues, 2);
@@ -173,10 +173,10 @@ public void testSingleValueWithMissing() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
-            MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues();
+            MultiGeoPointValues fieldValues = ((AbstractLeafGeoPointFieldData)fieldData).getGeoPointValues();
             assertValues(fieldValues, 0);
             assertMissing(fieldValues, 1);
             assertValues(fieldValues, 2);
@@ -189,10 +189,10 @@ public void testMultiValueAllSet() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
-            MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues();
+            MultiGeoPointValues fieldValues = ((AbstractLeafGeoPointFieldData)fieldData).getGeoPointValues();
             assertValues(fieldValues, 0);
             assertValues(fieldValues, 1);
             assertValues(fieldValues, 2);
@@ -205,10 +205,10 @@ public void testMultiValueWithMissing() throws Exception {
         IndexFieldData<?> indexFieldData = getForField("value");
         List<LeafReaderContext> readerContexts = refreshReader();
         for (LeafReaderContext readerContext : readerContexts) {
-            AtomicFieldData fieldData = indexFieldData.load(readerContext);
+            LeafFieldData fieldData = indexFieldData.load(readerContext);
             assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed()));
 
-            MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues();
+            MultiGeoPointValues fieldValues = ((AbstractLeafGeoPointFieldData)fieldData).getGeoPointValues();
 
             assertValues(fieldValues, 0);
             assertMissing(fieldValues, 1);
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 8cece535a6841..d70158714e614 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -132,8 +132,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
         IndexFieldData<?> ifd1 = ifdService.getForField(mapper1);
         IndexFieldData<?> ifd2 = ifdService.getForField(mapper2);
         LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0);
-        AtomicFieldData loadField1 = ifd1.load(leafReaderContext);
-        AtomicFieldData loadField2 = ifd2.load(leafReaderContext);
+        LeafFieldData loadField1 = ifd1.load(leafReaderContext);
+        LeafFieldData loadField2 = ifd2.load(leafReaderContext);
 
         assertEquals(2, onCacheCalled.get());
         assertEquals(0, onRemovalCalled.get());
@@ -201,7 +201,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
         });
         IndexFieldData<?> ifd = ifdService.getForField(mapper1);
         LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0);
-        AtomicFieldData load = ifd.load(leafReaderContext);
+        LeafFieldData load = ifd.load(leafReaderContext);
         assertEquals(1, onCacheCalled.get());
         assertEquals(0, onRemovalCalled.get());
         reader.close();
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
index 18f3691e18008..be3dde6e56b68 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
@@ -35,8 +35,8 @@
  *  eg. BytesRefFieldComparatorSource makes decisions based on whether the field data implements WithOrdinals. */
 public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTests {
 
-    public static IndexFieldData<AtomicFieldData> hideOrdinals(final IndexFieldData<?> in) {
-        return new IndexFieldData<AtomicFieldData>() {
+    public static IndexFieldData<LeafFieldData> hideOrdinals(final IndexFieldData<?> in) {
+        return new IndexFieldData<LeafFieldData>() {
 
             @Override
             public Index index() {
@@ -49,12 +49,12 @@ public String getFieldName() {
             }
 
             @Override
-            public AtomicFieldData load(LeafReaderContext context) {
+            public LeafFieldData load(LeafReaderContext context) {
                 return in.load(context);
             }
 
             @Override
-            public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
+            public LeafFieldData loadDirect(LeafReaderContext context) throws Exception {
                 return in.loadDirect(context);
             }
 
@@ -80,7 +80,7 @@ public void clear() {
 
     @SuppressWarnings("unchecked")
     @Override
-    public IndexFieldData<AtomicFieldData> getForField(String fieldName) {
+    public IndexFieldData<LeafFieldData> getForField(String fieldName) {
         return hideOrdinals(super.getForField(fieldName));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
index 82302c0493669..2b33fa874ec0e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
@@ -41,7 +41,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedNumericDVIndexFieldData;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
@@ -304,7 +304,7 @@ public void testDateNanoDocValues() throws IOException {
         // Read index and check the doc values
         DirectoryReader reader = DirectoryReader.open(w);
         assertTrue(reader.leaves().size() > 0);
-        AtomicNumericFieldData a = fieldData.load(reader.leaves().get(0).reader().getContext());
+        LeafNumericFieldData a = fieldData.load(reader.leaves().get(0).reader().getContext());
         SortedNumericDocValues docValues = a.getLongValues();
         assertEquals(0, docValues.nextDoc());
         assertEquals(1, docValues.nextDoc());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java
index 39366b5bc4d14..1782242e204ed 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -70,7 +70,7 @@ public static void testDocValues(Function<String, IndexService> createIndex) thr
         MappedFieldType ft = mapperService.fieldType(TypeFieldMapper.NAME);
         IndexOrdinalsFieldData fd = (IndexOrdinalsFieldData) ft.fielddataBuilder("test").build(mapperService.getIndexSettings(),
                 ft, new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService);
-        AtomicOrdinalsFieldData afd = fd.load(r.leaves().get(0));
+        LeafOrdinalsFieldData afd = fd.load(r.leaves().get(0));
         SortedSetDocValues values = afd.getOrdinalsValues();
         assertTrue(values.advanceExact(0));
         assertEquals(0, values.nextOrd());
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index ba68e1b6a2013..c103bfc1a4d8b 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -123,7 +123,7 @@ public void testClusterAlias() throws IOException {
         IndexFieldData<?> forField = context.getForField(mapper.fieldType());
         String expected = clusterAlias == null ? context.getIndexSettings().getIndexMetaData().getIndex().getName()
             : clusterAlias + ":" + context.getIndexSettings().getIndex().getName();
-        assertEquals(expected, ((AbstractAtomicOrdinalsFieldData)forField.load(null)).getOrdinalsValues().lookupOrd(0).utf8ToString());
+        assertEquals(expected, ((AbstractLeafOrdinalsFieldData)forField.load(null)).getOrdinalsValues().lookupOrd(0).utf8ToString());
     }
 
     public void testGetFullyQualifiedIndex() {
diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
index f72a13d382a80..60c95758bc904 100644
--- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
@@ -54,8 +54,8 @@
 import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -87,15 +87,15 @@ public class FunctionScoreTests extends ESTestCase {
     /**
      * Stub for IndexFieldData. Needed by some score functions. Returns 1 as count always.
      */
-    private static class IndexFieldDataStub implements IndexFieldData<AtomicFieldData> {
+    private static class IndexFieldDataStub implements IndexFieldData<LeafFieldData> {
         @Override
         public String getFieldName() {
             return "test";
         }
 
         @Override
-        public AtomicFieldData load(LeafReaderContext context) {
-            return new AtomicFieldData() {
+        public LeafFieldData load(LeafReaderContext context) {
+            return new LeafFieldData() {
 
                 @Override
                 public ScriptDocValues getScriptValues() {
@@ -139,7 +139,7 @@ public void close() {
         }
 
         @Override
-        public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
+        public LeafFieldData loadDirect(LeafReaderContext context) throws Exception {
             throw new UnsupportedOperationException(UNSUPPORTED);
         }
 
@@ -182,8 +182,8 @@ public String getFieldName() {
         }
 
         @Override
-        public AtomicNumericFieldData load(LeafReaderContext context) {
-            return new AtomicNumericFieldData() {
+        public LeafNumericFieldData load(LeafReaderContext context) {
+            return new LeafNumericFieldData() {
                 @Override
                 public SortedNumericDocValues getLongValues() {
                     throw new UnsupportedOperationException(UNSUPPORTED);
@@ -236,7 +236,7 @@ public void close() {
         }
 
         @Override
-        public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Exception {
+        public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception {
             throw new UnsupportedOperationException(UNSUPPORTED);
         }
 
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
index fe75e95457a45..6aaa4a1ed6a58 100644
--- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.search.lookup;
 
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -67,12 +67,12 @@ public void testFieldAliases() {
     }
 
     private IndexFieldData<?> createFieldData(ScriptDocValues scriptDocValues) {
-        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
-        doReturn(scriptDocValues).when(atomicFieldData).getScriptValues();
+        LeafFieldData leafFieldData = mock(LeafFieldData.class);
+        doReturn(scriptDocValues).when(leafFieldData).getScriptValues();
 
         IndexFieldData<?> fieldData = mock(IndexFieldData.class);
         when(fieldData.getFieldName()).thenReturn("field");
-        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+        doReturn(leafFieldData).when(fieldData).load(anyObject());
 
         return fieldData;
     }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
index 68a4dbd779514..6b77018c11638 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.xcontent.XContentSubParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicHistogramFieldData;
+import org.elasticsearch.index.fielddata.LeafHistogramFieldData;
 import org.elasticsearch.index.fielddata.HistogramValue;
 import org.elasticsearch.index.fielddata.HistogramValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -204,8 +204,8 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
                     return new IndexHistogramFieldData(indexSettings.getIndex(), fieldType.name()) {
 
                         @Override
-                        public AtomicHistogramFieldData load(LeafReaderContext context) {
-                            return new AtomicHistogramFieldData() {
+                        public LeafHistogramFieldData load(LeafReaderContext context) {
+                            return new LeafHistogramFieldData() {
                                 @Override
                                 public HistogramValues getHistogramValues() throws IOException {
                                     try {
@@ -258,7 +258,7 @@ public void close() {
                         }
 
                         @Override
-                        public AtomicHistogramFieldData loadDirect(LeafReaderContext context) throws Exception {
+                        public LeafHistogramFieldData loadDirect(LeafReaderContext context) throws Exception {
                             return load(context);
                         }
 
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
index 3996cdcd81de1..9bbb2a5d6d6c1 100644
--- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
@@ -31,13 +31,13 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.DynamicKeyFieldMapper;
@@ -409,15 +409,15 @@ public void clear() {
         }
 
         @Override
-        public AtomicOrdinalsFieldData load(LeafReaderContext context) {
-            AtomicOrdinalsFieldData fieldData = delegate.load(context);
-            return new KeyedFlatObjectAtomicFieldData(key, fieldData);
+        public LeafOrdinalsFieldData load(LeafReaderContext context) {
+            LeafOrdinalsFieldData fieldData = delegate.load(context);
+            return new KeyedFlatObjectLeafFieldData(key, fieldData);
         }
 
         @Override
-        public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
-            AtomicOrdinalsFieldData fieldData = delegate.loadDirect(context);
-            return new KeyedFlatObjectAtomicFieldData(key, fieldData);
+        public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+            LeafOrdinalsFieldData fieldData = delegate.loadDirect(context);
+            return new KeyedFlatObjectLeafFieldData(key, fieldData);
         }
 
         @Override
@@ -463,7 +463,7 @@ public IndexFieldData<?> build(IndexSettings indexSettings,
                                            MapperService mapperService) {
                 String fieldName = fieldType.name();
                 IndexOrdinalsFieldData delegate = new SortedSetDVOrdinalsIndexFieldData(indexSettings,
-                    cache, fieldName, breakerService, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+                    cache, fieldName, breakerService, AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
                 return new KeyedFlatObjectFieldData(key, delegate);
             }
         }
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectLeafFieldData.java
similarity index 94%
rename from x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java
rename to x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectLeafFieldData.java
index 7f5a5235095b8..608631ffde5cd 100644
--- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectLeafFieldData.java
@@ -11,11 +11,11 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -30,13 +30,13 @@
  * In order to support all usage patterns, the delegate's ordinal values are shifted
  * to range from 0 to the number of total values.
  */
-public class KeyedFlatObjectAtomicFieldData implements AtomicOrdinalsFieldData {
+public class KeyedFlatObjectLeafFieldData implements LeafOrdinalsFieldData {
 
     private final String key;
-    private final AtomicOrdinalsFieldData delegate;
+    private final LeafOrdinalsFieldData delegate;
 
-    KeyedFlatObjectAtomicFieldData(String key,
-                                   AtomicOrdinalsFieldData delegate) {
+    KeyedFlatObjectLeafFieldData(String key,
+                                 LeafOrdinalsFieldData delegate) {
         this.key = key;
         this.delegate = delegate;
     }
@@ -78,7 +78,7 @@ public void close() {
 
     @Override
     public ScriptDocValues<?> getScriptValues() {
-        return AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
+        return AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
             .apply(getOrdinalsValues());
     }
 
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
index 8974bfb020c9a..c821a9ea0df83 100644
--- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
@@ -8,7 +8,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.search.lookup.LeafDocLookup;
@@ -175,12 +175,12 @@ public void testScriptDocValuesLookup() {
     }
 
     private IndexFieldData<?> createFieldData(ScriptDocValues<?> scriptDocValues) {
-        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
-        doReturn(scriptDocValues).when(atomicFieldData).getScriptValues();
+        LeafFieldData leafFieldData = mock(LeafFieldData.class);
+        doReturn(scriptDocValues).when(leafFieldData).getScriptValues();
 
         IndexFieldData<?> fieldData = mock(IndexFieldData.class);
         when(fieldData.getFieldName()).thenReturn("field");
-        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+        doReturn(leafFieldData).when(fieldData).load(anyObject());
 
         return fieldData;
     }
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectLeafFieldDataTests.java
similarity index 75%
rename from x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java
rename to x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectLeafFieldDataTests.java
index bb6c195395d4e..b7e7aec3fe476 100644
--- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectLeafFieldDataTests.java
@@ -9,8 +9,8 @@
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -18,8 +18,8 @@
 
 import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
 
-public class KeyedFlatObjectAtomicFieldDataTests extends ESTestCase {
-    private AtomicOrdinalsFieldData delegate;
+public class KeyedFlatObjectLeafFieldDataTests extends ESTestCase {
+    private LeafOrdinalsFieldData delegate;
 
     @Before
     public void setUpDelegate() {
@@ -49,7 +49,7 @@ public void setUpDelegate() {
             }
         }
 
-        delegate = new MockAtomicOrdinalsFieldData(allTerms, documentOrds);
+        delegate = new MockLeafOrdinalsFieldData(allTerms, documentOrds);
     }
 
     private BytesRef prefixedValue(String key, String value) {
@@ -65,11 +65,11 @@ public void testFindOrdinalBounds() throws IOException {
         testFindOrdinalBounds("cantaloupe", delegate, 40, 40);
         testFindOrdinalBounds("cucumber", delegate, 41, 59);
 
-        AtomicOrdinalsFieldData emptyDelegate = new MockAtomicOrdinalsFieldData(new BytesRef[0], new long[0]);
+        LeafOrdinalsFieldData emptyDelegate = new MockLeafOrdinalsFieldData(new BytesRef[0], new long[0]);
         testFindOrdinalBounds("apple", emptyDelegate, -1, -1);
 
         BytesRef[] terms = new BytesRef[] { prefixedValue("prefix", "value") };
-        AtomicOrdinalsFieldData singleValueDelegate = new MockAtomicOrdinalsFieldData(terms, new long[0]);
+        LeafOrdinalsFieldData singleValueDelegate = new MockLeafOrdinalsFieldData(terms, new long[0]);
         testFindOrdinalBounds("prefix", singleValueDelegate, 0, 0);
         testFindOrdinalBounds("prefix1", singleValueDelegate, -1, -1);
 
@@ -78,7 +78,7 @@ public void testFindOrdinalBounds() throws IOException {
             prefixedValue("prefix1", "value1"),
             prefixedValue("prefix2", "value"),
             prefixedValue("prefix3", "value")};
-        AtomicOrdinalsFieldData oddLengthDelegate = new MockAtomicOrdinalsFieldData(terms, new long[0]);
+        LeafOrdinalsFieldData oddLengthDelegate = new MockLeafOrdinalsFieldData(terms, new long[0]);
         testFindOrdinalBounds("prefix", oddLengthDelegate, 0, 0);
         testFindOrdinalBounds("prefix1", oddLengthDelegate, 1, 2);
         testFindOrdinalBounds("prefix2", oddLengthDelegate, 3, 3);
@@ -86,31 +86,31 @@ public void testFindOrdinalBounds() throws IOException {
     }
 
     public void testFindOrdinalBounds(String key,
-                                      AtomicOrdinalsFieldData delegate,
+                                      LeafOrdinalsFieldData delegate,
                                       long expectedMinOrd,
                                       long expectedMacOrd) throws IOException {
         BytesRef bytesKey = new BytesRef(key);
 
-        long actualMinOrd = KeyedFlatObjectAtomicFieldData.findMinOrd(bytesKey, delegate.getOrdinalsValues());
+        long actualMinOrd = KeyedFlatObjectLeafFieldData.findMinOrd(bytesKey, delegate.getOrdinalsValues());
         assertEquals(expectedMinOrd,  actualMinOrd);
 
-        long actualMaxOrd = KeyedFlatObjectAtomicFieldData.findMaxOrd(bytesKey, delegate.getOrdinalsValues());
+        long actualMaxOrd = KeyedFlatObjectLeafFieldData.findMaxOrd(bytesKey, delegate.getOrdinalsValues());
         assertEquals(expectedMacOrd, actualMaxOrd);
     }
 
     public void testAdvanceExact() throws IOException {
-        AtomicOrdinalsFieldData avocadoFieldData = new KeyedFlatObjectAtomicFieldData("avocado", delegate);
+        LeafOrdinalsFieldData avocadoFieldData = new KeyedFlatObjectLeafFieldData("avocado", delegate);
         assertFalse(avocadoFieldData.getOrdinalsValues().advanceExact(0));
 
-        AtomicOrdinalsFieldData bananaFieldData = new KeyedFlatObjectAtomicFieldData("banana", delegate);
+        LeafOrdinalsFieldData bananaFieldData = new KeyedFlatObjectLeafFieldData("banana", delegate);
         assertTrue(bananaFieldData.getOrdinalsValues().advanceExact(0));
 
-        AtomicOrdinalsFieldData nonexistentFieldData = new KeyedFlatObjectAtomicFieldData("berry", delegate);
+        LeafOrdinalsFieldData nonexistentFieldData = new KeyedFlatObjectLeafFieldData("berry", delegate);
         assertFalse(nonexistentFieldData.getOrdinalsValues().advanceExact(0));
     }
 
     public void testNextOrd() throws IOException {
-        AtomicOrdinalsFieldData fieldData = new KeyedFlatObjectAtomicFieldData("banana", delegate);
+        LeafOrdinalsFieldData fieldData = new KeyedFlatObjectLeafFieldData("banana", delegate);
         SortedSetDocValues docValues = fieldData.getOrdinalsValues();
         docValues.advanceExact(0);
 
@@ -128,25 +128,25 @@ public void testNextOrd() throws IOException {
     }
 
     public void testLookupOrd() throws IOException {
-        AtomicOrdinalsFieldData appleFieldData = new KeyedFlatObjectAtomicFieldData("apple", delegate);
+        LeafOrdinalsFieldData appleFieldData = new KeyedFlatObjectLeafFieldData("apple", delegate);
         SortedSetDocValues appleDocValues = appleFieldData.getOrdinalsValues();
         assertEquals(new BytesRef("value0"), appleDocValues.lookupOrd(0));
 
-        AtomicOrdinalsFieldData cantaloupeFieldData = new KeyedFlatObjectAtomicFieldData("cantaloupe", delegate);
+        LeafOrdinalsFieldData cantaloupeFieldData = new KeyedFlatObjectLeafFieldData("cantaloupe", delegate);
         SortedSetDocValues cantaloupeDocValues = cantaloupeFieldData.getOrdinalsValues();
         assertEquals(new BytesRef("value40"), cantaloupeDocValues.lookupOrd(0));
 
-        AtomicOrdinalsFieldData cucumberFieldData = new KeyedFlatObjectAtomicFieldData("cucumber", delegate);
+        LeafOrdinalsFieldData cucumberFieldData = new KeyedFlatObjectLeafFieldData("cucumber", delegate);
         SortedSetDocValues cucumberDocValues = cucumberFieldData.getOrdinalsValues();
         assertEquals(new BytesRef("value41"), cucumberDocValues.lookupOrd(0));
     }
 
-    private static class MockAtomicOrdinalsFieldData extends AbstractAtomicOrdinalsFieldData {
+    private static class MockLeafOrdinalsFieldData extends AbstractLeafOrdinalsFieldData {
         private final SortedSetDocValues docValues;
 
-        MockAtomicOrdinalsFieldData(BytesRef[] allTerms,
-                                    long[] documentOrds) {
-            super(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+        MockLeafOrdinalsFieldData(BytesRef[] allTerms,
+                                  long[] documentOrds) {
+            super(AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
             this.docValues = new MockSortedSetDocValues(allTerms, documentOrds);
         }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java
index 6c18e473a0ca1..890f8af2246c8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -61,7 +61,7 @@ public void setup() throws Exception {
         String name = "_field";
         indexFieldDataCache = new DummyAccountingFieldDataCache();
         sortedSetDVOrdinalsIndexFieldData = new SortedSetDVOrdinalsIndexFieldData(indexSettings,indexFieldDataCache,  name,
-                circuitBreakerService, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+                circuitBreakerService, AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
         pagedBytesIndexFieldData = new PagedBytesIndexFieldData(indexSettings, name, indexFieldDataCache,
                 circuitBreakerService, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
                 TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
@@ -95,7 +95,7 @@ public void destroy() throws Exception {
     public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception {
         assertThat(indexFieldDataCache.topLevelBuilds, equalTo(0));
         IndexOrdinalsFieldData global = sortedSetDVOrdinalsIndexFieldData.loadGlobal(ir);
-        AtomicOrdinalsFieldData atomic = global.load(ir.leaves().get(0));
+        LeafOrdinalsFieldData atomic = global.load(ir.leaves().get(0));
         assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(numDocs));
         assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1));
 
@@ -108,13 +108,13 @@ public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception {
 
     public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception {
         for (LeafReaderContext context : ir.leaves()) {
-            AtomicOrdinalsFieldData atomic = sortedSetDVOrdinalsIndexFieldData.load(context);
+            LeafOrdinalsFieldData atomic = sortedSetDVOrdinalsIndexFieldData.load(context);
             assertThat(atomic.getOrdinalsValues().getValueCount(), greaterThanOrEqualTo(1L));
         }
 
         DirectoryReader ir = FieldSubsetReader.wrap(this.ir, new CharacterRunAutomaton(Automata.makeEmpty()));
         for (LeafReaderContext context : ir.leaves()) {
-            AtomicOrdinalsFieldData atomic = sortedSetDVOrdinalsIndexFieldData.load(context);
+            LeafOrdinalsFieldData atomic = sortedSetDVOrdinalsIndexFieldData.load(context);
             assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0L));
         }
         // dv based field data doesn't use index field data cache, so in the end noting should have been added
@@ -124,7 +124,7 @@ public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception {
     public void testPagedBytesIndexFieldData_global() throws Exception {
         assertThat(indexFieldDataCache.topLevelBuilds, equalTo(0));
         IndexOrdinalsFieldData global = pagedBytesIndexFieldData.loadGlobal(ir);
-        AtomicOrdinalsFieldData atomic = global.load(ir.leaves().get(0));
+        LeafOrdinalsFieldData atomic = global.load(ir.leaves().get(0));
         assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(numDocs));
         assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1));
 
@@ -138,14 +138,14 @@ public void testPagedBytesIndexFieldData_global() throws Exception {
     public void testPagedBytesIndexFieldData_segment() throws Exception {
         assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(0));
         for (LeafReaderContext context : ir.leaves()) {
-            AtomicOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context);
+            LeafOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context);
             assertThat(atomic.getOrdinalsValues().getValueCount(), greaterThanOrEqualTo(1L));
         }
         assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(ir.leaves().size()));
 
         DirectoryReader ir = FieldSubsetReader.wrap(this.ir, new CharacterRunAutomaton(Automata.makeEmpty()));
         for (LeafReaderContext context : ir.leaves()) {
-            AtomicOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context);
+            LeafOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context);
             assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0L));
         }
         assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(ir.leaves().size()));
@@ -168,15 +168,15 @@ private static class DummyAccountingFieldDataCache implements IndexFieldDataCach
         private int topLevelBuilds = 0;
 
         @Override
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData)
+        public <FD extends LeafFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData)
                 throws Exception {
             leafLevelBuilds++;
             return indexFieldData.loadDirect(context);
         }
 
         @Override
-        public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader,
-                                                                                            IFD indexFieldData) throws Exception {
+        public <FD extends LeafFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader,
+                                                                                          IFD indexFieldData) throws Exception {
             topLevelBuilds++;
             return (IFD) indexFieldData.localGlobalDirect(indexReader);
         }
diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java
index 090c9ab3c3799..289ea777684bc 100644
--- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.search.sort.SortOrder;
 
 
-public class VectorDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<VectorDVAtomicFieldData> {
+public class VectorDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<VectorDVLeafFieldData> {
 
     public VectorDVIndexFieldData(Index index, String fieldName) {
         super(index, fieldName);
@@ -44,12 +44,12 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu
     }
 
     @Override
-    public VectorDVAtomicFieldData load(LeafReaderContext context) {
-        return new VectorDVAtomicFieldData(context.reader(), fieldName);
+    public VectorDVLeafFieldData load(LeafReaderContext context) {
+        return new VectorDVLeafFieldData(context.reader(), fieldName);
     }
 
     @Override
-    public VectorDVAtomicFieldData loadDirect(LeafReaderContext context) {
+    public VectorDVLeafFieldData loadDirect(LeafReaderContext context) {
         return load(context);
     }
 
diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java
similarity index 90%
rename from x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java
index 5f092afe17795..ec588dbc0c034 100644
--- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 
@@ -20,12 +20,12 @@
 import java.util.Collection;
 import java.util.Collections;
 
-final class VectorDVAtomicFieldData implements AtomicFieldData {
+final class VectorDVLeafFieldData implements LeafFieldData {
 
     private final LeafReader reader;
     private final String field;
 
-    VectorDVAtomicFieldData(LeafReader reader, String field) {
+    VectorDVLeafFieldData(LeafReader reader, String field) {
         this.reader = reader;
         this.field = field;
     }

From 1f439a21cb70512cf2295019422d917da4750b9c Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 17 Mar 2020 14:04:42 +0000
Subject: [PATCH 075/285] Fix deprecation in history retention docs (#53655)

This commit adjusts a `deprecation[...]` message in the docs since such
messages must be on a single line. It also moves this message to the start of
the description of the deprecated setting as is the case with other such
messages.
---
 docs/reference/index-modules/history-retention.asciidoc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/reference/index-modules/history-retention.asciidoc b/docs/reference/index-modules/history-retention.asciidoc
index 6ace77c3533ff..fb4aa26ab9b07 100644
--- a/docs/reference/index-modules/history-retention.asciidoc
+++ b/docs/reference/index-modules/history-retention.asciidoc
@@ -54,12 +54,11 @@ reasonable recovery scenarios.
 
 `index.soft_deletes.enabled`::
 
+  deprecated:[7.6.0, Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.]
   Whether or not soft deletes are enabled on the index. Soft deletes can only be
   configured at index creation and only on indices created on or after 6.5.0.
   The default value is `true`.
 
-  deprecated::[7.6, Creating indices with soft-deletes disabled is
-  deprecated and will be removed in future Elasticsearch versions.]
 
 `index.soft_deletes.retention_lease.period`::
 

From 3c250056484733376c296cd020de76e1a0b8d4cc Mon Sep 17 00:00:00 2001
From: Gordon Brown <gordon.brown@elastic.co>
Date: Tue, 17 Mar 2020 10:20:48 -0600
Subject: [PATCH 076/285] Update the Skip version in hidden index YAML tests
 (#53641)

This commit adjusts the version ranges used for the _cat API hidden
index/alias tests.
---
 .../rest-api-spec/test/cat.aliases/40_hidden.yml  |  9 +++------
 .../rest-api-spec/test/cat.indices/20_hidden.yml  | 15 +++++----------
 2 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
index 6866ff595fefd..3aa7fdbb1f760 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
@@ -1,9 +1,8 @@
 ---
 "Test cat aliases output with a hidden index with a hidden alias":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
 
   - do:
       indices.create:
@@ -59,9 +58,8 @@
 ---
 "Test cat aliases output with a hidden index with a visible alias":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
 
   - do:
       indices.create:
@@ -106,9 +104,8 @@
 ---
 "Test cat aliases output with a visible index with a hidden alias":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
index 3a4fe28c85996..f357f362fd8eb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/20_hidden.yml
@@ -1,9 +1,8 @@
 ---
 "Test cat indices output for hidden index":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
   - do:
       indices.create:
         index: index1
@@ -40,9 +39,8 @@
 ---
 "Test cat indices output for dot-hidden index and dot-prefixed pattern":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
   - do:
       indices.create:
         index: .index1
@@ -79,9 +77,8 @@
 ---
 "Test cat indices output with a hidden index with a visible alias":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
 
   - do:
       indices.create:
@@ -142,9 +139,8 @@
 ---
 "Test cat indices output with a hidden index with a hidden alias":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
 
   - do:
       indices.create:
@@ -203,9 +199,8 @@
 ---
 "Test cat indices output with a hidden index, dot-hidden alias and dot pattern":
   - skip:
-      version: "- 7.99.99"
+      version: "- 7.6.99"
       reason: "hidden indices and aliases were added in 7.7.0"
-      # TODO: Update this in/after backport of https://github.com/elastic/elasticsearch/pull/53248
 
   - do:
       indices.create:

From a295161697e7f1d8abc8e57782660bd2414d6ccf Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Tue, 17 Mar 2020 09:44:29 -0700
Subject: [PATCH 077/285] add tests to SumAggregatorTests (#53568)

This adds tests for supported ValuesSourceTypes, unmapped fields,
scripting, and the missing param. The tests for unmapped fields and
scripting are migrated from the SumIT integration test
---
 .../metrics/SumAggregatorTests.java           | 257 +++++++++++++++++-
 .../search/aggregations/metrics/SumIT.java    | 168 ++----------
 2 files changed, 263 insertions(+), 162 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java
index ff76aa4d0edef..ceeed769d3082 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java
@@ -25,6 +25,8 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
@@ -36,20 +38,47 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.TriConsumer;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 
 public class SumAggregatorTests extends AggregatorTestCase {
 
     private static final String FIELD_NAME = "field";
+    private static final String VALUE_SCRIPT_NAME = "value_script";
+    private static final String FIELD_SCRIPT_NAME = "field_script";
 
     public void testNoDocs() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
@@ -166,26 +195,198 @@ public void testSummationAccuracy() throws IOException {
 
     private void verifySummationOfDoubles(double[] values, double expected, double delta) throws IOException {
         testCase(new MatchAllDocsQuery(),
+            sum("_name").field(FIELD_NAME),
             iw -> {
                 for (double value : values) {
                     iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, NumericUtils.doubleToSortableLong(value))));
                 }
             },
             result -> assertEquals(expected, result.getValue(), delta),
-            NumberFieldMapper.NumberType.DOUBLE
+            singleton(defaultFieldType(NumberType.DOUBLE))
+        );
+    }
+
+    public void testUnmapped() throws IOException {
+        sumRandomDocsTestCase(randomIntBetween(1, 5),
+            sum("_name").field("unknown_field"),
+            (sum, docs, result) -> {
+                assertEquals(0d, result.getValue(), 0d);
+                assertFalse(AggregationInspectionHelper.hasValue(result));
+            }
+        );
+    }
+
+    public void testPartiallyUnmapped() throws IOException {
+        final MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+
+        final SumAggregationBuilder builder = sum("_name")
+            .field(fieldType.name());
+
+        final int numDocs = randomIntBetween(10, 100);
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        int sum = 0;
+        for (int i = 0; i < numDocs; i++) {
+            final long value = randomLongBetween(0, 1000);
+            sum += value;
+            docs.add(singleton(new NumericDocValuesField(fieldType.name(), value)));
+        }
+
+        try (Directory mappedDirectory = newDirectory(); Directory unmappedDirectory = newDirectory()) {
+            try (RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory)) {
+                mappedWriter.addDocuments(docs);
+            }
+
+            new RandomIndexWriter(random(), unmappedDirectory).close();
+
+            try (IndexReader mappedReader = DirectoryReader.open(mappedDirectory);
+                 IndexReader unmappedReader =  DirectoryReader.open(unmappedDirectory);
+                 MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) {
+
+                final IndexSearcher searcher = newSearcher(multiReader, true, true);
+
+                final InternalSum internalSum = search(searcher, new MatchAllDocsQuery(), builder, fieldType);
+                assertEquals(sum, internalSum.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(internalSum));
+            }
+        }
+    }
+
+    public void testValueScriptSingleValuedField() throws IOException {
+        sumRandomDocsTestCase(1,
+            sum("_name")
+                .field(FIELD_NAME)
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap())),
+            (sum, docs, result) -> {
+                assertEquals(sum + docs.size(), result.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(result));
+            }
+        );
+    }
+
+    public void testValueScriptMultiValuedField() throws IOException {
+        final int valuesPerField = randomIntBetween(2, 5);
+        sumRandomDocsTestCase(valuesPerField,
+            sum("_name")
+                .field(FIELD_NAME)
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap())),
+            (sum, docs, result) -> {
+                assertEquals(sum + (docs.size() * valuesPerField), result.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(result));
+            }
+        );
+    }
+
+    public void testFieldScriptSingleValuedField() throws IOException {
+        sumRandomDocsTestCase(1,
+            sum("_name")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", FIELD_NAME))),
+            (sum, docs, result) -> {
+                assertEquals(sum + docs.size(), result.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(result));
+            }
+        );
+    }
+
+    public void testFieldScriptMultiValuedField() throws IOException {
+        final int valuesPerField = randomIntBetween(2, 5);
+        sumRandomDocsTestCase(valuesPerField,
+            sum("_name")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", FIELD_NAME))),
+            (sum, docs, result) -> {
+                assertEquals(sum + (docs.size() * valuesPerField), result.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(result));
+            }
+        );
+    }
+
+    public void testMissing() throws IOException {
+        final MappedFieldType aggField = defaultFieldType();
+        final MappedFieldType irrelevantField = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        irrelevantField.setName("irrelevant_field");
+
+        final int numDocs = randomIntBetween(10, 100);
+        final long missingValue = randomLongBetween(1, 1000);
+        long sum = 0;
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                final long value = randomLongBetween(0, 1000);
+                sum += value;
+                docs.add(singleton(new NumericDocValuesField(aggField.name(), value)));
+            } else {
+                sum += missingValue;
+                docs.add(singleton(new NumericDocValuesField(irrelevantField.name(), randomLong())));
+            }
+        }
+        final long finalSum = sum;
+
+        testCase(new MatchAllDocsQuery(),
+            sum("_name")
+                .field(aggField.name())
+                .missing(missingValue),
+            writer -> writer.addDocuments(docs),
+            internalSum -> {
+                assertEquals(finalSum, internalSum.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(internalSum));
+            },
+            List.of(aggField, irrelevantField)
+        );
+    }
+
+    public void testMissingUnmapped() throws IOException {
+        final long missingValue = randomLongBetween(1, 1000);
+        sumRandomDocsTestCase(randomIntBetween(1, 5),
+            sum("_name")
+                .field("unknown_field")
+                .missing(missingValue),
+            (sum, docs, result) -> {
+                assertEquals(docs.size() * missingValue, result.getValue(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(result));
+            }
+        );
+    }
+
+    private void sumRandomDocsTestCase(int valuesPerField,
+                                       SumAggregationBuilder builder,
+                                       TriConsumer<Long, List<Set<IndexableField>>, InternalSum> verify) throws IOException {
+
+        final MappedFieldType fieldType = defaultFieldType();
+
+        final int numDocs = randomIntBetween(10, 100);
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        long sum = 0;
+        for (int iDoc = 0; iDoc < numDocs; iDoc++) {
+            Set<IndexableField> doc = new HashSet<>();
+            for (int iValue = 0; iValue < valuesPerField; iValue++) {
+                final long value = randomLongBetween(0, 1000);
+                sum += value;
+                doc.add(new SortedNumericDocValuesField(fieldType.name(), value));
+            }
+            docs.add(doc);
+        }
+        final long finalSum = sum;
+
+        testCase(new MatchAllDocsQuery(),
+            builder,
+            writer -> writer.addDocuments(docs),
+            internalSum -> verify.apply(finalSum, docs, internalSum),
+            singleton(fieldType)
         );
     }
 
     private void testCase(Query query,
                           CheckedConsumer<RandomIndexWriter, IOException> indexer,
                           Consumer<InternalSum> verify) throws IOException {
-        testCase(query, indexer, verify, NumberFieldMapper.NumberType.LONG);
+        testCase(query, sum("_name").field(FIELD_NAME), indexer, verify, singleton(defaultFieldType()));
     }
 
     private void testCase(Query query,
+                          SumAggregationBuilder aggregationBuilder,
                           CheckedConsumer<RandomIndexWriter, IOException> indexer,
                           Consumer<InternalSum> verify,
-                          NumberFieldMapper.NumberType fieldNumberType) throws IOException {
+                          Collection<MappedFieldType> fieldTypes) throws IOException {
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
                 indexer.accept(indexWriter);
@@ -194,21 +395,49 @@ private void testCase(Query query,
             try (IndexReader indexReader = DirectoryReader.open(directory)) {
                 IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
 
-                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(fieldNumberType);
-                fieldType.setName(FIELD_NAME);
-                fieldType.setHasDocValues(true);
+                final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]);
+                final InternalSum internalSum = search(indexSearcher, query, aggregationBuilder, fieldTypesArray);
+                verify.accept(internalSum);
+            }
+        }
+    }
 
-                SumAggregationBuilder aggregationBuilder = new SumAggregationBuilder("_name");
-                aggregationBuilder.field(FIELD_NAME);
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return singletonList(CoreValuesSourceType.NUMERIC);
+    }
 
-                SumAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
-                aggregator.preCollection();
-                indexSearcher.search(query, aggregator);
-                aggregator.postCollection();
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new SumAggregationBuilder("_name")
+            .field(fieldName);
+    }
 
-                verify.accept((InternalSum) aggregator.buildAggregation(0L));
+    @Override
+    protected ScriptService getMockScriptService() {
+        final Map<String, Function<Map<String, Object>, Object>> scripts = Map.of(
+            VALUE_SCRIPT_NAME, vars -> ((Number) vars.get("_value")).doubleValue() + 1,
+            FIELD_SCRIPT_NAME, vars -> {
+                final String fieldName = (String) vars.get("field");
+                final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc");
+                return lookup.get(fieldName).stream()
+                    .map(value -> ((Number) value).longValue() + 1)
+                    .collect(toList());
             }
-        }
+        );
+        final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap());
+        final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
+
+    private static MappedFieldType defaultFieldType() {
+        return defaultFieldType(NumberType.LONG);
     }
 
+    private static MappedFieldType defaultFieldType(NumberType numberType) {
+        final MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(numberType);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+        return fieldType;
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index b63bd845c7a2c..fb6b021eda0bf 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -35,9 +35,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -48,8 +46,6 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.search.aggregations.metrics.MetricAggScriptPlugin.METRIC_SCRIPT_ENGINE;
 import static org.elasticsearch.search.aggregations.metrics.MetricAggScriptPlugin.RANDOM_SCRIPT;
-import static org.elasticsearch.search.aggregations.metrics.MetricAggScriptPlugin.SUM_VALUES_FIELD_SCRIPT;
-import static org.elasticsearch.search.aggregations.metrics.MetricAggScriptPlugin.VALUE_FIELD_SCRIPT;
 import static org.elasticsearch.search.aggregations.metrics.MetricAggScriptPlugin.VALUE_SCRIPT;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -111,20 +107,9 @@ public void testEmptyAggregation() throws Exception {
         assertThat(sum.getValue(), equalTo(0.0));
     }
 
+    /** This test has been moved to {@link SumAggregatorTests#testUnmapped()} */
     @Override
-    public void testUnmapped() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx_unmapped")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("value"))
-                .get();
-
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo(0.0));
-    }
+    public void testUnmapped() throws Exception {}
 
     @Override
     public void testSingleValuedField() throws Exception {
@@ -179,120 +164,33 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat((double) ((InternalAggregation)sum).getProperty("value"), equalTo(expectedSumValue));
     }
 
+    /** This test has been moved to {@link SumAggregatorTests#testPartiallyUnmapped()} */
     @Override
-    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("value"))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-    }
+    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testValueScriptSingleValuedField()} */
     @Override
-    public void testSingleValuedFieldWithValueScript() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("value").script(
-                    new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, VALUE_SCRIPT, Collections.emptyMap())))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-    }
+    public void testSingleValuedFieldWithValueScript() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testValueScriptSingleValuedField()} */
     @Override
-    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("increment", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("value").script(new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, VALUE_SCRIPT, params)))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-    }
+    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptSingleValuedField()} */
     @Override
-    public void testScriptSingleValued() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").script(
-                    new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, VALUE_FIELD_SCRIPT, Collections.singletonMap("field", "value"))))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-    }
+    public void testScriptSingleValued() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptSingleValuedField()} */
     @Override
-    public void testScriptSingleValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("inc", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").script(new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, VALUE_FIELD_SCRIPT, params)))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-    }
+    public void testScriptSingleValuedWithParams() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptMultiValuedField()} */
     @Override
-    public void testScriptMultiValued() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").script(
-                    new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, SUM_VALUES_FIELD_SCRIPT, Collections.emptyMap())))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12));
-    }
+    public void testScriptMultiValued() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptMultiValuedField()} */
     @Override
-    public void testScriptMultiValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("inc", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        sum("sum").script(new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, SUM_VALUES_FIELD_SCRIPT, params)))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12+12+13));
-    }
+    public void testScriptMultiValuedWithParams() throws Exception {}
 
     @Override
     public void testMultiValuedField() throws Exception {
@@ -310,39 +208,13 @@ public void testMultiValuedField() throws Exception {
         assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12));
     }
 
+    /** this test has been moved to {@link SumAggregatorTests#testValueScriptMultiValuedField()} */
     @Override
-    public void testMultiValuedFieldWithValueScript() throws Exception {
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("values").script(
-                    new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, VALUE_SCRIPT, Collections.emptyMap())))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 2 + 3 + 3 + 4 + 4 + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 + 11 + 11 + 12));
-    }
+    public void testMultiValuedFieldWithValueScript() throws Exception {}
 
+    /** this test has been moved to {@link SumAggregatorTests#testValueScriptMultiValuedField()} */
     @Override
-    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("increment", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
-                .addAggregation(sum("sum").field("values")
-                    .script(new Script(ScriptType.INLINE, METRIC_SCRIPT_ENGINE, VALUE_SCRIPT, params)))
-                .get();
-
-        assertHitCount(searchResponse, 10);
-
-        Sum sum = searchResponse.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getName(), equalTo("sum"));
-        assertThat(sum.getValue(), equalTo((double) 2 + 3 + 3 + 4 + 4 + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 + 11 + 11 + 12));
-    }
+    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {}
 
     @Override
     public void testOrderByEmptyAggregation() throws Exception {

From 263e525e496498b9a2006675178105cb20114d2c Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Tue, 17 Mar 2020 10:55:07 -0600
Subject: [PATCH 078/285] Add REST API for ComponentTemplate CRUD (#53558)

* Add REST API for ComponentTemplate CRUD

This adds the Put/Get/DeleteComponentTemplate APIs that allow inserting, retrieving, and removing
ComponentTemplateMetadata into the cluster state metadata.

These APIs are currently only available behind a feature flag system property -
`es.itv2_feature_flag_registered`.

Relates to #53101

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../client/RestHighLevelClientTests.java      |   5 +-
 .../archives/integ-test-zip/build.gradle      |   7 +
 .../cluster.delete_component_template.json    |  35 ++++
 .../api/cluster.get_component_template.json   |  41 ++++
 .../api/cluster.put_component_template.json   |  45 +++++
 .../cluster.component_template/10_basic.yml   |  47 +++++
 .../elasticsearch/action/ActionModule.java    |  37 ++++
 .../delete/DeleteComponentTemplateAction.java |  90 +++++++++
 ...ransportDeleteComponentTemplateAction.java |  79 ++++++++
 .../get/GetComponentTemplateAction.java       | 171 ++++++++++++++++
 .../TransportGetComponentTemplateAction.java  |  94 +++++++++
 .../put/PutComponentTemplateAction.java       | 151 ++++++++++++++
 .../TransportPutComponentTemplateAction.java  |  88 +++++++++
 .../cluster/metadata/ComponentTemplate.java   | 149 +-------------
 .../MetaDataIndexTemplateService.java         | 100 ++++++++++
 .../cluster/metadata/Template.java            | 187 ++++++++++++++++++
 .../RestDeleteComponentTemplateAction.java    |  53 +++++
 .../RestGetComponentTemplateAction.java       |  81 ++++++++
 .../RestPutComponentTemplateAction.java       |  60 ++++++
 .../GetComponentTemplateResponseTests.java    |  54 +++++
 .../metadata/ComponentTemplateTests.java      |  14 +-
 .../MetaDataIndexTemplateServiceTests.java    |  24 ++-
 .../metadata/ToAndFromJsonMetaDataTests.java  |   4 +-
 23 files changed, 1452 insertions(+), 164 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
 rename server/src/test/java/org/elasticsearch/{action/admin/indices/template/put => cluster/metadata}/MetaDataIndexTemplateServiceTests.java (93%)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index a8e8037930741..870db8a63a970 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -790,7 +790,10 @@ public void testApiNamingConventions() throws Exception {
             "indices.get_upgrade",
             "indices.put_alias",
             "render_search_template",
-            "scripts_painless_execute"
+            "scripts_painless_execute",
+            "cluster.put_component_template",
+            "cluster.get_component_template",
+            "cluster.delete_component_template"
         };
         //These API are not required for high-level client feature completeness
         String[] notRequiredApi = new String[] {
diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle
index ed8f893d7f840..5fb5b36553b91 100644
--- a/distribution/archives/integ-test-zip/build.gradle
+++ b/distribution/archives/integ-test-zip/build.gradle
@@ -1,3 +1,4 @@
+import org.elasticsearch.gradle.info.BuildParams
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -32,3 +33,9 @@ integTest.runner {
     systemProperty 'tests.logfile', '--external--'
   }
 }
+
+testClusters.integTest {
+  if (BuildParams.isSnapshotBuild() == false) {
+    systemProperty 'es.itv2_feature_flag_registered', 'true'
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
new file mode 100644
index 0000000000000..6ddfe6c7ead5f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
@@ -0,0 +1,35 @@
+{
+  "cluster.delete_component_template":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-component-templates.html",
+      "description":"Deletes a component template"
+    },
+    "stability":"stable",
+    "url":{
+      "paths":[
+        {
+          "path":"/_component_template/{name}",
+          "methods":[
+            "DELETE"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the template"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
new file mode 100644
index 0000000000000..27bd093e620f0
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
@@ -0,0 +1,41 @@
+{
+  "cluster.get_component_template":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-component-templates.html",
+      "description":"Returns one or more component templates"
+    },
+    "stability":"stable",
+    "url":{
+      "paths":[
+        {
+          "path":"/_component_template",
+          "methods":[
+            "GET"
+          ]
+        },
+        {
+          "path":"/_component_template/{name}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "name":{
+              "type":"list",
+              "description":"The comma separated names of the component templates"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "local":{
+        "type":"boolean",
+        "description":"Return local information, do not retrieve the state from master node (default: false)"
+      }
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
new file mode 100644
index 0000000000000..3bdac2357d023
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
@@ -0,0 +1,45 @@
+{
+  "cluster.put_component_template":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-component-templates.html",
+      "description":"Creates or updates a component template"
+    },
+    "stability":"stable",
+    "url":{
+      "paths":[
+        {
+          "path":"/_component_template/{name}",
+          "methods":[
+            "PUT",
+            "POST"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the template"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "create":{
+        "type":"boolean",
+        "description":"Whether the index template should only be added if new or can also replace an existing one",
+        "default":false
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    },
+    "body":{
+      "description":"The template definition",
+      "required":true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
new file mode 100644
index 0000000000000..1ce79e3ca18c7
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
@@ -0,0 +1,47 @@
+---
+"Basic CRUD":
+  - skip:
+      version: " - 7.99.99"
+      reason: not backported yet
+
+  - do:
+      cluster.put_component_template:
+        name: test
+        body:
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                field:
+                  type: keyword
+            aliases:
+              aliasname: {}
+          version: 2
+          _meta:
+            foo: bar
+            baz:
+              eggplant: true
+
+  - do:
+      cluster.get_component_template:
+        name: test
+
+  - match: {component_templates.0.name: test}
+  - match: {component_templates.0.component_template.version: 2}
+  - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
+  - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
+  - match: {component_templates.0.component_template.template.mappings: {properties: {field: {type: keyword}}}}
+  - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}
+
+  - do:
+      cluster.delete_component_template:
+        name: test
+
+  - do:
+      catch: missing
+      cluster.get_component_template:
+        name: test
+
+  - is_false: test
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index affbb7a41dd31..aa8d92129f577 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Build;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
@@ -142,11 +143,17 @@
 import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
+import org.elasticsearch.action.admin.indices.template.get.TransportGetComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplatesAction;
+import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.upgrade.get.TransportUpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusAction;
@@ -269,11 +276,13 @@
 import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction;
 import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction;
+import org.elasticsearch.rest.action.admin.indices.RestDeleteComponentTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestFlushAction;
 import org.elasticsearch.rest.action.admin.indices.RestForceMergeAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction;
+import org.elasticsearch.rest.action.admin.indices.RestGetComponentTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction;
@@ -286,6 +295,7 @@
 import org.elasticsearch.rest.action.admin.indices.RestIndicesShardStoresAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndicesStatsAction;
 import org.elasticsearch.rest.action.admin.indices.RestOpenIndexAction;
+import org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction;
 import org.elasticsearch.rest.action.admin.indices.RestRecoveryAction;
@@ -361,6 +371,23 @@ public class ActionModule extends AbstractModule {
 
     private static final Logger logger = LogManager.getLogger(ActionModule.class);
 
+    private static final boolean ITV2_FEATURE_FLAG_REGISTERED;
+
+    static {
+        final String property = System.getProperty("es.itv2_feature_flag_registered");
+        if (Build.CURRENT.isSnapshot() && property != null) {
+            throw new IllegalArgumentException("es.itv2_feature_flag_registered is only supported in non-snapshot builds");
+        }
+        if (Build.CURRENT.isSnapshot() || "true".equals(property)) {
+            ITV2_FEATURE_FLAG_REGISTERED = true;
+        } else if ("false".equals(property) || property == null) {
+            ITV2_FEATURE_FLAG_REGISTERED = false;
+        } else {
+            throw new IllegalArgumentException("expected es.itv2_feature_flag_registered to be unset, true, or false but was [" +
+                property + "]");
+        }
+    }
+
     private final Settings settings;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopedSettings;
@@ -486,6 +513,11 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(PutIndexTemplateAction.INSTANCE, TransportPutIndexTemplateAction.class);
         actions.register(GetIndexTemplatesAction.INSTANCE, TransportGetIndexTemplatesAction.class);
         actions.register(DeleteIndexTemplateAction.INSTANCE, TransportDeleteIndexTemplateAction.class);
+        if (ITV2_FEATURE_FLAG_REGISTERED) {
+            actions.register(PutComponentTemplateAction.INSTANCE, TransportPutComponentTemplateAction.class);
+            actions.register(GetComponentTemplateAction.INSTANCE, TransportGetComponentTemplateAction.class);
+            actions.register(DeleteComponentTemplateAction.INSTANCE, TransportDeleteComponentTemplateAction.class);
+        }
         actions.register(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class);
         actions.register(RefreshAction.INSTANCE, TransportRefreshAction.class);
         actions.register(FlushAction.INSTANCE, TransportFlushAction.class);
@@ -621,6 +653,11 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestGetIndexTemplateAction());
         registerHandler.accept(new RestPutIndexTemplateAction());
         registerHandler.accept(new RestDeleteIndexTemplateAction());
+        if (ITV2_FEATURE_FLAG_REGISTERED) {
+            registerHandler.accept(new RestPutComponentTemplateAction());
+            registerHandler.accept(new RestGetComponentTemplateAction());
+            registerHandler.accept(new RestDeleteComponentTemplateAction());
+        }
 
         registerHandler.accept(new RestPutMappingAction());
         registerHandler.accept(new RestGetMappingAction());
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java
new file mode 100644
index 0000000000000..a0dc94a92f14b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.delete;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class DeleteComponentTemplateAction extends ActionType<AcknowledgedResponse> {
+
+    public static final DeleteComponentTemplateAction INSTANCE = new DeleteComponentTemplateAction();
+    public static final String NAME = "cluster:admin/component_template/delete";
+
+    private DeleteComponentTemplateAction() {
+        super(NAME, AcknowledgedResponse::new);
+    }
+
+    public static class Request extends MasterNodeRequest<Request> {
+
+        private String name;
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            name = in.readString();
+        }
+
+        public Request() { }
+
+        /**
+         * Constructs a new delete index request for the specified name.
+         */
+        public Request(String name) {
+            this.name = name;
+        }
+
+        /**
+         * Set the index template name to delete.
+         */
+        public Request name(String name) {
+            this.name = name;
+            return this;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+            if (name == null) {
+                validationException = addValidationError("name is missing", validationException);
+            }
+            return validationException;
+        }
+
+        /**
+         * The index template name to delete.
+         */
+        public String name() {
+            return name;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(name);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java
new file mode 100644
index 0000000000000..95b087868d31f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.delete;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+
+public class TransportDeleteComponentTemplateAction
+    extends TransportMasterNodeAction<DeleteComponentTemplateAction.Request, AcknowledgedResponse> {
+
+    private static final Logger logger = LogManager.getLogger(TransportDeleteComponentTemplateAction.class);
+
+    private final MetaDataIndexTemplateService indexTemplateService;
+
+    @Inject
+    public TransportDeleteComponentTemplateAction(TransportService transportService, ClusterService clusterService,
+                                                  ThreadPool threadPool, MetaDataIndexTemplateService indexTemplateService,
+                                                  ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(DeleteComponentTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters,
+            DeleteComponentTemplateAction.Request::new, indexNameExpressionResolver);
+        this.indexTemplateService = indexTemplateService;
+    }
+
+    @Override
+    protected String executor() {
+        // we go async right away
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(DeleteComponentTemplateAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected void masterOperation(Task task, final DeleteComponentTemplateAction.Request request, final ClusterState state,
+                                   final ActionListener<AcknowledgedResponse> listener) {
+        indexTemplateService.removeComponentTemplate(request.name(), request.masterNodeTimeout(), listener);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
new file mode 100644
index 0000000000000..34242c1d6aa9d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.get;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Action to retrieve one or more component templates
+ */
+public class GetComponentTemplateAction extends ActionType<GetComponentTemplateAction.Response> {
+
+    public static final GetComponentTemplateAction INSTANCE = new GetComponentTemplateAction();
+    public static final String NAME = "cluster:admin/component_template/get";
+
+    private GetComponentTemplateAction() {
+        super(NAME, GetComponentTemplateAction.Response::new);
+    }
+
+    /**
+     * Request that to retrieve one or more component templates
+     */
+    public static class Request extends MasterNodeReadRequest<Request> {
+
+        private String[] names;
+
+        public Request() { }
+
+        public Request(String... names) {
+            this.names = names;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            names = in.readStringArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(names);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+            if (names == null) {
+                validationException = addValidationError("names is null or empty", validationException);
+            } else {
+                for (String name : names) {
+                    if (name == null || Strings.hasText(name) == false) {
+                        validationException = addValidationError("name is missing", validationException);
+                    }
+                }
+            }
+            return validationException;
+        }
+
+        /**
+         * Sets the names of the component templates.
+         */
+        public Request names(String... names) {
+            this.names = names;
+            return this;
+        }
+
+        /**
+         * The names of the component templates.
+         */
+        public String[] names() {
+            return this.names;
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+        public static final ParseField NAME = new ParseField("name");
+        public static final ParseField COMPONENT_TEMPLATES = new ParseField("component_templates");
+        public static final ParseField COMPONENT_TEMPLATE = new ParseField("component_template");
+
+        private final Map<String, ComponentTemplate> componentTemplates;
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            int size = in.readVInt();
+            componentTemplates = new HashMap<>();
+            for (int i = 0 ; i < size ; i++) {
+                componentTemplates.put(in.readString(), new ComponentTemplate(in));
+            }
+        }
+
+        public Response(Map<String, ComponentTemplate> componentTemplates) {
+            this.componentTemplates = componentTemplates;
+        }
+
+        public Map<String, ComponentTemplate> getComponentTemplates() {
+            return componentTemplates;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(componentTemplates.size());
+            for (Map.Entry<String, ComponentTemplate> componentTemplate : componentTemplates.entrySet()) {
+                out.writeString(componentTemplate.getKey());
+                componentTemplate.getValue().writeTo(out);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response that = (Response) o;
+            return Objects.equals(componentTemplates, that.componentTemplates);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(componentTemplates);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.startArray(COMPONENT_TEMPLATES.getPreferredName());
+            for (Map.Entry<String, ComponentTemplate> componentTemplate : this.componentTemplates.entrySet()) {
+                builder.startObject();
+                builder.field(NAME.getPreferredName(), componentTemplate.getKey());
+                builder.field(COMPONENT_TEMPLATE.getPreferredName(), componentTemplate.getValue());
+                builder.endObject();
+            }
+            builder.endArray();
+            builder.endObject();
+            return builder;
+        }
+
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
new file mode 100644
index 0000000000000..dcb8ea57d01e3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.get;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TransportGetComponentTemplateAction extends
+    TransportMasterNodeReadAction<GetComponentTemplateAction.Request, GetComponentTemplateAction.Response> {
+
+    @Inject
+    public TransportGetComponentTemplateAction(TransportService transportService, ClusterService clusterService,
+                                               ThreadPool threadPool, ActionFilters actionFilters,
+                                               IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(GetComponentTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters,
+            GetComponentTemplateAction.Request::new, indexNameExpressionResolver);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected GetComponentTemplateAction.Response read(StreamInput in) throws IOException {
+        return new GetComponentTemplateAction.Response(in);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(GetComponentTemplateAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+    }
+
+    @Override
+    protected void masterOperation(Task task, GetComponentTemplateAction.Request request, ClusterState state,
+                                   ActionListener<GetComponentTemplateAction.Response> listener) {
+        Map<String, ComponentTemplate> allTemplates = state.metaData().componentTemplates();
+
+        // If we did not ask for a specific name, then we return all templates
+        if (request.names().length == 0) {
+            listener.onResponse(new GetComponentTemplateAction.Response(allTemplates));
+            return;
+        }
+
+        final Map<String, ComponentTemplate> results = new HashMap<>();
+        for (String name : request.names()) {
+            if (Regex.isSimpleMatchPattern(name)) {
+                for (Map.Entry<String, ComponentTemplate> entry : allTemplates.entrySet()) {
+                    if (Regex.simpleMatch(name, entry.getKey())) {
+                        results.put(entry.getKey(), entry.getValue());
+                    }
+                }
+            } else if (allTemplates.containsKey(name)) {
+                results.put(name, allTemplates.get(name));
+            }
+        }
+
+        listener.onResponse(new GetComponentTemplateAction.Response(results));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java
new file mode 100644
index 0000000000000..448be77793e5e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.put;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * An action for putting a single component template into the cluster state
+ */
+public class PutComponentTemplateAction extends ActionType<AcknowledgedResponse> {
+
+    public static final PutComponentTemplateAction INSTANCE = new PutComponentTemplateAction();
+    public static final String NAME = "cluster:admin/component_template/put";
+
+    private PutComponentTemplateAction() {
+        super(NAME, AcknowledgedResponse::new);
+    }
+
+    /**
+     * A request for putting a single component template into the cluster state
+     */
+    public static class Request extends MasterNodeRequest<Request> {
+        private final String name;
+        @Nullable
+        private String cause;
+        private boolean create;
+        private ComponentTemplate componentTemplate;
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.name = in.readString();
+            this.cause = in.readOptionalString();
+            this.create = in.readBoolean();
+            this.componentTemplate = new ComponentTemplate(in);
+        }
+
+        /**
+         * Constructs a new put component template request with the provided name.
+         */
+        public Request(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(name);
+            out.writeOptionalString(cause);
+            out.writeBoolean(create);
+            this.componentTemplate.writeTo(out);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+            if (name == null || Strings.hasText(name) == false) {
+                validationException = addValidationError("name is missing", validationException);
+            }
+            if (componentTemplate == null) {
+                validationException = addValidationError("a component template is required", validationException);
+            }
+            return validationException;
+        }
+
+        /**
+         * The name of the index template.
+         */
+        public String name() {
+            return this.name;
+        }
+
+        /**
+         * Set to {@code true} to force only creation, not an update of an index template. If it already
+         * exists, it will fail with an {@link IllegalArgumentException}.
+         */
+        public Request create(boolean create) {
+            this.create = create;
+            return this;
+        }
+
+        public boolean create() {
+            return create;
+        }
+
+        /**
+         * The cause for this index template creation.
+         */
+        public Request cause(@Nullable String cause) {
+            this.cause = cause;
+            return this;
+        }
+
+        @Nullable
+        public String cause() {
+            return this.cause;
+        }
+
+        /**
+         * The component template that will be inserted into the cluster state
+         */
+        public Request componentTemplate(ComponentTemplate template) {
+            this.componentTemplate = template;
+            return this;
+        }
+
+        public ComponentTemplate componentTemplate() {
+            return this.componentTemplate;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder("PutComponentRequest[");
+            sb.append("name=").append(name);
+            sb.append(", cause=").append(cause);
+            sb.append(", create=").append(create);
+            sb.append(", component_template=").append(componentTemplate);
+            sb.append("]");
+            return sb.toString();
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
new file mode 100644
index 0000000000000..c915a56231472
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.put;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+
+public class TransportPutComponentTemplateAction
+    extends TransportMasterNodeAction<PutComponentTemplateAction.Request, AcknowledgedResponse> {
+
+    private final MetaDataIndexTemplateService indexTemplateService;
+
+    @Inject
+    public TransportPutComponentTemplateAction(TransportService transportService, ClusterService clusterService,
+                                               ThreadPool threadPool, MetaDataIndexTemplateService indexTemplateService,
+                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(PutComponentTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters,
+            PutComponentTemplateAction.Request::new, indexNameExpressionResolver);
+        this.indexTemplateService = indexTemplateService;
+    }
+
+    @Override
+    protected String executor() {
+        // we go async right away
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(PutComponentTemplateAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected void masterOperation(Task task, final PutComponentTemplateAction.Request request, final ClusterState state,
+                                   final ActionListener<AcknowledgedResponse> listener) {
+        ComponentTemplate componentTemplate = request.componentTemplate();
+        Template template = componentTemplate.template();
+        // Normalize the index settings if necessary
+        if (template.settings() != null) {
+            Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
+            template = new Template(settings.build(), template.mappings(), template.aliases());
+            componentTemplate = new ComponentTemplate(template, componentTemplate.version(), componentTemplate.metadata());
+        }
+        indexTemplateService.putComponentTemplate(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
+            componentTemplate, listener);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
index 2b160b1b2fb11..0cc8c8f8a940d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
@@ -24,26 +24,19 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
 /**
- * A component template is a re-usable template as well as metadata about the template. Each
+ * A component template is a re-usable {@link Template} as well as metadata about the template. Each
  * component template is expected to be valid on its own. For example, if a component template
  * contains a field "foo", it's expected to contain all the necessary settings/mappings/etc for the
  * "foo" field. These component templates make up the individual pieces composing an index template.
@@ -157,144 +150,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.endObject();
         return builder;
     }
-
-    static class Template extends AbstractDiffable<Template> implements ToXContentObject {
-        private static final ParseField SETTINGS = new ParseField("settings");
-        private static final ParseField MAPPINGS = new ParseField("mappings");
-        private static final ParseField ALIASES = new ParseField("aliases");
-
-        @SuppressWarnings("unchecked")
-        private static final ConstructingObjectParser<Template, Void> PARSER = new ConstructingObjectParser<>("template", false,
-            a -> new Template((Settings) a[0], (CompressedXContent) a[1], (Map<String, AliasMetaData>) a[2]));
-
-        static {
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
-                new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
-                Map<String, AliasMetaData> aliasMap = new HashMap<>();
-                while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    AliasMetaData alias = AliasMetaData.Builder.fromXContent(p);
-                    aliasMap.put(alias.alias(), alias);
-                }
-                return aliasMap;
-            }, ALIASES);
-        }
-
-        @Nullable
-        private final Settings settings;
-        @Nullable
-        private final CompressedXContent mappings;
-        @Nullable
-        private final Map<String, AliasMetaData> aliases;
-
-        Template(@Nullable Settings settings, @Nullable CompressedXContent mappings, @Nullable Map<String, AliasMetaData> aliases) {
-            this.settings = settings;
-            this.mappings = mappings;
-            this.aliases = aliases;
-        }
-
-        Template(StreamInput in) throws IOException {
-            if (in.readBoolean()) {
-                this.settings = Settings.readSettingsFromStream(in);
-            } else {
-                this.settings = null;
-            }
-            if (in.readBoolean()) {
-                this.mappings = CompressedXContent.readCompressedString(in);
-            } else {
-                this.mappings = null;
-            }
-            if (in.readBoolean()) {
-                this.aliases = in.readMap(StreamInput::readString, AliasMetaData::new);
-            } else {
-                this.aliases = null;
-            }
-        }
-
-        public Settings settings() {
-            return settings;
-        }
-
-        public CompressedXContent mappings() {
-            return mappings;
-        }
-
-        public Map<String, AliasMetaData> aliases() {
-            return aliases;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            if (this.settings == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                Settings.writeSettingsToStream(this.settings, out);
-            }
-            if (this.mappings == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                this.mappings.writeTo(out);
-            }
-            if (this.aliases == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                out.writeMap(this.aliases, StreamOutput::writeString, (stream, aliasMetaData) -> aliasMetaData.writeTo(stream));
-            }
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(settings, mappings, aliases);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (obj == null) {
-                return false;
-            }
-            if (obj.getClass() != getClass()) {
-                return false;
-            }
-            Template other = (Template) obj;
-            return Objects.equals(settings, other.settings) &&
-                Objects.equals(mappings, other.mappings) &&
-                Objects.equals(aliases, other.aliases);
-        }
-
-        @Override
-        public String toString() {
-            return Strings.toString(this);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            if (this.settings != null) {
-                builder.startObject(SETTINGS.getPreferredName());
-                this.settings.toXContent(builder, params);
-                builder.endObject();
-            }
-            if (this.mappings != null) {
-                Map<String, Object> uncompressedMapping =
-                    XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2();
-                if (uncompressedMapping.size() > 0) {
-                    builder.field(MAPPINGS.getPreferredName());
-                    builder.map(uncompressedMapping);
-                }
-            }
-            if (this.aliases != null) {
-                builder.startObject(ALIASES.getPreferredName());
-                for (AliasMetaData alias : this.aliases.values()) {
-                    AliasMetaData.Builder.toXContent(alias, builder, params);
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            return builder;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index f47dd8f7dec31..ecce58e5a7154 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -23,7 +23,9 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -132,6 +134,104 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
         });
     }
 
+    /**
+     * Add the given component template to the cluster state. If {@code create} is true, an
+     * exception will be thrown if the component template already exists
+     */
+    public void putComponentTemplate(final String cause, final boolean create, final String name, final TimeValue masterTimeout,
+                                     final ComponentTemplate template, final ActionListener<AcknowledgedResponse> listener) {
+        clusterService.submitStateUpdateTask("create-component-template [" + name + "], cause [" + cause + "]",
+            new ClusterStateUpdateTask(Priority.URGENT) {
+
+                @Override
+                public TimeValue timeout() {
+                    return masterTimeout;
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return addComponentTemplate(currentState, create, name, template);
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    listener.onResponse(new AcknowledgedResponse(true));
+                }
+            });
+    }
+
+    // Package visible for testing
+    static ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
+                                             final String name, final ComponentTemplate template) {
+        if (create && currentState.metaData().componentTemplates().containsKey(name)) {
+            throw new IllegalArgumentException("component template [" + name + "] already exists");
+        }
+
+        // TODO: validation of component template
+        // validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
+
+        logger.info("adding component template [{}]", name);
+        return ClusterState.builder(currentState)
+            .metaData(MetaData.builder(currentState.metaData()).put(name, template))
+            .build();
+    }
+
+    /**
+     * Remove the given component template from the cluster state. The component template name
+     * supports simple regex wildcards for removing multiple component templates at a time.
+     */
+    public void removeComponentTemplate(final String name, final TimeValue masterTimeout,
+                                        final ActionListener<AcknowledgedResponse> listener) {
+        clusterService.submitStateUpdateTask("remove-component-template [" + name + "]",
+            new ClusterStateUpdateTask(Priority.URGENT) {
+
+                @Override
+                public TimeValue timeout() {
+                    return masterTimeout;
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    Set<String> templateNames = new HashSet<>();
+                    for (String templateName : currentState.metaData().componentTemplates().keySet()) {
+                        if (Regex.simpleMatch(name, templateName)) {
+                            templateNames.add(templateName);
+                        }
+                    }
+                    if (templateNames.isEmpty()) {
+                        // if its a match all pattern, and no templates are found (we have none), don't
+                        // fail with index missing...
+                        if (Regex.isMatchAllPattern(name)) {
+                            return currentState;
+                        }
+                        // TODO: perhaps introduce a ComponentTemplateMissingException?
+                        throw new IndexTemplateMissingException(name);
+                    }
+                    MetaData.Builder metaData = MetaData.builder(currentState.metaData());
+                    for (String templateName : templateNames) {
+                        logger.info("removing component template [{}]", templateName);
+                        metaData.removeComponentTemplate(templateName);
+                    }
+                    return ClusterState.builder(currentState).metaData(metaData).build();
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    listener.onResponse(new AcknowledgedResponse(true));
+                }
+            });
+    }
+
     public void putTemplate(final PutRequest request, final PutListener listener) {
         Settings.Builder updatedSettingsBuilder = Settings.builder();
         updatedSettingsBuilder.put(request.settings).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
new file mode 100644
index 0000000000000..b86eaf3598e10
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A template consists of optional settings, mappings, or alias configuration for an index, however,
+ * it is entirely independent from an index. It's a building block forming part of a regular index
+ * template and a {@link ComponentTemplate}.
+ */
+public class Template extends AbstractDiffable<Template> implements ToXContentObject {
+    private static final ParseField SETTINGS = new ParseField("settings");
+    private static final ParseField MAPPINGS = new ParseField("mappings");
+    private static final ParseField ALIASES = new ParseField("aliases");
+
+    @SuppressWarnings("unchecked")
+    static final ConstructingObjectParser<Template, Void> PARSER = new ConstructingObjectParser<>("template", false,
+        a -> new Template((Settings) a[0], (CompressedXContent) a[1], (Map<String, AliasMetaData>) a[2]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
+            new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            Map<String, AliasMetaData> aliasMap = new HashMap<>();
+            while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                AliasMetaData alias = AliasMetaData.Builder.fromXContent(p);
+                aliasMap.put(alias.alias(), alias);
+            }
+            return aliasMap;
+        }, ALIASES);
+    }
+
+    @Nullable
+    private final Settings settings;
+    @Nullable
+    private final CompressedXContent mappings;
+    @Nullable
+    private final Map<String, AliasMetaData> aliases;
+
+    public Template(@Nullable Settings settings, @Nullable CompressedXContent mappings, @Nullable Map<String, AliasMetaData> aliases) {
+        this.settings = settings;
+        this.mappings = mappings;
+        this.aliases = aliases;
+    }
+
+    Template(StreamInput in) throws IOException {
+        if (in.readBoolean()) {
+            this.settings = Settings.readSettingsFromStream(in);
+        } else {
+            this.settings = null;
+        }
+        if (in.readBoolean()) {
+            this.mappings = CompressedXContent.readCompressedString(in);
+        } else {
+            this.mappings = null;
+        }
+        if (in.readBoolean()) {
+            this.aliases = in.readMap(StreamInput::readString, AliasMetaData::new);
+        } else {
+            this.aliases = null;
+        }
+    }
+
+    public Settings settings() {
+        return settings;
+    }
+
+    public CompressedXContent mappings() {
+        return mappings;
+    }
+
+    public Map<String, AliasMetaData> aliases() {
+        return aliases;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (this.settings == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            Settings.writeSettingsToStream(this.settings, out);
+        }
+        if (this.mappings == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            this.mappings.writeTo(out);
+        }
+        if (this.aliases == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            out.writeMap(this.aliases, StreamOutput::writeString, (stream, aliasMetaData) -> aliasMetaData.writeTo(stream));
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(settings, mappings, aliases);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        Template other = (Template) obj;
+        return Objects.equals(settings, other.settings) &&
+            Objects.equals(mappings, other.mappings) &&
+            Objects.equals(aliases, other.aliases);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (this.settings != null) {
+            builder.startObject(SETTINGS.getPreferredName());
+            this.settings.toXContent(builder, params);
+            builder.endObject();
+        }
+        if (this.mappings != null) {
+            Map<String, Object> uncompressedMapping =
+                XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2();
+            if (uncompressedMapping.size() > 0) {
+                builder.field(MAPPINGS.getPreferredName());
+                builder.map(uncompressedMapping);
+            }
+        }
+        if (this.aliases != null) {
+            builder.startObject(ALIASES.getPreferredName());
+            for (AliasMetaData alias : this.aliases.values()) {
+                AliasMetaData.Builder.toXContent(alias, builder, params);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteComponentTemplateAction.java
new file mode 100644
index 0000000000000..c804a05025c2a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteComponentTemplateAction.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.template.delete.DeleteComponentTemplateAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+
+public class RestDeleteComponentTemplateAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(DELETE, "/_component_template/{name}"));
+    }
+
+    @Override
+    public String getName() {
+        return "delete_component_template_action";
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+
+        DeleteComponentTemplateAction.Request deleteReq = new DeleteComponentTemplateAction.Request(request.param("name"));
+        deleteReq.masterNodeTimeout(request.paramAsTime("master_timeout", deleteReq.masterNodeTimeout()));
+
+        return channel -> client.execute(DeleteComponentTemplateAction.INSTANCE, deleteReq, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java
new file mode 100644
index 0000000000000..227f5a7e68ea6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.HEAD;
+import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class RestGetComponentTemplateAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(GET, "/_component_template"),
+            new Route(GET, "/_component_template/{name}"),
+            new Route(HEAD, "/_component_template/{name}"));
+    }
+
+    @Override
+    public String getName() {
+        return "get_component_template_action";
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        final String[] names = Strings.splitStringByCommaToArray(request.param("name"));
+
+        final GetComponentTemplateAction.Request getRequest = new GetComponentTemplateAction.Request(names);
+
+        getRequest.local(request.paramAsBoolean("local", getRequest.local()));
+        getRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRequest.masterNodeTimeout()));
+
+        final boolean implicitAll = getRequest.names().length == 0;
+
+        return channel ->
+            client.execute(GetComponentTemplateAction.INSTANCE, getRequest, new RestToXContentListener<>(channel) {
+                    @Override
+                    protected RestStatus getStatus(final GetComponentTemplateAction.Response response) {
+                        final boolean templateExists = response.getComponentTemplates().isEmpty() == false;
+                        return (templateExists || implicitAll) ? OK : NOT_FOUND;
+                    }
+                });
+    }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Settings.FORMAT_PARAMS;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java
new file mode 100644
index 0000000000000..8c43f33bf7a07
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+public class RestPutComponentTemplateAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(POST, "/_component_template/{name}"),
+            new Route(PUT, "/_component_template/{name}"));
+    }
+
+    @Override
+    public String getName() {
+        return "put_component_template_action";
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+
+        PutComponentTemplateAction.Request putRequest = new PutComponentTemplateAction.Request(request.param("name"));
+        putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout()));
+        putRequest.create(request.paramAsBoolean("create", false));
+        putRequest.cause(request.param("cause", "api"));
+        putRequest.componentTemplate(ComponentTemplate.parse(request.contentParser()));
+
+        return channel -> client.execute(PutComponentTemplateAction.INSTANCE, putRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
new file mode 100644
index 0000000000000..0ee1f46bea404
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.get;
+
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComponentTemplateTests;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class GetComponentTemplateResponseTests extends AbstractWireSerializingTestCase<GetComponentTemplateAction.Response> {
+    @Override
+    protected Writeable.Reader<GetComponentTemplateAction.Response> instanceReader() {
+        return GetComponentTemplateAction.Response::new;
+    }
+
+    @Override
+    protected GetComponentTemplateAction.Response createTestInstance() {
+        if (randomBoolean()) {
+            return new GetComponentTemplateAction.Response(Collections.emptyMap());
+        }
+        Map<String, ComponentTemplate> templates = new HashMap<>();
+        for (int i = 0; i < randomIntBetween(1, 4); i++) {
+            templates.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance());
+        }
+        return new GetComponentTemplateAction.Response(templates);
+    }
+
+    @Override
+    protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) throws IOException {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
index f0c1b523ac9c1..470d573bb5212 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -76,7 +76,7 @@ public static ComponentTemplate randomInstance() {
         if (randomBoolean()) {
             aliases = randomAliases();
         }
-        ComponentTemplate.Template template = new ComponentTemplate.Template(settings, mappings, aliases);
+        Template template = new Template(settings, mappings, aliases);
 
         Map<String, Object> meta = null;
         if (randomBoolean()) {
@@ -130,21 +130,21 @@ public static ComponentTemplate mutateTemplate(ComponentTemplate orig) {
             case 0:
                 switch (randomIntBetween(0, 2)) {
                     case 0:
-                        ComponentTemplate.Template ot = orig.template();
+                        Template ot = orig.template();
                         return new ComponentTemplate(
-                            new ComponentTemplate.Template(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings),
+                            new Template(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings),
                                 ot.mappings(), ot.aliases()),
                             orig.version(), orig.metadata());
                     case 1:
-                        ComponentTemplate.Template ot2 = orig.template();
+                        Template ot2 = orig.template();
                         return new ComponentTemplate(
-                            new ComponentTemplate.Template(ot2.settings(),
+                            new Template(ot2.settings(),
                                 randomValueOtherThan(ot2.mappings(), ComponentTemplateTests::randomMappings), ot2.aliases()),
                             orig.version(), orig.metadata());
                     case 2:
-                        ComponentTemplate.Template ot3 = orig.template();
+                        Template ot3 = orig.template();
                         return new ComponentTemplate(
-                            new ComponentTemplate.Template(ot3.settings(), ot3.mappings(),
+                            new Template(ot3.settings(), ot3.mappings(),
                                 randomValueOtherThan(ot3.aliases(), ComponentTemplateTests::randomAliases)),
                             orig.version(), orig.metadata());
                     default:
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
similarity index 93%
rename from server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
rename to server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
index 788339d1feb69..94d37a52b89be 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
@@ -17,15 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.indices.template.put;
+package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.AliasValidator;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
-import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
@@ -202,6 +197,23 @@ public void testPutGlobalTemplateWithIndexHiddenSetting() throws Exception {
         assertThat(errors.get(0).getMessage(), containsString("global templates may not specify the setting index.hidden"));
     }
 
+    public void testAddComponentTemplate() {
+        ClusterState state = ClusterState.EMPTY_STATE;
+        ComponentTemplate template = ComponentTemplateTests.randomInstance();
+        state = MetaDataIndexTemplateService.addComponentTemplate(state, false, "foo", template);
+
+        assertNotNull(state.metaData().componentTemplates().get("foo"));
+        assertThat(state.metaData().componentTemplates().get("foo"), equalTo(template));
+
+        final ClusterState throwState = ClusterState.builder(state).build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> MetaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", template));
+        assertThat(e.getMessage(), containsString("component template [foo] already exists"));
+
+        state = MetaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", template);
+        assertNotNull(state.metaData().componentTemplates().get("bar"));
+    }
+
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
                 Settings.EMPTY,
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index b03c41484893f..7257a21b15900 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -118,7 +118,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
                 .put("component_template", new ComponentTemplate(
-                    new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
+                    new Template(Settings.builder().put("setting", "value").build(),
                         new CompressedXContent("{\"baz\":\"eggplant\"}"),
                         Collections.singletonMap("alias", AliasMetaData.builder("alias").build())),
                     5L, Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))))
@@ -298,7 +298,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
         assertThat(parsedMetaData.componentTemplates().get("component_template").metadata(),
             equalTo(Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))));
         assertThat(parsedMetaData.componentTemplates().get("component_template").template(),
-            equalTo(new ComponentTemplate.Template(Settings.builder().put("setting", "value").build(),
+            equalTo(new Template(Settings.builder().put("setting", "value").build(),
                 new CompressedXContent("{\"baz\":\"eggplant\"}"),
                 Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
     }

From f0eeffc1bcad809e5bdc5d482de8d1deb18d69bd Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Tue, 17 Mar 2020 09:57:47 -0700
Subject: [PATCH 079/285] Improve performance of docs snippet length checkstyle
 rule (#53636)

The multiline regex rule used to detect docs code snippets greater than
76 characters in length has considerable cost to. For the high level
rest client project alone, this was taking upwards of 3 minutes. This
commit updates the rule regex pattern to use non-greedy matching when
appropriate which results in a lot fewer backtracks and about a 70%
reduction in execution time on the high level rest client module.
---
 buildSrc/src/main/resources/checkstyle.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml
index 7a59ee5e0fa8b..bb4e0a124b69c 100644
--- a/buildSrc/src/main/resources/checkstyle.xml
+++ b/buildSrc/src/main/resources/checkstyle.xml
@@ -26,7 +26,7 @@
   -->
   <module name="RegexpMultiline">
     <property name="id" value="SnippetLength"/>
-    <property name="format" value="^( *)\/\/\s*tag(.+)\s*\n(.*\n)*\1.{77,}\n(.*\n)*\1\/\/\s*end\2\s*$"/>
+    <property name="format" value="^( *?)//\s*?tag(.+?)\s*?\n(.*?\n)*?\1.{77,}?\n(.*?\n)*?\1//\s*?end\2\s*$"/>
     <property name="fileExtensions" value="java"/>
     <property name="message" value="Code snippets longer than 76 characters get cut off when rendered in the docs"/>
   </module>

From d63cda1bcb7994d038dc60e854890c300d61d570 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Tue, 17 Mar 2020 10:26:35 -0700
Subject: [PATCH 080/285] Upgrade jackson to 2.10.3 and GeoIP to 2.13.1
 (#53642)

Re-applies the change from #53523 along with test fixes.

closes #53626
closes #53624
closes #53622
closes #53625

Co-authored-by: Nik Everett <nik9000@gmail.com>
Co-authored-by: Lee Hinman <dakrone@users.noreply.github.com>
Co-authored-by: Jake Landis <jake.landis@elastic.co>
---
 buildSrc/version.properties                   |  8 ++------
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 libs/x-content/build.gradle                   |  1 +
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../jackson-dataformat-cbor-2.10.3.jar.sha1   |  1 +
 .../jackson-dataformat-cbor-2.8.11.jar.sha1   |  1 -
 .../jackson-dataformat-smile-2.10.3.jar.sha1  |  1 +
 .../jackson-dataformat-smile-2.8.11.jar.sha1  |  1 -
 .../jackson-dataformat-yaml-2.10.3.jar.sha1   |  1 +
 .../jackson-dataformat-yaml-2.8.11.jar.sha1   |  1 -
 .../licenses/snakeyaml-1.17.jar.sha1          |  1 -
 .../licenses/snakeyaml-1.24.jar.sha1          |  1 +
 .../xcontent/MapXContentParserTests.java      | 17 +++++++++++++++--
 .../common/xcontent/XContentParserTests.java  | 19 +++++++++++--------
 .../ingest/common/JsonProcessorTests.java     |  9 +++++++--
 modules/ingest-geoip/build.gradle             | 10 ++++------
 .../licenses/geoip2-2.13.1.jar.sha1           |  1 +
 .../licenses/geoip2-2.9.0.jar.sha1            |  1 -
 .../jackson-annotations-2.10.3.jar.sha1       |  1 +
 .../jackson-annotations-2.8.11.jar.sha1       |  1 -
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../licenses/maxmind-db-1.2.2.jar.sha1        |  1 -
 .../licenses/maxmind-db-1.3.1.jar.sha1        |  1 +
 plugins/discovery-ec2/build.gradle            |  2 +-
 .../jackson-annotations-2.10.3.jar.sha1       |  1 +
 .../jackson-annotations-2.8.11.jar.sha1       |  1 -
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 plugins/repository-s3/build.gradle            |  2 +-
 .../jackson-annotations-2.10.3.jar.sha1       |  1 +
 .../jackson-annotations-2.8.11.jar.sha1       |  1 -
 .../licenses/jackson-databind-2.10.3.jar.sha1 |  1 +
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../xcontent/XContentParserUtilsTests.java    |  6 +++---
 .../index/IndexingSlowLogTests.java           |  8 ++++----
 .../org/elasticsearch/test/RandomObjects.java | 12 ++++--------
 x-pack/plugin/sql/jdbc/build.gradle           |  5 +++++
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../jackson-dataformat-cbor-2.10.3.jar.sha1   |  1 +
 .../jackson-dataformat-cbor-2.8.11.jar.sha1   |  1 -
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 .../xpack/sql/client/RemoteFailureTests.java  |  5 +++--
 .../licenses/jackson-core-2.10.3.jar.sha1     |  1 +
 .../licenses/jackson-core-2.8.11.jar.sha1     |  1 -
 51 files changed, 80 insertions(+), 62 deletions(-)
 create mode 100644 client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
 delete mode 100644 libs/x-content/licenses/snakeyaml-1.17.jar.sha1
 create mode 100644 libs/x-content/licenses/snakeyaml-1.24.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
 create mode 100644 modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
 create mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index dea67de1657d1..9ef5da5062255 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -7,12 +7,8 @@ bundled_jdk = 13.0.2+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-# note that ingest-geoip has a hard-coded version; if you modify this version,
-# you should also inspect that version to see if it can be advanced along with
-# the com.maxmind.geoip2:geoip2 dependency
-jackson           = 2.8.11
-jacksondatabind   = 2.8.11.6
-snakeyaml         = 1.17
+jackson           = 2.10.3
+snakeyaml         = 1.24
 icu4j             = 62.1
 supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
diff --git a/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1 b/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/client/sniffer/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
index 14adf7d102d78..be056980ef50d 100644
--- a/libs/x-content/build.gradle
+++ b/libs/x-content/build.gradle
@@ -49,6 +49,7 @@ forbiddenApisMain {
 thirdPartyAudit.ignoreMissingClasses(
   // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
   'com.fasterxml.jackson.databind.ObjectMapper',
+  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
 )
 
 dependencyLicenses {
diff --git a/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/libs/x-content/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/libs/x-content/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..6c96c20f48232
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
@@ -0,0 +1 @@
+1ba01fef9c3b7ed388d91e71dc733b315c7374cd
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
deleted file mode 100644
index 378ba524422bc..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..24ac4b32095e4
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-smile-2.10.3.jar.sha1
@@ -0,0 +1 @@
+ff397547ff168e77279a1cd549e2ca4923c991aa
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
deleted file mode 100644
index 510afb3df53e6..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-smile-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..8f1ecab9ecb38
--- /dev/null
+++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.10.3.jar.sha1
@@ -0,0 +1 @@
+4dc1a172812d9da27c1afd6a08f4f12aad7b14dd
\ No newline at end of file
diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
deleted file mode 100644
index 78a68d715ec3d..0000000000000
--- a/libs/x-content/licenses/jackson-dataformat-yaml-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a
\ No newline at end of file
diff --git a/libs/x-content/licenses/snakeyaml-1.17.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
deleted file mode 100644
index 9ac6e87f2244a..0000000000000
--- a/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7a27ea250c5130b2922b86dea63cbb1cc10a660c
\ No newline at end of file
diff --git a/libs/x-content/licenses/snakeyaml-1.24.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.24.jar.sha1
new file mode 100644
index 0000000000000..a6e7bd5a1e7c1
--- /dev/null
+++ b/libs/x-content/licenses/snakeyaml-1.24.jar.sha1
@@ -0,0 +1 @@
+13a9c0d6776483c3876e3ff9384f9bb55b17001b
\ No newline at end of file
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java
index 0d2113152ebe8..b3fb9eee4662d 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.EnumSet;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentParserTests.generateRandomObject;
@@ -73,7 +74,13 @@ public void testRandomObject() throws IOException {
     }
 
     public void compareTokens(CheckedConsumer<XContentBuilder, IOException> consumer) throws IOException {
-        final XContentType xContentType = randomFrom(XContentType.values());
+        for (XContentType xContentType : EnumSet.allOf(XContentType.class)) {
+            logger.info("--> testing with xcontent type: {}", xContentType);
+            compareTokens(consumer, xContentType);
+        }
+    }
+
+    public void compareTokens(CheckedConsumer<XContentBuilder, IOException> consumer, XContentType xContentType) throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
             consumer.accept(builder);
             final Map<String, Object> map;
@@ -94,7 +101,13 @@ public void compareTokens(CheckedConsumer<XContentBuilder, IOException> consumer
                         assertEquals(token, mapToken);
                         assertEquals(parser.currentName(), mapParser.currentName());
                         if (token != null && (token.isValue() || token == XContentParser.Token.VALUE_NULL)) {
-                            assertEquals(parser.textOrNull(), mapParser.textOrNull());
+                            if (xContentType != XContentType.YAML || token != XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+                                // YAML struggles with converting byte arrays into text, because it
+                                // does weird base64 decoding to the values. We don't do this
+                                // weirdness in the MapXContentParser, so don't try to stringify it.
+                                // The .binaryValue() comparison below still works correctly though.
+                                assertEquals(parser.textOrNull(), mapParser.textOrNull());
+                            }
                             switch (token) {
                                 case VALUE_STRING:
                                     assertEquals(parser.text(), mapParser.text());
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
index 0cfa01876c590..66c0041723e8c 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -78,14 +78,17 @@ public void testFloat() throws IOException {
 
             assertEquals(value, number.floatValue(), 0.0f);
 
-            if (xContentType == XContentType.CBOR) {
-                // CBOR parses back a float
-                assertTrue(number instanceof Float);
-            } else {
-                // JSON, YAML and SMILE parses back the float value as a double
-                // This will change for SMILE in Jackson 2.9 where all binary based
-                // formats will return a float
-                assertTrue(number instanceof Double);
+            switch (xContentType) {
+                case CBOR:
+                case SMILE:
+                    assertThat(number, instanceOf(Float.class));
+                    break;
+                case JSON:
+                case YAML:
+                    assertThat(number, instanceOf(Double.class));
+                    break;
+                default:
+                    throw new AssertionError("unexpected x-content type [" + xContentType + "]");
             }
         }
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index 099e8e1866b8e..c9e81ccf4c811 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -65,7 +65,7 @@ public void testInvalidValue() {
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
         assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
-            "was expecting ('true', 'false' or 'null')"));
+            "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"));
     }
 
     public void testByteArray() {
@@ -75,7 +75,12 @@ public void testByteArray() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
-        assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
+        assertThat(
+            exception.getCause().getMessage(),
+            containsString(
+                "Unrecognized token 'B': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
+            )
+        );
     }
 
     public void testNull() throws Exception {
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index 6789f96d883b8..dfa165e566170 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -25,13 +25,11 @@ esplugin {
 }
 
 dependencies {
-  // Upgrade to 2.10.0 or higher when jackson-core gets upgraded to 2.9.x. Blocked by #27032
-  compile('com.maxmind.geoip2:geoip2:2.9.0')
+  compile('com.maxmind.geoip2:geoip2:2.13.1')
   // geoip2 dependencies:
-  // do not hardcode this to the version in version.properties, it needs to be upgraded separately with geoip2
-  compile("com.fasterxml.jackson.core:jackson-annotations:2.8.11")
-  compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.6")
-  compile('com.maxmind.db:maxmind-db:1.2.2')
+  compile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
+  compile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
+  compile('com.maxmind.db:maxmind-db:1.3.1')
 
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
 }
diff --git a/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
new file mode 100644
index 0000000000000..253d9f12e7a3a
--- /dev/null
+++ b/modules/ingest-geoip/licenses/geoip2-2.13.1.jar.sha1
@@ -0,0 +1 @@
+f27d1a49d5a29dd4a7ac5006ce2eb16b8b9bb888
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
deleted file mode 100644
index 8cb79bbb9207a..0000000000000
--- a/modules/ingest-geoip/licenses/geoip2-2.9.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c12b463a2c10824225c0b27952c49b464cb0e4c6
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..9c725f2d90e69
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-annotations-2.10.3.jar.sha1
@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
deleted file mode 100644
index 30e7d1a7b1a74..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-annotations-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..688ae92d10792
--- /dev/null
+++ b/modules/ingest-geoip/licenses/jackson-databind-2.10.3.jar.sha1
@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
deleted file mode 100644
index 3dd3ad36f4cc3..0000000000000
--- a/modules/ingest-geoip/licenses/maxmind-db-1.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-78c22a03de1e222b0751855aff7bb6e6db5569e5
\ No newline at end of file
diff --git a/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
new file mode 100644
index 0000000000000..aebff2c3a849c
--- /dev/null
+++ b/modules/ingest-geoip/licenses/maxmind-db-1.3.1.jar.sha1
@@ -0,0 +1 @@
+211bca628225bc0f719051b16deb03a747d7a14f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index a9ae35393de16..bc507dc5d9742 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -36,7 +36,7 @@ dependencies {
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..9c725f2d90e69
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.10.3.jar.sha1
@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
deleted file mode 100644
index 30e7d1a7b1a74..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-annotations-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..688ae92d10792
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/jackson-databind-2.10.3.jar.sha1
@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index a92f07b71ee01..04624f8512976 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -43,7 +43,7 @@ dependencies {
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jacksondatabind}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   compile "joda-time:joda-time:${versions.joda}"
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..9c725f2d90e69
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-annotations-2.10.3.jar.sha1
@@ -0,0 +1 @@
+0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
deleted file mode 100644
index 30e7d1a7b1a74..0000000000000
--- a/plugins/repository-s3/licenses/jackson-annotations-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-391de20b4e29cb3fb07d2454ace64be2c82ac91f
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..688ae92d10792
--- /dev/null
+++ b/plugins/repository-s3/licenses/jackson-databind-2.10.3.jar.sha1
@@ -0,0 +1 @@
+aae92628b5447fa25af79871ca98668da6edd439
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
index db3dabdcafe49..0541f864d8686 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
@@ -91,11 +91,11 @@ public void testStoredFieldsValueBoolean() throws IOException {
     public void testStoredFieldsValueBinary() throws IOException {
         final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8");
         assertParseFieldsSimpleValue(value, (xcontentType, result) -> {
-            if (xcontentType == XContentType.JSON || xcontentType == XContentType.YAML) {
-                //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+            if (xcontentType == XContentType.JSON) {
+                // binary values will be parsed back and returned as base64 strings when reading from json
                 assertArrayEquals(value, Base64.getDecoder().decode((String) result));
             } else {
-                //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+                // cbor, smile, and yaml support binary
                 assertArrayEquals(value, ((BytesArray) result).array());
             }
         });
diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index dbff49460015b..78d25e996ab2a 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -103,13 +103,13 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         final UncheckedIOException e = expectThrows(UncheckedIOException.class,
             () -> IndexingSlowLogMessage.of(index, doc, 10, true, 3));
         assertThat(e, hasToString(containsString("_failed_to_convert_[Unrecognized token 'invalid':"
-            + " was expecting ('true', 'false' or 'null')\\n"
-            + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
+            + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\\n"
+            + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
         assertNotNull(e.getCause());
         assertThat(e.getCause(), instanceOf(JsonParseException.class));
         assertThat(e.getCause(), hasToString(containsString("Unrecognized token 'invalid':"
-                + " was expecting ('true', 'false' or 'null')\n"
-                + " at [Source: org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper")));
+                + " was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n"
+                + " at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper)")));
     }
 
     public void testReformatSetting() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
index 69bd367568ad5..7718e96398b44 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
@@ -136,20 +136,16 @@ private static List<Object> randomStoredFieldValues(Random random, int numValues
      */
     public static Object getExpectedParsedValue(XContentType xContentType, Object value) {
         if (value instanceof BytesArray) {
-            if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
-                //JSON and YAML write the base64 format
+            if (xContentType == XContentType.JSON) {
+                //JSON writes base64 format
                 return Base64.getEncoder().encodeToString(((BytesArray)value).toBytesRef().bytes);
             }
         }
         if (value instanceof Float) {
-            if (xContentType == XContentType.CBOR) {
-                //with CBOR we get back a float
+            if (xContentType == XContentType.CBOR || xContentType == XContentType.SMILE) {
+                // with binary content types we pass back the object as is
                 return value;
             }
-            if (xContentType == XContentType.SMILE) {
-                //with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float)
-                return ((Float)value).doubleValue();
-            }
             //with JSON AND YAML we get back a double, but with float precision.
             return Double.parseDouble(value.toString());
         }
diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle
index 7cdec567ef8f4..dc42f90476571 100644
--- a/x-pack/plugin/sql/jdbc/build.gradle
+++ b/x-pack/plugin/sql/jdbc/build.gradle
@@ -44,3 +44,8 @@ dependencyLicenses {
 shadowJar {
   relocate 'com.fasterxml', 'org.elasticsearch.fasterxml'
 }
+
+thirdPartyAudit.ignoreMissingClasses(
+  'com.fasterxml.jackson.databind.ObjectMapper',
+  'com.fasterxml.jackson.databind.cfg.MapperBuilder'
+)
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..6c96c20f48232
--- /dev/null
+++ b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.10.3.jar.sha1
@@ -0,0 +1 @@
+1ba01fef9c3b7ed388d91e71dc733b315c7374cd
\ No newline at end of file
diff --git a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
deleted file mode 100644
index 378ba524422bc..0000000000000
--- a/x-pack/plugin/sql/jdbc/licenses/jackson-dataformat-cbor-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b9826e16c3366764bfb7ad7362554f0471046c3
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/sql-client/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
index 2029493bcbec4..1409e80837768 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
@@ -92,7 +92,8 @@ public void testNoType() {
     public void testInvalidJson() {
         IOException e = expectThrows(IOException.class, () -> parse("invalid_json.txt"));
         assertEquals(
-            "Can't parse error from Elasticsearch [Unrecognized token 'I': was expecting 'null', 'true', 'false' or NaN] "
+            "Can't parse error from Elasticsearch [Unrecognized token 'I': "
+                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] "
                 + "at [line 1 col 1]. Response:\n"
                 + "I'm not json at all",
             e.getMessage());
@@ -118,7 +119,7 @@ public void testTotalGarbage() {
             }).streamInput()));
         assertThat(e.getMessage(),
             startsWith("Can't parse error from Elasticsearch [Unrecognized token 'ÿ': "
-                + "was expecting ('true', 'false' or 'null')] at [line 1 col 1]. Response:\n"));
+                + "was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')] at [line 1 col 4]. Response:\n"));
     }
 
     public void testTooBig() {
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
new file mode 100644
index 0000000000000..f23937b0d82a4
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.10.3.jar.sha1
@@ -0,0 +1 @@
+f7ee7b55c7d292ac72fbaa7648c089f069c938d2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
deleted file mode 100644
index e7ad1e74ed6b8..0000000000000
--- a/x-pack/plugin/sql/sql-proto/licenses/jackson-core-2.8.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-876ead1db19f0c9e79c9789273a3ef8c6fd6c29b
\ No newline at end of file

From 7a2d2ce58101e6c92016a88a7a3ec45d4b343669 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 17 Mar 2020 14:31:32 -0400
Subject: [PATCH 081/285] Drop Eclipse instructions (#53667)

Hacking Elasticsearch with Eclipse is fairly broken. Some things work,
but we can't in good concience point contributors to Eclipse. This drops
the instrucions for Eclipse from CONTRIBUTING.md. We *do* want to fix
Eclipse. But we don't see that happening in the next few months. Once we
*have* fixed Eclipse we'll rewrite the instructions.

Relates #53664
---
 CONTRIBUTING.md | 27 ++++++---------------------
 1 file changed, 6 insertions(+), 21 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index eeea7b412cbf8..8598b6001a667 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -111,9 +111,9 @@ using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
 script on Windows in the root of the repository. The examples below show the
 usage on Unix.
 
-We support development in the Eclipse and IntelliJ IDEs.
-For Eclipse, the minimum version that we support is [4.13][eclipse].
-For IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
+We support development in IntelliJ versions [IntelliJ 2017.2][intellij] and
+onwards. We would like to support Eclipse, but few of us use it and has fallen
+into [disrepair][eclipse].
 
 [Docker](https://docs.docker.com/install/) is required for building some Elasticsearch artifacts and executing certain test suites. You can run Elasticsearch without building all the artifacts with:
 
@@ -125,11 +125,6 @@ You can access Elasticsearch with:
 
 ### Configuring IDEs And Running Tests
 
-Eclipse users can automatically configure their IDE: `./gradlew eclipse`
-then `File: Import: Gradle : Existing Gradle Project`.
-Additionally you will want to ensure that Eclipse is using 2048m of heap by modifying
-`eclipse.ini` accordingly to avoid GC overhead and OOM errors.
-
 IntelliJ users can automatically configure their IDE: `./gradlew idea`
 then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
@@ -154,16 +149,13 @@ The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This
 can be accomplished by passing the flag `-ea` to the JVM on startup.
 
-For IntelliJ, go to
+You can enable these in IntelliJ by going to
 `Run->Edit Configurations...->Defaults->JUnit->VM options` and input
 `-ea`.
 
-For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
-`VM Arguments`.
-
 Some tests related to locale testing also require the flag
 `-Djava.locale.providers` to be set. Set the VM options/VM arguments for
-IntelliJ or Eclipse like describe above to use
+IntelliJ like describe above to use
 `-Djava.locale.providers=SPI,COMPAT`.
 
 ### REST Endpoint Conventions
@@ -213,9 +205,6 @@ Please follow these formatting guidelines:
   of the code, while keeping lines to maximum length of 76 characters.
 * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause
   the build to fail. This can be done automatically by your IDE:
-   * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are
-     two boxes labeled "`Number of (static )? imports needed for .*`". Set
-     their values to 99999 or some other absurdly high value.
    * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`.
      There are two configuration options: `Class count to use import with
      '*'` and `Names count to use static import with '*'`. Set their values
@@ -234,9 +223,6 @@ Please follow these formatting guidelines:
 
 #### Editor / IDE Support
 
-Eclipse IDEs can import the file [.eclipseformat.xml]
-directly.
-
 IntelliJ IDEs can
 [import](https://blog.jetbrains.com/idea/2014/01/intellij-idea-13-importing-code-formatter-settings-from-eclipse/)
 the same settings file, and / or use the [Eclipse Code
@@ -608,6 +594,5 @@ Finally, we require that you run `./gradlew check` before submitting a
 non-documentation contribution. This is mentioned above, but it is worth
 repeating in this section because it has come up in this context.
 
-[eclipse]: https://download.eclipse.org/eclipse/downloads/drops4/R-4.13-201909161045/
 [intellij]: https://blog.jetbrains.com/idea/2017/07/intellij-idea-2017-2-is-here-smart-sleek-and-snappy/
-[shadow-plugin]: https://github.com/johnrengelman/shadow
+[eclipse]: https://github.com/elastic/elasticsearch/issues/53664

From c2832f11778a72f1a78d8a0942578af266ecfa39 Mon Sep 17 00:00:00 2001
From: Karen Metts <35154725+karenzone@users.noreply.github.com>
Date: Tue, 17 Mar 2020 14:38:03 -0400
Subject: [PATCH 082/285] Remove link to old settings (#53638)

---
 docs/reference/settings/monitoring-settings.asciidoc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc
index 565780cc5b7a7..7ccf510b253de 100644
--- a/docs/reference/settings/monitoring-settings.asciidoc
+++ b/docs/reference/settings/monitoring-settings.asciidoc
@@ -17,10 +17,8 @@ file.
 
 To adjust how monitoring data is displayed in the monitoring UI, configure
 {kibana-ref}/monitoring-settings-kb.html[`xpack.monitoring` settings] in
-`kibana.yml`. To control how monitoring data is collected from
-Logstash, configure
-{logstash-ref}/monitoring-internal-collection.html#monitoring-settings[`xpack.monitoring` settings]
-in `logstash.yml`.
+`kibana.yml`. To control how monitoring data is collected from Logstash,
+configure monitoring settings in `logstash.yml`.
 
 For more information, see <<monitor-elasticsearch-cluster>>.
 

From 8c0f90ed553a20f1e425abad182ada76fabc7a9f Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Tue, 17 Mar 2020 20:57:54 +0100
Subject: [PATCH 083/285] [Transform] data nanos/date histogram IT (#53654)

add an integration test for date nanos in combination with date_histogram
---
 .../integration/TransformPivotRestIT.java     | 23 ++++++++-------
 .../integration/TransformRestTestCase.java    | 29 ++++++++++++++-----
 .../TransformTaskFailedStateIT.java           |  4 +--
 3 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
index 3cc0fb79a12a8..8769a6983fd89 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
@@ -56,6 +56,7 @@ public void createIndexes() throws IOException {
         }
 
         createReviewsIndex();
+        createReviewsIndexNano();
         indicesCreated = true;
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
 
@@ -496,9 +497,17 @@ public void testBiggerPivot() throws Exception {
     }
 
     public void testDateHistogramPivot() throws Exception {
-        String transformId = "simple_date_histogram_pivot";
-        String transformIndex = "pivot_reviews_via_date_histogram";
-        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+        assertDateHistogramPivot(REVIEWS_INDEX_NAME);
+    }
+
+    public void testDateHistogramPivotNanos() throws Exception {
+        assertDateHistogramPivot(REVIEWS_DATE_NANO_INDEX_NAME);
+    }
+
+    private void assertDateHistogramPivot(String indexName) throws Exception {
+        String transformId = "simple_date_histogram_pivot_" + indexName;
+        String transformIndex = "pivot_reviews_via_date_histogram_" + indexName;
+        setupDataAccessRole(DATA_ACCESS_ROLE, indexName, transformIndex);
 
         final Request createTransformRequest = createRequestWithAuth(
             "PUT",
@@ -506,13 +515,7 @@ public void testDateHistogramPivot() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = "{"
-            + " \"source\": {\"index\":\""
-            + REVIEWS_INDEX_NAME
-            + "\"},"
-            + " \"dest\": {\"index\":\""
-            + transformIndex
-            + "\"},";
+        String config = "{" + " \"source\": {\"index\":\"" + indexName + "\"}," + " \"dest\": {\"index\":\"" + transformIndex + "\"},";
 
         config += " \"pivot\": {"
             + "   \"group_by\": {"
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
index 1ff4d4e1dbe1b..6ca7b0f18f5c7 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
@@ -50,6 +50,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
     private static final String BASIC_AUTH_VALUE_SUPER_USER = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);
 
     protected static final String REVIEWS_INDEX_NAME = "reviews";
+    protected static final String REVIEWS_DATE_NANO_INDEX_NAME = "reviews_nano";
 
     private static boolean useDeprecatedEndpoints;
 
@@ -75,18 +76,20 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
         return super.buildClient(settings, hosts);
     }
 
-    protected void createReviewsIndex(String indexName, int numDocs) throws IOException {
+    protected void createReviewsIndex(String indexName, int numDocs, String dateType) throws IOException {
         int[] distributionTable = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 2, 1, 1, 1 };
 
         // create mapping
         try (XContentBuilder builder = jsonBuilder()) {
             builder.startObject();
             {
-                builder.startObject("mappings")
-                    .startObject("properties")
-                    .startObject("timestamp")
-                    .field("type", "date")
-                    .endObject()
+                builder.startObject("mappings").startObject("properties").startObject("timestamp").field("type", dateType);
+
+                if (dateType.equals("date_nanos")) {
+                    builder.field("format", "strict_date_optional_time_nanos");
+                }
+
+                builder.endObject()
                     .startObject("user_id")
                     .field("type", "keyword")
                     .endObject()
@@ -128,7 +131,13 @@ protected void createReviewsIndex(String indexName, int numDocs) throws IOExcept
             int sec = 10 + (i % 49);
             String location = (user + 10) + "," + (user + 15);
 
-            String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec + "Z";
+            String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec;
+            if (dateType.equals("date_nanos")) {
+                String randomNanos = "," + randomIntBetween(100000000, 999999999);
+                date_string += randomNanos;
+            }
+            date_string += "Z";
+
             bulk.append("{\"user_id\":\"")
                 .append("user_")
                 .append(user)
@@ -170,7 +179,7 @@ protected void createReviewsIndex() throws IOException {
     }
 
     protected void createReviewsIndex(String indexName) throws IOException {
-        createReviewsIndex(indexName, 1000);
+        createReviewsIndex(indexName, 1000, "date");
     }
 
     protected void createPivotReviewsTransform(String transformId, String transformIndex, String query) throws IOException {
@@ -182,6 +191,10 @@ protected void createPivotReviewsTransform(String transformId, String transformI
         createPivotReviewsTransform(transformId, transformIndex, query, pipeline, null);
     }
 
+    protected void createReviewsIndexNano() throws IOException {
+        createReviewsIndex(REVIEWS_DATE_NANO_INDEX_NAME, 1000, "date_nanos");
+    }
+
     protected void createContinuousPivotReviewsTransform(String transformId, String transformIndex, String authHeader) throws IOException {
 
         final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformTaskFailedStateIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformTaskFailedStateIT.java
index 9d8ccc4000712..3658dd082bd97 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformTaskFailedStateIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformTaskFailedStateIT.java
@@ -64,7 +64,7 @@ public void cleanUpPotentiallyFailedTransform() throws Exception {
 
     public void testForceStopFailedTransform() throws Exception {
         String transformId = "test-force-stop-failed-transform";
-        createReviewsIndex(REVIEWS_INDEX_NAME, 10);
+        createReviewsIndex(REVIEWS_INDEX_NAME, 10, "date");
         String transformIndex = "failure_pivot_reviews";
         createDestinationIndexWithBadMapping(transformIndex);
         createContinuousPivotReviewsTransform(transformId, transformIndex, null);
@@ -102,7 +102,7 @@ public void testForceStopFailedTransform() throws Exception {
 
     public void testStartFailedTransform() throws Exception {
         String transformId = "test-force-start-failed-transform";
-        createReviewsIndex(REVIEWS_INDEX_NAME, 10);
+        createReviewsIndex(REVIEWS_INDEX_NAME, 10, "date");
         String transformIndex = "failure_pivot_reviews";
         createDestinationIndexWithBadMapping(transformIndex);
         createContinuousPivotReviewsTransform(transformId, transformIndex, null);

From fa6d197e7bea7b7bd39731883d81f41d0cafd13e Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Tue, 17 Mar 2020 21:05:50 +0100
Subject: [PATCH 084/285] =?UTF-8?q?[Transform]=20enhance=20the=20output=20?=
 =?UTF-8?q?of=20preview=20to=20return=20full=20desti=E2=80=A6=20(#53572)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

changes the output format of preview regarding deduced mappings and enhances
it to return all the details about auto-index creation. This allows the user
to customize the index creation. Using HLRC you can create a index request
from the output of the response.
---
 .../transform/PreviewTransformResponse.java   | 160 +++++++++++++++--
 .../PreviewTransformResponseTests.java        | 132 +++++++++++---
 .../hlrc/PreviewTransformResponseTests.java   | 118 +++++++++++++
 .../action/PreviewTransformAction.java        | 110 ++++++------
 .../TransformDestIndexSettings.java           | 145 +++++++++++++++
 .../PreviewTransformsActionResponseTests.java |  37 ++--
 ...viewTransformsActionResponseWireTests.java |  41 +++--
 .../TransformDestIndexSettingsTests.java      |  72 ++++++++
 .../test/transform/preview_transforms.yml     |  16 +-
 .../TransportPreviewTransformAction.java      |  49 +++--
 .../action/TransportStartTransformAction.java |  11 +-
 .../TransportUpdateTransformAction.java       |  17 +-
 .../transform/persistence/TransformIndex.java | 167 +++++++++++-------
 .../persistence/TransformIndexTests.java      |  22 +--
 14 files changed, 857 insertions(+), 240 deletions(-)
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/transform/hlrc/PreviewTransformResponseTests.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettings.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/PreviewTransformResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/PreviewTransformResponse.java
index 215d529f94993..12a37d1f9d791 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/PreviewTransformResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/PreviewTransformResponse.java
@@ -19,40 +19,167 @@
 
 package org.elasticsearch.client.transform;
 
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PreviewTransformResponse {
 
-    private static final String PREVIEW = "preview";
-    private static final String MAPPINGS = "mappings";
+    public static class GeneratedDestIndexSettings {
+        static final ParseField MAPPINGS = new ParseField("mappings");
+        private static final ParseField SETTINGS = new ParseField("settings");
+        private static final ParseField ALIASES = new ParseField("aliases");
 
-    @SuppressWarnings("unchecked")
-    public static PreviewTransformResponse fromXContent(final XContentParser parser) throws IOException {
-        Map<String, Object> previewMap = parser.mapOrdered();
-        Object previewDocs = previewMap.get(PREVIEW);
-        Object mappings = previewMap.get(MAPPINGS);
-        return new PreviewTransformResponse((List<Map<String, Object>>) previewDocs, (Map<String, Object>) mappings);
+        private final Map<String, Object> mappings;
+        private final Settings settings;
+        private final Set<Alias> aliases;
+
+        private static final ConstructingObjectParser<GeneratedDestIndexSettings, Void> PARSER = new ConstructingObjectParser<>(
+            "transform_preview_generated_dest_index",
+            true,
+            args -> {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> mappings = (Map<String, Object>) args[0];
+                Settings settings = (Settings) args[1];
+                @SuppressWarnings("unchecked")
+                Set<Alias> aliases = (Set<Alias>) args[2];
+
+                return new GeneratedDestIndexSettings(mappings, settings, aliases);
+            }
+        );
+
+        static {
+            PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), MAPPINGS);
+            PARSER.declareObject(optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
+            PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
+                Set<Alias> aliases = new HashSet<>();
+                while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    aliases.add(Alias.fromXContent(p));
+                }
+                return aliases;
+            }, ALIASES);
+        }
+
+        public GeneratedDestIndexSettings(Map<String, Object> mappings, Settings settings, Set<Alias> aliases) {
+            this.mappings = mappings == null ? Collections.emptyMap() : Collections.unmodifiableMap(mappings);
+            this.settings = settings == null ? Settings.EMPTY : settings;
+            this.aliases = aliases == null ? Collections.emptySet() : Collections.unmodifiableSet(aliases);
+        }
+
+        public Map<String, Object> getMappings() {
+            return mappings;
+        }
+
+        public Settings getSettings() {
+            return settings;
+        }
+
+        public Set<Alias> getAliases() {
+            return aliases;
+        }
+
+        public static GeneratedDestIndexSettings fromXContent(final XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == this) {
+                return true;
+            }
+
+            if (obj == null || obj.getClass() != getClass()) {
+                return false;
+            }
+
+            GeneratedDestIndexSettings other = (GeneratedDestIndexSettings) obj;
+            return Objects.equals(other.mappings, mappings)
+                && Objects.equals(other.settings, settings)
+                && Objects.equals(other.aliases, aliases);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(mappings, settings, aliases);
+        }
     }
 
-    private List<Map<String, Object>> docs;
-    private Map<String, Object> mappings;
+    public static final ParseField PREVIEW = new ParseField("preview");
+    public static final ParseField GENERATED_DEST_INDEX_SETTINGS = new ParseField("generated_dest_index");
+
+    private final List<Map<String, Object>> docs;
+    private final GeneratedDestIndexSettings generatedDestIndexSettings;
+
+    private static final ConstructingObjectParser<PreviewTransformResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "data_frame_transform_preview",
+        true,
+        args -> {
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> docs = (List<Map<String, Object>>) args[0];
+            GeneratedDestIndexSettings generatedDestIndex = (GeneratedDestIndexSettings) args[1];
+
+            // ensure generatedDestIndex is not null
+            if (generatedDestIndex == null) {
+                // BWC parsing the output from nodes < 7.7
+                @SuppressWarnings("unchecked")
+                Map<String, Object> mappings = (Map<String, Object>) args[2];
+                generatedDestIndex = new GeneratedDestIndexSettings(mappings, null, null);
+            }
 
-    public PreviewTransformResponse(List<Map<String, Object>> docs, Map<String, Object> mappings) {
+            return new PreviewTransformResponse(docs, generatedDestIndex);
+        }
+    );
+    static {
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> p.mapOrdered(), PREVIEW);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> GeneratedDestIndexSettings.fromXContent(p), GENERATED_DEST_INDEX_SETTINGS);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), GeneratedDestIndexSettings.MAPPINGS);
+    }
+
+    public PreviewTransformResponse(List<Map<String, Object>> docs, GeneratedDestIndexSettings generatedDestIndexSettings) {
         this.docs = docs;
-        this.mappings = mappings;
+        this.generatedDestIndexSettings = generatedDestIndexSettings;
     }
 
     public List<Map<String, Object>> getDocs() {
         return docs;
     }
 
+    public GeneratedDestIndexSettings getGeneratedDestIndexSettings() {
+        return generatedDestIndexSettings;
+    }
+
     public Map<String, Object> getMappings() {
-        return mappings;
+        return generatedDestIndexSettings.getMappings();
+    }
+
+    public Settings getSettings() {
+        return generatedDestIndexSettings.getSettings();
+    }
+
+    public Set<Alias> getAliases() {
+        return generatedDestIndexSettings.getAliases();
+    }
+
+    public CreateIndexRequest getCreateIndexRequest(String index) {
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
+        createIndexRequest.aliases(generatedDestIndexSettings.getAliases());
+        createIndexRequest.settings(generatedDestIndexSettings.getSettings());
+        createIndexRequest.mapping(generatedDestIndexSettings.getMappings());
+
+        return createIndexRequest;
     }
 
     @Override
@@ -66,12 +193,15 @@ public boolean equals(Object obj) {
         }
 
         PreviewTransformResponse other = (PreviewTransformResponse) obj;
-        return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
+        return Objects.equals(other.docs, docs) && Objects.equals(other.generatedDestIndexSettings, generatedDestIndexSettings);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(docs, mappings);
+        return Objects.hash(docs, generatedDestIndexSettings);
     }
 
+    public static PreviewTransformResponse fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/PreviewTransformResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/PreviewTransformResponseTests.java
index 8e1dbefa127a8..fcbc746de2263 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/PreviewTransformResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/PreviewTransformResponseTests.java
@@ -19,47 +19,75 @@
 
 package org.elasticsearch.client.transform;
 
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+import static org.hamcrest.Matchers.equalTo;
 
 public class PreviewTransformResponseTests extends ESTestCase {
 
     public void testFromXContent() throws IOException {
-        xContentTester(this::createParser,
-                this::createTestInstance,
-                this::toXContent,
-                PreviewTransformResponse::fromXContent)
-                .supportsUnknownFields(true)
-                .randomFieldsExcludeFilter(path -> path.isEmpty() == false)
-                .test();
+        xContentTester(this::createParser, this::createTestInstance, this::toXContent, PreviewTransformResponse::fromXContent)
+            .supportsUnknownFields(true)
+            .randomFieldsExcludeFilter(path -> path.isEmpty() == false)
+            .test();
     }
 
-    private PreviewTransformResponse createTestInstance() {
-        int numDocs = randomIntBetween(5, 10);
-        List<Map<String, Object>> docs = new ArrayList<>(numDocs);
-        for (int i=0; i<numDocs; i++) {
-            int numFields = randomIntBetween(1, 4);
-            Map<String, Object> doc = new HashMap<>();
-            for (int j=0; j<numFields; j++) {
-                doc.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
-            }
-            docs.add(doc);
-        }
-        int numMappingEntries = randomIntBetween(5, 10);
-        Map<String, Object> mappings = new HashMap<>(numMappingEntries);
-        for (int i = 0; i < numMappingEntries; i++) {
-            mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
+    public void testCreateIndexRequest() throws IOException {
+        PreviewTransformResponse previewResponse = randomPreviewResponse();
+
+        CreateIndexRequest createIndexRequest = previewResponse.getCreateIndexRequest("dest_index");
+        assertEquals("dest_index", createIndexRequest.index());
+        assertThat(createIndexRequest.aliases(), equalTo(previewResponse.getAliases()));
+        assertThat(createIndexRequest.settings(), equalTo(previewResponse.getSettings()));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        builder.map(previewResponse.getMappings());
+
+        assertThat(BytesReference.bytes(builder), equalTo(createIndexRequest.mappings()));
+    }
+
+    public void testBWCPre77XContent() throws IOException {
+        PreviewTransformResponse response = randomPreviewResponse();
+
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+
+        builder.startObject();
+        builder.startArray("preview");
+        for (Map<String, Object> doc : response.getDocs()) {
+            builder.map(doc);
         }
+        builder.endArray();
+        builder.field("mappings", response.getGeneratedDestIndexSettings().getMappings());
+        builder.endObject();
+        XContentParser parser = createParser(builder);
+        PreviewTransformResponse oldResponse = PreviewTransformResponse.fromXContent(parser);
 
-        return new PreviewTransformResponse(docs, mappings);
+        assertThat(response.getDocs(), equalTo(oldResponse.getDocs()));
+        assertThat(response.getMappings(), equalTo(oldResponse.getMappings()));
+        assertTrue(oldResponse.getAliases().isEmpty());
+        assertThat(oldResponse.getSettings(), equalTo(Settings.EMPTY));
+    }
+
+    private PreviewTransformResponse createTestInstance() {
+        return randomPreviewResponse();
     }
 
     private void toXContent(PreviewTransformResponse response, XContentBuilder builder) throws IOException {
@@ -69,7 +97,63 @@ private void toXContent(PreviewTransformResponse response, XContentBuilder build
             builder.map(doc);
         }
         builder.endArray();
-        builder.field("mappings", response.getMappings());
+        builder.startObject("generated_dest_index");
+        builder.field("mappings", response.getGeneratedDestIndexSettings().getMappings());
+
+        builder.startObject("settings");
+        response.getGeneratedDestIndexSettings().getSettings().toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        builder.startObject("aliases");
+        for (Alias alias : response.getGeneratedDestIndexSettings().getAliases()) {
+            alias.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        }
+        builder.endObject();
         builder.endObject();
+        builder.endObject();
+    }
+
+    private static PreviewTransformResponse randomPreviewResponse() {
+        int size = randomIntBetween(0, 10);
+        List<Map<String, Object>> data = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            data.add(Map.of(randomAlphaOfLength(10), Map.of("value1", randomIntBetween(1, 100))));
+        }
+
+        return new PreviewTransformResponse(data, randomGeneratedDestIndexSettings());
+    }
+
+    private static PreviewTransformResponse.GeneratedDestIndexSettings randomGeneratedDestIndexSettings() {
+        int size = randomIntBetween(0, 10);
+
+        Map<String, Object> mappings = null;
+        if (randomBoolean()) {
+            mappings = new HashMap<>(size);
+
+            for (int i = 0; i < size; i++) {
+                mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
+            }
+        }
+
+        Settings settings = null;
+        if (randomBoolean()) {
+            Settings.Builder settingsBuilder = Settings.builder();
+            size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                settingsBuilder.put(randomAlphaOfLength(10), randomBoolean());
+            }
+            settings = settingsBuilder.build();
+        }
+
+        Set<Alias> aliases = null;
+        if (randomBoolean()) {
+            aliases = new HashSet<>();
+            size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                aliases.add(new Alias(randomAlphaOfLength(10)));
+            }
+        }
+
+        return new PreviewTransformResponse.GeneratedDestIndexSettings(mappings, settings, aliases);
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/hlrc/PreviewTransformResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/hlrc/PreviewTransformResponseTests.java
new file mode 100644
index 0000000000000..1aed99f320247
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/hlrc/PreviewTransformResponseTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.transform.hlrc;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.client.transform.PreviewTransformResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction;
+import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Response;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PreviewTransformResponseTests extends AbstractResponseTestCase<
+    PreviewTransformAction.Response,
+    org.elasticsearch.client.transform.PreviewTransformResponse> {
+
+    public static Response randomPreviewResponse() {
+        int size = randomIntBetween(0, 10);
+        List<Map<String, Object>> data = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            data.add(Map.of(randomAlphaOfLength(10), Map.of("value1", randomIntBetween(1, 100))));
+        }
+
+        return new Response(data, randomGeneratedDestIndexSettings());
+    }
+
+    private static TransformDestIndexSettings randomGeneratedDestIndexSettings() {
+        int size = randomIntBetween(0, 10);
+
+        Map<String, Object> mappings = null;
+
+        if (randomBoolean()) {
+            mappings = new HashMap<>(size);
+
+            for (int i = 0; i < size; i++) {
+                mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
+            }
+        }
+
+        Settings settings = null;
+        if (randomBoolean()) {
+            Settings.Builder settingsBuilder = Settings.builder();
+            size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                settingsBuilder.put(randomAlphaOfLength(10), randomBoolean());
+            }
+            settings = settingsBuilder.build();
+        }
+
+        Set<Alias> aliases = null;
+
+        if (randomBoolean()) {
+            aliases = new HashSet<>();
+            size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                aliases.add(new Alias(randomAlphaOfLength(10)));
+            }
+        }
+
+        return new TransformDestIndexSettings(mappings, settings, aliases);
+    }
+
+    @Override
+    protected Response createServerTestInstance(XContentType xContentType) {
+        return randomPreviewResponse();
+    }
+
+    @Override
+    protected PreviewTransformResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.transform.PreviewTransformResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(Response serverTestInstance, PreviewTransformResponse clientInstance) {
+        assertThat(serverTestInstance.getDocs(), equalTo(clientInstance.getDocs()));
+        assertThat(
+            serverTestInstance.getGeneratedDestIndexSettings().getAliases(),
+            equalTo(clientInstance.getGeneratedDestIndexSettings().getAliases())
+        );
+        assertThat(
+            serverTestInstance.getGeneratedDestIndexSettings().getMappings(),
+            equalTo(clientInstance.getGeneratedDestIndexSettings().getMappings())
+        );
+        assertThat(
+            serverTestInstance.getGeneratedDestIndexSettings().getSettings(),
+            equalTo(clientInstance.getGeneratedDestIndexSettings().getSettings())
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
index 66c57e2bf0080..2515429d6af0a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
@@ -12,11 +12,12 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -26,6 +27,7 @@
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -35,6 +37,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PreviewTransformAction extends ActionType<PreviewTransformAction.Response> {
 
@@ -74,7 +77,7 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti
                 }
             }
             content.put(TransformField.DESTINATION.getPreferredName(), tempDestination);
-            content.put(TransformField.ID.getPreferredName(), "transform-preview");
+            content.putIfAbsent(TransformField.ID.getPreferredName(), "transform-preview");
             try (
                 XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().map(content);
                 XContentParser newParser = XContentType.JSON.xContent()
@@ -84,7 +87,7 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti
                         BytesReference.bytes(xContentBuilder).streamInput()
                     )
             ) {
-                return new Request(TransformConfig.fromXContent(newParser, "transform-preview", false));
+                return new Request(TransformConfig.fromXContent(newParser, null, false));
             }
         }
 
@@ -140,18 +143,36 @@ public boolean equals(Object obj) {
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private List<Map<String, Object>> docs;
-        private Map<String, Object> mappings;
-        public static ParseField PREVIEW = new ParseField("preview");
-        public static ParseField MAPPINGS = new ParseField("mappings");
+        public static final ParseField PREVIEW = new ParseField("preview");
+        public static final ParseField GENERATED_DEST_INDEX_SETTINGS = new ParseField("generated_dest_index");
 
-        static final ObjectParser<Response, Void> PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new);
+        private final List<Map<String, Object>> docs;
+        private final TransformDestIndexSettings generatedDestIndexSettings;
+
+        private static final ConstructingObjectParser<Response, Void> PARSER = new ConstructingObjectParser<>(
+            "data_frame_transform_preview",
+            true,
+            args -> {
+                @SuppressWarnings("unchecked")
+                List<Map<String, Object>> docs = (List<Map<String, Object>>) args[0];
+                TransformDestIndexSettings generatedDestIndex = (TransformDestIndexSettings) args[1];
+
+                return new Response(docs, generatedDestIndex);
+            }
+        );
         static {
-            PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), PREVIEW);
-            PARSER.declareObject(Response::setMappings, (p, c) -> p.mapOrdered(), MAPPINGS);
+            PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> p.mapOrdered(), PREVIEW);
+            PARSER.declareObject(
+                optionalConstructorArg(),
+                (p, c) -> TransformDestIndexSettings.fromXContent(p),
+                GENERATED_DEST_INDEX_SETTINGS
+            );
         }
 
-        public Response() {}
+        public Response(List<Map<String, Object>> docs, TransformDestIndexSettings generatedDestIndexSettings) {
+            this.docs = docs;
+            this.generatedDestIndexSettings = generatedDestIndexSettings;
+        }
 
         public Response(StreamInput in) throws IOException {
             int size = in.readInt();
@@ -159,50 +180,22 @@ public Response(StreamInput in) throws IOException {
             for (int i = 0; i < size; i++) {
                 this.docs.add(in.readMap());
             }
-            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_7_0
+                this.generatedDestIndexSettings = new TransformDestIndexSettings(in);
+            } else if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
                 Map<String, Object> objectMap = in.readMap();
-                this.mappings = objectMap == null ? null : Map.copyOf(objectMap);
+                this.generatedDestIndexSettings = new TransformDestIndexSettings(objectMap, null, null);
+            } else {
+                this.generatedDestIndexSettings = new TransformDestIndexSettings(null, null, null);
             }
         }
 
-        public Response(List<Map<String, Object>> docs) {
-            this.docs = new ArrayList<>(docs);
-        }
-
-        public void setDocs(List<Map<String, Object>> docs) {
-            this.docs = new ArrayList<>(docs);
-        }
-
-        public void setMappings(Map<String, Object> mappings) {
-            this.mappings = Map.copyOf(mappings);
+        public List<Map<String, Object>> getDocs() {
+            return docs;
         }
 
-        /**
-         * This takes the a {@code Map<String, String>} of the type "fieldname: fieldtype" and transforms it into the
-         * typical mapping format.
-         *
-         * Example:
-         *
-         * input:
-         * {"field1.subField1": "long", "field2": "keyword"}
-         *
-         * output:
-         * {
-         *     "properties": {
-         *         "field1.subField1": {
-         *             "type": "long"
-         *         },
-         *         "field2": {
-         *             "type": "keyword"
-         *         }
-         *     }
-         * }
-         * @param mappings A Map of the form {"fieldName": "fieldType"}
-         */
-        public void setMappingsFromStringMap(Map<String, String> mappings) {
-            Map<String, Object> fieldMappings = new HashMap<>();
-            mappings.forEach((k, v) -> fieldMappings.put(k, Map.of("type", v)));
-            this.mappings = Map.of("properties", fieldMappings);
+        public TransformDestIndexSettings getGeneratedDestIndexSettings() {
+            return generatedDestIndexSettings;
         }
 
         @Override
@@ -211,8 +204,10 @@ public void writeTo(StreamOutput out) throws IOException {
             for (Map<String, Object> doc : docs) {
                 out.writeMapWithConsistentOrder(doc);
             }
-            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
-                out.writeMap(mappings);
+            if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_7_0
+                generatedDestIndexSettings.writeTo(out);
+            } else if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                out.writeMap(generatedDestIndexSettings.getMappings());
             }
         }
 
@@ -220,9 +215,7 @@ public void writeTo(StreamOutput out) throws IOException {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field(PREVIEW.getPreferredName(), docs);
-            if (mappings != null) {
-                builder.field(MAPPINGS.getPreferredName(), mappings);
-            }
+            builder.field(GENERATED_DEST_INDEX_SETTINGS.getPreferredName(), generatedDestIndexSettings);
             builder.endObject();
             return builder;
         }
@@ -238,12 +231,17 @@ public boolean equals(Object obj) {
             }
 
             Response other = (Response) obj;
-            return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
+            return Objects.equals(other.docs, docs) && Objects.equals(other.generatedDestIndexSettings, generatedDestIndexSettings);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(docs, mappings);
+            return Objects.hash(docs, generatedDestIndexSettings);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this, true, true);
         }
 
         public static Response fromXContent(final XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettings.java
new file mode 100644
index 0000000000000..8d876a8a4c3b6
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettings.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.transform.transforms;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent.Params;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class TransformDestIndexSettings extends AbstractDiffable<TransformDestIndexSettings> implements Writeable, ToXContentObject {
+
+    public static final ParseField MAPPINGS = new ParseField("mappings");
+    public static final ParseField SETTINGS = new ParseField("settings");
+    public static final ParseField ALIASES = new ParseField("aliases");
+
+    private static final ConstructingObjectParser<TransformDestIndexSettings, Void> STRICT_PARSER = createParser(false);
+
+    private final Map<String, Object> mappings;
+    private final Settings settings;
+    private final Set<Alias> aliases;
+
+    private static ConstructingObjectParser<TransformDestIndexSettings, Void> createParser(boolean lenient) {
+        ConstructingObjectParser<TransformDestIndexSettings, Void> PARSER = new ConstructingObjectParser<>(
+            "transform_preview_generated_dest_index",
+            lenient,
+            args -> {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> mappings = (Map<String, Object>) args[0];
+                Settings settings = (Settings) args[1];
+                @SuppressWarnings("unchecked")
+                Set<Alias> aliases = (Set<Alias>) args[2];
+
+                return new TransformDestIndexSettings(mappings, settings, aliases);
+            }
+        );
+
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), MAPPINGS);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
+            Set<Alias> aliases = new HashSet<>();
+            while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                aliases.add(Alias.fromXContent(p));
+            }
+            return aliases;
+        }, ALIASES);
+
+        return PARSER;
+    }
+
+    public TransformDestIndexSettings(Map<String, Object> mappings, Settings settings, Set<Alias> aliases) {
+        this.mappings = mappings == null ? Collections.emptyMap() : Collections.unmodifiableMap(mappings);
+        this.settings = settings == null ? Settings.EMPTY : settings;
+        this.aliases = aliases == null ? Collections.emptySet() : Collections.unmodifiableSet(aliases);
+    }
+
+    public TransformDestIndexSettings(StreamInput in) throws IOException {
+        mappings = in.readMap();
+        settings = Settings.readSettingsFromStream(in);
+        aliases = new HashSet<>(in.readList(Alias::new));
+    }
+
+    public Map<String, Object> getMappings() {
+        return mappings;
+    }
+
+    public Settings getSettings() {
+        return settings;
+    }
+
+    public Set<Alias> getAliases() {
+        return aliases;
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        // note: we write out the full object, even if parts are empty to gain visibility of options
+        builder.startObject();
+        builder.field(MAPPINGS.getPreferredName(), mappings);
+
+        builder.startObject(SETTINGS.getPreferredName());
+        settings.toXContent(builder, params);
+        builder.endObject();
+
+        builder.startObject(ALIASES.getPreferredName());
+        for (Alias alias : aliases) {
+            alias.toXContent(builder, params);
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    public static TransformDestIndexSettings fromXContent(final XContentParser parser) {
+        return STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(mappings);
+        Settings.writeSettingsToStream(settings, out);
+        out.writeCollection(aliases);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+
+        TransformDestIndexSettings other = (TransformDestIndexSettings) obj;
+        return Objects.equals(other.mappings, mappings)
+            && Objects.equals(other.settings, settings)
+            && Objects.equals(other.aliases, aliases);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(mappings, settings, aliases);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
index 9dd8cb6a5c8ee..6916679bf5956 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
@@ -10,15 +10,24 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Response;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettingsTests;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 public class PreviewTransformsActionResponseTests extends AbstractSerializingTestCase<Response> {
 
+    public static Response randomPreviewResponse() {
+        int size = randomIntBetween(0, 10);
+        List<Map<String, Object>> data = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            data.add(Map.of(randomAlphaOfLength(10), Map.of("value1", randomIntBetween(1, 100))));
+        }
+
+        return new Response(data, TransformDestIndexSettingsTests.randomDestIndexSettings());
+    }
 
     @Override
     protected Response doParseInstance(XContentParser parser) throws IOException {
@@ -32,31 +41,7 @@ protected Writeable.Reader<Response> instanceReader() {
 
     @Override
     protected Response createTestInstance() {
-        int size = randomIntBetween(0, 10);
-        List<Map<String, Object>> data = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            data.add(Map.of(randomAlphaOfLength(10), Map.of("value1", randomIntBetween(1, 100))));
-        }
-
-        Response response = new Response(data);
-        if (randomBoolean()) {
-            size = randomIntBetween(0, 10);
-            if (randomBoolean()) {
-                Map<String, Object> mappings = new HashMap<>(size);
-                for (int i = 0; i < size; i++) {
-                    mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
-                }
-                response.setMappings(mappings);
-            } else {
-                Map<String, String> mappings = new HashMap<>(size);
-                for (int i = 0; i < size; i++) {
-                    mappings.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
-                }
-                response.setMappingsFromStringMap(mappings);
-            }
-        }
-
-        return response;
+        return randomPreviewResponse();
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java
index 43bd5e3977643..63ef4f517ff9e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java
@@ -6,28 +6,20 @@
 
 package org.elasticsearch.xpack.core.transform.action;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Response;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
+import java.io.IOException;
 import java.util.Map;
 
 public class PreviewTransformsActionResponseWireTests extends AbstractWireSerializingTransformTestCase<Response> {
 
     @Override
     protected Response createTestInstance() {
-        int size = randomIntBetween(0, 10);
-        List<Map<String, Object>> data = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            Map<String, Object> datum = new HashMap<>();
-            Map<String, Object> entry = new HashMap<>();
-            entry.put("value1", randomIntBetween(1, 100));
-            datum.put(randomAlphaOfLength(10), entry);
-            data.add(datum);
-        }
-        return new Response(data);
+        return PreviewTransformsActionResponseTests.randomPreviewResponse();
     }
 
     @Override
@@ -35,4 +27,27 @@ protected Reader<Response> instanceReader() {
         return Response::new;
     }
 
+    public void testBackwardsSerialization76() throws IOException {
+        Response response = PreviewTransformsActionResponseTests.randomPreviewResponse();
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setVersion(Version.V_7_6_0);
+            output.writeInt(response.getDocs().size());
+            for (Map<String, Object> doc : response.getDocs()) {
+                output.writeMapWithConsistentOrder(doc);
+            }
+
+            output.writeMap(response.getGeneratedDestIndexSettings().getMappings());
+            try (StreamInput in = output.bytes().streamInput()) {
+                in.setVersion(Version.V_7_6_0);
+                Response streamedResponse = new Response(in);
+                assertEquals(
+                    response.getGeneratedDestIndexSettings().getMappings(),
+                    streamedResponse.getGeneratedDestIndexSettings().getMappings()
+                );
+                assertEquals(response.getDocs(), streamedResponse.getDocs());
+            }
+        }
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java
new file mode 100644
index 0000000000000..73c380daefe21
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.transform.transforms;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class TransformDestIndexSettingsTests extends AbstractSerializingTransformTestCase<TransformDestIndexSettings> {
+
+    public static TransformDestIndexSettings randomDestIndexSettings() {
+        int size = randomIntBetween(0, 10);
+
+        Map<String, Object> mappings = null;
+
+        if (randomBoolean()) {
+            mappings = new HashMap<>(size);
+
+            for (int i = 0; i < size; i++) {
+                mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
+            }
+        }
+
+        Settings settings = null;
+        if (randomBoolean()) {
+            Settings.Builder settingsBuilder = Settings.builder();
+            size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                settingsBuilder.put(randomAlphaOfLength(10), randomBoolean());
+            }
+            settings = settingsBuilder.build();
+        }
+
+        Set<Alias> aliases = null;
+
+        if (randomBoolean()) {
+            aliases = new HashSet<>();
+            size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                aliases.add(new Alias(randomAlphaOfLength(10)));
+            }
+        }
+
+        return new TransformDestIndexSettings(mappings, settings, aliases);
+    }
+
+    @Override
+    protected TransformDestIndexSettings doParseInstance(XContentParser parser) throws IOException {
+        return TransformDestIndexSettings.fromXContent(parser);
+    }
+
+    @Override
+    protected Reader<TransformDestIndexSettings> instanceReader() {
+        return TransformDestIndexSettings::new;
+    }
+
+    @Override
+    protected TransformDestIndexSettings createTestInstance() {
+        return randomDestIndexSettings();
+    }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml
index 7abb83834f515..1064cb5414814 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml
@@ -98,11 +98,11 @@ setup:
   - match: { preview.2.avg_response: 42.0 }
   - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
   - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
-  - match: { mappings.properties.airline.type: "keyword" }
-  - match: { mappings.properties.by-hour.type: "date" }
-  - match: { mappings.properties.avg_response.type: "double" }
-  - match: { mappings.properties.time\.max.type: "date" }
-  - match: { mappings.properties.time\.min.type: "date" }
+  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
+  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
+  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
+  - match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
+  - match: { generated_dest_index.mappings.properties.time\.min.type: "date" }
 
   - do:
       ingest.put_pipeline:
@@ -146,9 +146,9 @@ setup:
   - match: { preview.2.by-hour: 1487379600000 }
   - match: { preview.2.avg_response: 42.0 }
   - match: { preview.2.my_field: 42 }
-  - match: { mappings.properties.airline.type: "keyword" }
-  - match: { mappings.properties.by-hour.type: "date" }
-  - match: { mappings.properties.avg_response.type: "double" }
+  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
+  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
+  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
 
 ---
 "Test preview transform with invalid config":
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
index 4896194279e46..46f18afa23c25 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
@@ -8,6 +8,7 @@
 
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ingest.SimulatePipelineAction;
@@ -48,11 +49,14 @@
 import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction;
 import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
+import org.elasticsearch.xpack.transform.persistence.TransformIndex;
 import org.elasticsearch.xpack.transform.transforms.pivot.AggregationResultUtils;
 import org.elasticsearch.xpack.transform.transforms.pivot.Pivot;
 import org.elasticsearch.xpack.transform.utils.SourceDestValidations;
 
+import java.time.Clock;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -62,8 +66,9 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.transform.transforms.TransformIndexer.COMPOSITE_AGGREGATION_NAME;
 
-public class TransportPreviewTransformAction extends
-    HandledTransportAction<PreviewTransformAction.Request, PreviewTransformAction.Response> {
+public class TransportPreviewTransformAction extends HandledTransportAction<
+    PreviewTransformAction.Request,
+    PreviewTransformAction.Response> {
 
     private static final Logger logger = LogManager.getLogger(TransportPreviewTransformAction.class);
     private static final int NUMBER_OF_PREVIEW_BUCKETS = 100;
@@ -165,7 +170,14 @@ protected void doExecute(Task task, PreviewTransformAction.Request request, Acti
                     return;
                 }
 
-                getPreview(pivot, config.getSource(), config.getDestination().getPipeline(), config.getDestination().getIndex(), listener);
+                getPreview(
+                    config.getId(), // note: @link{PreviewTransformAction} sets an id, so this is never null
+                    pivot,
+                    config.getSource(),
+                    config.getDestination().getPipeline(),
+                    config.getDestination().getIndex(),
+                    listener
+                );
 
             }, listener::onFailure)
         );
@@ -173,27 +185,34 @@ protected void doExecute(Task task, PreviewTransformAction.Request request, Acti
 
     @SuppressWarnings("unchecked")
     private void getPreview(
+        String transformId,
         Pivot pivot,
         SourceConfig source,
         String pipeline,
         String dest,
         ActionListener<PreviewTransformAction.Response> listener
     ) {
-        final PreviewTransformAction.Response previewResponse = new PreviewTransformAction.Response();
+        final SetOnce<Map<String, String>> mappings = new SetOnce<>();
+
         ActionListener<SimulatePipelineResponse> pipelineResponseActionListener = ActionListener.wrap(simulatePipelineResponse -> {
-            List<Map<String, Object>> response = new ArrayList<>(simulatePipelineResponse.getResults().size());
+            List<Map<String, Object>> docs = new ArrayList<>(simulatePipelineResponse.getResults().size());
             for (var simulateDocumentResult : simulatePipelineResponse.getResults()) {
                 try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) {
                     XContentBuilder content = simulateDocumentResult.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
                     Map<String, Object> tempMap = XContentHelper.convertToMap(BytesReference.bytes(content), true, XContentType.JSON).v2();
-                    response.add((Map<String, Object>) XContentMapValues.extractValue("doc._source", tempMap));
+                    docs.add((Map<String, Object>) XContentMapValues.extractValue("doc._source", tempMap));
                 }
             }
-            previewResponse.setDocs(response);
-            listener.onResponse(previewResponse);
+            TransformDestIndexSettings generateddestIndexSettings = TransformIndex.createTransformDestIndexSettings(
+                mappings.get(),
+                transformId,
+                Clock.systemUTC()
+            );
+
+            listener.onResponse(new PreviewTransformAction.Response(docs, generateddestIndexSettings));
         }, listener::onFailure);
         pivot.deduceMappings(client, source, ActionListener.wrap(deducedMappings -> {
-            previewResponse.setMappingsFromStringMap(deducedMappings);
+            mappings.set(deducedMappings);
             ClientHelper.executeWithHeadersAsync(
                 threadPool.getThreadContext().getHeaders(),
                 ClientHelper.TRANSFORM_ORIGIN,
@@ -214,11 +233,17 @@ private void getPreview(
                         // remove all internal fields
 
                         if (pipeline == null) {
-                            List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
+                            List<Map<String, Object>> docs = pivot.extractResults(agg, deducedMappings, stats)
                                 .peek(doc -> doc.keySet().removeIf(k -> k.startsWith("_")))
                                 .collect(Collectors.toList());
-                            previewResponse.setDocs(results);
-                            listener.onResponse(previewResponse);
+
+                            TransformDestIndexSettings generateddestIndexSettings = TransformIndex.createTransformDestIndexSettings(
+                                mappings.get(),
+                                transformId,
+                                Clock.systemUTC()
+                            );
+
+                            listener.onResponse(new PreviewTransformAction.Response(docs, generateddestIndexSettings));
                         } else {
                             List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats).map(doc -> {
                                 Map<String, Object> src = new HashMap<>();
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
index be5dd13aee314..9598e2f400e6f 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.xpack.core.transform.TransformMessages;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
 import org.elasticsearch.xpack.core.transform.transforms.TransformState;
 import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
 import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
@@ -298,8 +299,14 @@ private void createDestinationIndex(final TransformConfig config, final ActionLi
 
         final Pivot pivot = new Pivot(config.getPivotConfig());
 
-        ActionListener<Map<String, String>> deduceMappingsListener = ActionListener.wrap(
-            mappings -> TransformIndex.createDestinationIndex(client, Clock.systemUTC(), config, mappings, listener),
+        ActionListener<Map<String, String>> deduceMappingsListener = ActionListener.wrap(mappings -> {
+            TransformDestIndexSettings generateddestIndexSettings = TransformIndex.createTransformDestIndexSettings(
+                mappings,
+                config.getId(),
+                Clock.systemUTC()
+            );
+            TransformIndex.createDestinationIndex(client, config, generateddestIndexSettings, listener);
+        },
             deduceTargetMappingsException -> listener.onFailure(
                 new RuntimeException(TransformMessages.REST_PUT_TRANSFORM_FAILED_TO_DEDUCE_DEST_MAPPINGS, deduceTargetMappingsException)
             )
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
index 9764a2ddbac5f..1df334036cd9a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction.Response;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
 import org.elasticsearch.xpack.transform.TransformServices;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
 import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
@@ -350,14 +351,20 @@ private void updateTransform(
     }
 
     private void createDestination(Pivot pivot, TransformConfig config, ActionListener<Void> listener) {
-        ActionListener<Map<String, String>> deduceMappingsListener = ActionListener.wrap(
-            mappings -> TransformIndex.createDestinationIndex(
+        ActionListener<Map<String, String>> deduceMappingsListener = ActionListener.wrap(mappings -> {
+            TransformDestIndexSettings generateddestIndexSettings = TransformIndex.createTransformDestIndexSettings(
+                mappings,
+                config.getId(),
+                Clock.systemUTC()
+            );
+            TransformIndex.createDestinationIndex(
                 client,
-                Clock.systemUTC(),
                 config,
-                mappings,
+                generateddestIndexSettings,
                 ActionListener.wrap(r -> listener.onResponse(null), listener::onFailure)
-            ),
+            );
+        },
+
             deduceTargetMappingsException -> listener.onFailure(
                 new RuntimeException(TransformMessages.REST_PUT_TRANSFORM_FAILED_TO_DEDUCE_DEST_MAPPINGS, deduceTargetMappingsException)
             )
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java
index 90938fb693415..c22ad441098c8 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java
@@ -10,99 +10,134 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
 
-import java.io.IOException;
 import java.time.Clock;
+import java.util.HashMap;
 import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
+import java.util.Set;
 
 public final class TransformIndex {
     private static final Logger logger = LogManager.getLogger(TransformIndex.class);
 
     private static final String PROPERTIES = "properties";
-    private static final String TYPE = "type";
     private static final String META = "_meta";
 
-    private TransformIndex() {
-    }
+    private TransformIndex() {}
 
-    public static void createDestinationIndex(Client client,
-                                              Clock clock,
-                                              TransformConfig transformConfig,
-                                              Map<String, String> mappings,
-                                              ActionListener<Boolean> listener) {
+    public static void createDestinationIndex(
+        Client client,
+        TransformConfig transformConfig,
+        TransformDestIndexSettings destIndexSettings,
+        ActionListener<Boolean> listener
+    ) {
         CreateIndexRequest request = new CreateIndexRequest(transformConfig.getDestination().getIndex());
 
-        // TODO: revisit number of shards, number of replicas
-        request.settings(Settings.builder() // <1>
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"));
-
-        request.mapping(createMappingXContent(mappings, transformConfig.getId(), clock));
-
-        client.execute(CreateIndexAction.INSTANCE, request, ActionListener.wrap(createIndexResponse -> {
-            listener.onResponse(true);
-        }, e -> {
-            String message = TransformMessages.getMessage(TransformMessages.FAILED_TO_CREATE_DESTINATION_INDEX,
-                    transformConfig.getDestination().getIndex(), transformConfig.getId());
-            logger.error(message);
-            listener.onFailure(new RuntimeException(message, e));
-        }));
-    }
-
-    private static XContentBuilder createMappingXContent(Map<String, String> mappings,
-                                                         String id,
-                                                         Clock clock) {
-        try {
-            XContentBuilder builder = jsonBuilder().startObject();
-            builder.startObject(SINGLE_MAPPING_NAME);
-            addProperties(builder, mappings);
-            addMetaData(builder, id, clock);
-            builder.endObject(); // _doc type
-            return builder.endObject();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
+        request.settings(destIndexSettings.getSettings());
+        request.mapping(destIndexSettings.getMappings());
+        for (Alias alias : destIndexSettings.getAliases()) {
+            request.alias(alias);
         }
+
+        client.execute(
+            CreateIndexAction.INSTANCE,
+            request,
+            ActionListener.wrap(createIndexResponse -> { listener.onResponse(true); }, e -> {
+                String message = TransformMessages.getMessage(
+                    TransformMessages.FAILED_TO_CREATE_DESTINATION_INDEX,
+                    transformConfig.getDestination().getIndex(),
+                    transformConfig.getId()
+                );
+                logger.error(message);
+                listener.onFailure(new RuntimeException(message, e));
+            })
+        );
     }
 
-    private static XContentBuilder addProperties(XContentBuilder builder,
-                                                 Map<String, String> mappings) throws IOException {
-        builder.startObject(PROPERTIES);
-        for (Entry<String, String> field : mappings.entrySet()) {
-            String fieldName = field.getKey();
-            String fieldType = field.getValue();
+    public static TransformDestIndexSettings createTransformDestIndexSettings(Map<String, String> mappings, String id, Clock clock) {
+        Map<String, Object> indexMappings = new HashMap<>();
+        indexMappings.put(PROPERTIES, createMappingsFromStringMap(mappings));
+        indexMappings.put(META, createMetaData(id, clock));
 
-            builder.startObject(fieldName);
-            builder.field(TYPE, fieldType);
+        Settings settings = createSettings();
 
-            builder.endObject();
-        }
-        builder.endObject(); // PROPERTIES
-        return builder;
+        // transform does not create aliases, however the user might customize this in future
+        Set<Alias> aliases = null;
+        return new TransformDestIndexSettings(indexMappings, settings, aliases);
+    }
+
+    /*
+     * Return meta data that stores some useful information about the transform index, stored as "_meta":
+     *
+     * {
+     *   "created_by" : "transform",
+     *   "_transform" : {
+     *     "transform" : "id",
+     *     "version" : {
+     *       "created" : "8.0.0"
+     *     },
+     *     "creation_date_in_millis" : 1584025695202
+     *   }
+     * }
+     */
+    private static Map<String, Object> createMetaData(String id, Clock clock) {
+
+        Map<String, Object> metaData = new HashMap<>();
+        metaData.put(TransformField.CREATED_BY, TransformField.TRANSFORM_SIGNATURE);
+
+        Map<String, Object> transformMetaData = new HashMap<>();
+        transformMetaData.put(TransformField.CREATION_DATE_MILLIS, clock.millis());
+        transformMetaData.put(TransformField.VERSION.getPreferredName(), Map.of(TransformField.CREATED, Version.CURRENT));
+        transformMetaData.put(TransformField.TRANSFORM, id);
+
+        metaData.put(TransformField.META_FIELDNAME, transformMetaData);
+        return metaData;
+    }
+
+    /**
+     * creates generated index settings, hardcoded at the moment, in future this might be customizable or generation could
+     * be based on source settings.
+     */
+    private static Settings createSettings() {
+        return Settings.builder() // <1>
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .build();
     }
 
-    private static XContentBuilder addMetaData(XContentBuilder builder, String id, Clock clock) throws IOException {
-        return builder.startObject(META)
-            .field(TransformField.CREATED_BY, TransformField.TRANSFORM_SIGNATURE)
-            .startObject(TransformField.META_FIELDNAME)
-                .field(TransformField.CREATION_DATE_MILLIS, clock.millis())
-                .startObject(TransformField.VERSION.getPreferredName())
-                    .field(TransformField.CREATED, Version.CURRENT)
-                .endObject()
-                .field(TransformField.TRANSFORM, id)
-            .endObject() // META_FIELDNAME
-        .endObject(); // META
+    /**
+     * This takes the a {@code Map<String, String>} of the type "fieldname: fieldtype" and transforms it into the
+     * typical mapping format.
+     *
+     * Example:
+     *
+     * input:
+     * {"field1.subField1": "long", "field2": "keyword"}
+     *
+     * output:
+     * {
+     *   "field1.subField1": {
+     *     "type": "long"
+     *   },
+     *   "field2": {
+     *     "type": "keyword"
+     *   }
+     * }
+     * @param mappings A Map of the form {"fieldName": "fieldType"}
+     */
+    private static Map<String, Object> createMappingsFromStringMap(Map<String, String> mappings) {
+        Map<String, Object> fieldMappings = new HashMap<>();
+        mappings.forEach((k, v) -> fieldMappings.put(k, Map.of("type", v)));
+
+        return fieldMappings;
     }
 }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java
index 03bdc9348db4a..9d924915f80a3 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java
@@ -42,23 +42,19 @@ public class TransformIndexTests extends ESTestCase {
     private Clock clock = Clock.fixed(Instant.ofEpochMilli(CURRENT_TIME_MILLIS), ZoneId.systemDefault());
 
     public void testCreateDestinationIndex() throws IOException {
-        doAnswer(
-            invocationOnMock -> {
-                @SuppressWarnings("unchecked")
-                ActionListener<CreateIndexResponse> listener = (ActionListener<CreateIndexResponse>) invocationOnMock.getArguments()[2];
-                listener.onResponse(null);
-                return null;
-            })
-            .when(client).execute(any(), any(), any());
+        doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<CreateIndexResponse> listener = (ActionListener<CreateIndexResponse>) invocationOnMock.getArguments()[2];
+            listener.onResponse(null);
+            return null;
+        }).when(client).execute(any(), any(), any());
 
         TransformIndex.createDestinationIndex(
             client,
-            clock,
             TransformConfigTests.randomTransformConfig(TRANSFORM_ID),
-            new HashMap<>(),
-            ActionListener.wrap(
-                value -> assertTrue(value),
-                e -> fail(e.getMessage())));
+            TransformIndex.createTransformDestIndexSettings(new HashMap<>(), TRANSFORM_ID, clock),
+            ActionListener.wrap(value -> assertTrue(value), e -> fail(e.getMessage()))
+        );
 
         ArgumentCaptor<CreateIndexRequest> createIndexRequestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
         verify(client).execute(eq(CreateIndexAction.INSTANCE), createIndexRequestCaptor.capture(), any());

From 8ffe5d1f9499f751bc173d6965a2c2c136308eda Mon Sep 17 00:00:00 2001
From: bellengao <gbl_long@163.com>
Date: Wed, 18 Mar 2020 04:22:30 +0800
Subject: [PATCH 085/285] Support array for all string ingest processors

---
 .../ingest/processors/bytes.asciidoc          |  2 +-
 .../reference/ingest/processors/gsub.asciidoc |  2 +-
 .../ingest/processors/html_strip.asciidoc     |  2 +-
 .../ingest/processors/lowercase.asciidoc      |  2 +-
 .../reference/ingest/processors/trim.asciidoc |  2 +-
 .../ingest/processors/uppercase.asciidoc      |  2 +-
 .../ingest/processors/url-decode.asciidoc     |  2 +-
 .../common/AbstractStringProcessor.java       | 29 ++++++++++++++--
 .../AbstractStringProcessorTestCase.java      | 33 ++++++++++++++++++-
 9 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/docs/reference/ingest/processors/bytes.asciidoc b/docs/reference/ingest/processors/bytes.asciidoc
index 76f054cac64c2..5a551f8a82eac 100644
--- a/docs/reference/ingest/processors/bytes.asciidoc
+++ b/docs/reference/ingest/processors/bytes.asciidoc
@@ -1,6 +1,6 @@
 [[bytes-processor]]
 === Bytes Processor
-Converts a human readable byte value (e.g. 1kb) to its value in bytes (e.g. 1024).
+Converts a human readable byte value (e.g. 1kb) to its value in bytes (e.g. 1024). If the field is an array of strings, all members of the array will be converted.
 
 Supported human readable units are "b", "kb", "mb", "gb", "tb", "pb" case insensitive. An error will occur if
 the field is not a supported format or resultant value exceeds 2^63.
diff --git a/docs/reference/ingest/processors/gsub.asciidoc b/docs/reference/ingest/processors/gsub.asciidoc
index f6919eb1e95f1..2defa6e7cd138 100644
--- a/docs/reference/ingest/processors/gsub.asciidoc
+++ b/docs/reference/ingest/processors/gsub.asciidoc
@@ -1,7 +1,7 @@
 [[gsub-processor]]
 === Gsub Processor
 Converts a string field by applying a regular expression and a replacement.
-If the field is not a string, the processor will throw an exception.
+If the field is an array of string, all members of the array will be converted. If any non-string values are encountered, the processor will throw an exception. 
 
 [[gsub-options]]
 .Gsub Options
diff --git a/docs/reference/ingest/processors/html_strip.asciidoc b/docs/reference/ingest/processors/html_strip.asciidoc
index 2fa3cd7bbb8ae..bd4e8e8ccd920 100644
--- a/docs/reference/ingest/processors/html_strip.asciidoc
+++ b/docs/reference/ingest/processors/html_strip.asciidoc
@@ -1,6 +1,6 @@
 [[htmlstrip-processor]]
 === HTML Strip Processor
-Removes HTML from field.
+Removes HTML tags from the field. If the field is an array of strings, HTML tags will be removed from all members of the array.
 
 NOTE: Each HTML tag is replaced with a `\n` character.
 
diff --git a/docs/reference/ingest/processors/lowercase.asciidoc b/docs/reference/ingest/processors/lowercase.asciidoc
index 878b74ed9ba24..903d69625352f 100644
--- a/docs/reference/ingest/processors/lowercase.asciidoc
+++ b/docs/reference/ingest/processors/lowercase.asciidoc
@@ -1,6 +1,6 @@
 [[lowercase-processor]]
 === Lowercase Processor
-Converts a string to its lowercase equivalent.
+Converts a string to its lowercase equivalent. If the field is an array of strings, all members of the array will be converted.
 
 [[lowercase-options]]
 .Lowercase Options
diff --git a/docs/reference/ingest/processors/trim.asciidoc b/docs/reference/ingest/processors/trim.asciidoc
index 7c28767076ecc..ef3611161e2e2 100644
--- a/docs/reference/ingest/processors/trim.asciidoc
+++ b/docs/reference/ingest/processors/trim.asciidoc
@@ -1,6 +1,6 @@
 [[trim-processor]]
 === Trim Processor
-Trims whitespace from field.
+Trims whitespace from field. If the field is an array of strings, all members of the array will be trimmed.
 
 NOTE: This only works on leading and trailing whitespace.
 
diff --git a/docs/reference/ingest/processors/uppercase.asciidoc b/docs/reference/ingest/processors/uppercase.asciidoc
index 7565be1c7c303..3e26cedcf9cce 100644
--- a/docs/reference/ingest/processors/uppercase.asciidoc
+++ b/docs/reference/ingest/processors/uppercase.asciidoc
@@ -1,6 +1,6 @@
 [[uppercase-processor]]
 === Uppercase Processor
-Converts a string to its uppercase equivalent.
+Converts a string to its uppercase equivalent. If the field is an array of strings, all members of the array will be converted.
 
 [[uppercase-options]]
 .Uppercase Options
diff --git a/docs/reference/ingest/processors/url-decode.asciidoc b/docs/reference/ingest/processors/url-decode.asciidoc
index 76fc00c80f679..268fce1c18c2a 100644
--- a/docs/reference/ingest/processors/url-decode.asciidoc
+++ b/docs/reference/ingest/processors/url-decode.asciidoc
@@ -1,6 +1,6 @@
 [[urldecode-processor]]
 === URL Decode Processor
-URL-decodes a string
+URL-decodes a string. If the field is an array of strings, all members of the array will be decoded.
 
 [[urldecode-options]]
 .URL Decode Options
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index 546519aa5f606..ded75e95be73d 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -58,7 +60,8 @@ String getTargetField() {
 
     @Override
     public final IngestDocument execute(IngestDocument document) {
-        String val = document.getFieldValue(field, String.class, ignoreMissing);
+        Object val = document.getFieldValue(field, Object.class, ignoreMissing);
+        Object newValue;
 
         if (val == null && ignoreMissing) {
             return document;
@@ -66,7 +69,29 @@ public final IngestDocument execute(IngestDocument document) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
         }
 
-        document.setFieldValue(targetField, process(val));
+        if (val instanceof List) {
+            List<?> list = (List<?>) val;
+            List<Object> newList = new ArrayList<>(list.size());
+            for (Object value : list) {
+                if (value instanceof String) {
+                    newList.add(process((String) value));
+                } else {
+                    throw new IllegalArgumentException("value [" + value + "] of type [" + value.getClass().getName() +
+                        "] in list field [" + field + "] cannot be cast to [" + String.class.getName() + "]");
+                }
+            }
+            newValue = newList;
+        } else {
+            if (val instanceof String) {
+                newValue = process((String) val);
+            } else {
+                throw new IllegalArgumentException("field [" + field + "] of type [" + val.getClass().getName() + "] cannot be cast to [" +
+                    String.class.getName() + "]");
+            }
+
+        }
+
+        document.setFieldValue(targetField, newValue);
         return document;
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
index f667f84e5d7b1..b9c83be40fff7 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
@@ -24,8 +24,10 @@
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsString;
@@ -41,7 +43,7 @@ protected String modifyInput(String input) {
 
     protected abstract T expectedResult(String input);
 
-    protected Class<?> expectedResultType(){
+    protected Class<?> expectedResultType() {
         return String.class;  // most results types are Strings
     }
 
@@ -52,6 +54,19 @@ public void testProcessor() throws Exception {
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(expectedResult(fieldValue)));
+
+        int numItems = randomIntBetween(1, 10);
+        List<String> fieldValueList = new ArrayList<>();
+        List<T> expectedList = new ArrayList<>();
+        for (int i = 0; i < numItems; i++) {
+            String randomString = RandomDocumentPicks.randomString(random());
+            fieldValueList.add(modifyInput(randomString));
+            expectedList.add(expectedResult(randomString));
+        }
+        String multiValueFieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValueList);
+        Processor multiValueProcessor = newProcessor(multiValueFieldName, randomBoolean(), multiValueFieldName);
+        multiValueProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(multiValueFieldName, List.class), equalTo(expectedList));
     }
 
     public void testFieldNotFound() throws Exception {
@@ -94,6 +109,14 @@ public void testNonStringValue() throws Exception {
         Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
         assertThat(e.getMessage(), equalTo("field [" + fieldName +
             "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+
+        List<Integer> fieldValueList = new ArrayList<>();
+        int randomValue = randomInt();
+        fieldValueList.add(randomValue);
+        ingestDocument.setFieldValue(fieldName, fieldValueList);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("value [" + randomValue + "] of type [java.lang.Integer] in list field [" + fieldName +
+            "] cannot be cast to [java.lang.String]"));
     }
 
     public void testNonStringValueWithIgnoreMissing() throws Exception {
@@ -104,6 +127,14 @@ public void testNonStringValueWithIgnoreMissing() throws Exception {
         Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
         assertThat(e.getMessage(), equalTo("field [" + fieldName +
             "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+
+        List<Integer> fieldValueList = new ArrayList<>();
+        int randomValue = randomInt();
+        fieldValueList.add(randomValue);
+        ingestDocument.setFieldValue(fieldName, fieldValueList);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("value [" + randomValue + "] of type [java.lang.Integer] in list field [" + fieldName +
+            "] cannot be cast to [java.lang.String]"));
     }
 
     public void testTargetField() throws Exception {

From 49131091614c9270bdfb35efa7f0aaf29c5829b6 Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim.vernum@elastic.co>
Date: Wed, 18 Mar 2020 10:56:48 +1100
Subject: [PATCH 086/285] Add "grant_api_key" cluster privilege (#53527)

This change adds a new cluster privilege "grant_api_key" that allows
the use of the new /_security/api_key/grant endpoint
---
 .../docs/en/rest-api/security/get-builtin-privileges.asciidoc | 1 +
 .../security/authz/privilege/ClusterPrivilegeResolver.java    | 4 ++++
 .../org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java | 2 +-
 .../resources/rest-api-spec/test/privileges/11_builtin.yml    | 2 +-
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc b/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc
index 6dc32629f4650..fea1b35f53476 100644
--- a/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc
+++ b/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc
@@ -64,6 +64,7 @@ A successful call returns an object with "cluster" and "index" fields.
     "all",
     "create_snapshot",
     "delegate_pki",
+    "grant_api_key",
     "manage",
     "manage_api_key",
     "manage_autoscaling",
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
index 7715b81c30c4b..7fa41f033ea44 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.core.ilm.action.StartILMAction;
 import org.elasticsearch.xpack.core.ilm.action.StopILMAction;
 import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationAction;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenAction;
 import org.elasticsearch.xpack.core.security.action.token.RefreshTokenAction;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
@@ -48,6 +49,7 @@ public class ClusterPrivilegeResolver {
     private static final Set<String> MANAGE_OIDC_PATTERN = Set.of("cluster:admin/xpack/security/oidc/*");
     private static final Set<String> MANAGE_TOKEN_PATTERN = Set.of("cluster:admin/xpack/security/token/*");
     private static final Set<String> MANAGE_API_KEY_PATTERN = Set.of("cluster:admin/xpack/security/api_key/*");
+    private static final Set<String> GRANT_API_KEY_PATTERN = Set.of(GrantApiKeyAction.NAME + "*");
     private static final Set<String> MONITOR_PATTERN = Set.of("cluster:monitor/*");
     private static final Set<String> MONITOR_ML_PATTERN = Set.of("cluster:monitor/xpack/ml/*");
     private static final Set<String> MONITOR_TRANSFORM_PATTERN = Set.of("cluster:monitor/data_frame/*", "cluster:monitor/transform/*");
@@ -106,6 +108,7 @@ public class ClusterPrivilegeResolver {
     public static final NamedClusterPrivilege MANAGE_SAML = new ActionClusterPrivilege("manage_saml", MANAGE_SAML_PATTERN);
     public static final NamedClusterPrivilege MANAGE_OIDC = new ActionClusterPrivilege("manage_oidc", MANAGE_OIDC_PATTERN);
     public static final NamedClusterPrivilege MANAGE_API_KEY = new ActionClusterPrivilege("manage_api_key", MANAGE_API_KEY_PATTERN);
+    public static final NamedClusterPrivilege GRANT_API_KEY = new ActionClusterPrivilege("grant_api_key", GRANT_API_KEY_PATTERN);
     public static final NamedClusterPrivilege MANAGE_PIPELINE = new ActionClusterPrivilege("manage_pipeline", Set.of("cluster:admin" +
         "/ingest/pipeline/*"));
     public static final NamedClusterPrivilege MANAGE_AUTOSCALING = new ActionClusterPrivilege(
@@ -148,6 +151,7 @@ public class ClusterPrivilegeResolver {
         MANAGE_SAML,
         MANAGE_OIDC,
         MANAGE_API_KEY,
+        GRANT_API_KEY,
         MANAGE_PIPELINE,
         MANAGE_ROLLUP,
         MANAGE_AUTOSCALING,
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
index daa0cfc303142..ca642e93cad4d 100644
--- a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
+++ b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
@@ -46,7 +46,7 @@ public class ApiKeyRestIT extends SecurityOnTrialLicenseRestTestCase {
     @Before
     public void createUsers() throws IOException {
         createUser(SYSTEM_USER, SYSTEM_USER_PASSWORD, List.of("system_role"));
-        createRole("system_role", Set.of("manage_api_key"));
+        createRole("system_role", Set.of("grant_api_key"));
         createUser(END_USER, END_USER_PASSWORD, List.of("user_role"));
         createRole("user_role", Set.of("monitor"));
     }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml
index 61971fd91b9b9..2dca2483aaf2d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml
@@ -15,5 +15,5 @@ setup:
   # This is fragile - it needs to be updated every time we add a new cluster/index privilege
   # I would much prefer we could just check that specific entries are in the array, but we don't have
   # an assertion for that
-  - length: { "cluster" : 35 }
+  - length: { "cluster" : 36 }
   - length: { "index" : 18 }

From bb89c62f3cdb6d3797b4f9715bf2b87bff172ca8 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Wed, 18 Mar 2020 08:40:12 +0200
Subject: [PATCH 087/285] Fix potential bug in concurrent token refresh support
 (#53668)

Ensure that we do not proceed execution after calling the
listerer's onFailure
---
 .../org/elasticsearch/xpack/security/authc/TokenService.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index c12b14477b8e7..f32a57c1d7533 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -1101,9 +1101,10 @@ void decryptAndReturnSupersedingTokens(String refreshToken, RefreshTokenStatus r
             if (decryptedTokens.length != 2) {
                 logger.warn("Decrypted tokens string is not correctly formatted");
                 listener.onFailure(invalidGrantException("could not refresh the requested token"));
+            } else {
+                listener.onResponse(new Tuple<>(prependVersionAndEncodeAccessToken(refreshTokenStatus.getVersion(), decryptedTokens[0]),
+                    prependVersionAndEncodeRefreshToken(refreshTokenStatus.getVersion(), decryptedTokens[1])));
             }
-            listener.onResponse(new Tuple<>(prependVersionAndEncodeAccessToken(refreshTokenStatus.getVersion(), decryptedTokens[0]),
-                prependVersionAndEncodeRefreshToken(refreshTokenStatus.getVersion(), decryptedTokens[1])));
         } catch (GeneralSecurityException | IOException e) {
             logger.warn("Could not get stored superseding token values", e);
             listener.onFailure(invalidGrantException("could not refresh the requested token"));

From 98c509acdc7f2b568526590fc7b3a4cdf2eae00a Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Wed, 18 Mar 2020 08:37:01 +0100
Subject: [PATCH 088/285] [Transform] disable BWC as preparation for #53695
 (#53696)

disable BWC as preparation for #53695
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 2dbf94ae7dabb..a151683b4847d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53695" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 221b005a45bf4b3d30987fb982844b818b157c05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= <przemyslaw.witek@elastic.co>
Date: Wed, 18 Mar 2020 09:27:59 +0100
Subject: [PATCH 089/285] Make ML index aliases hidden (#53160)

---
 .../core/ml/annotations/AnnotationIndex.java  |  8 +++--
 .../xpack/core/ml/utils/MlIndexAndAlias.java  |  4 +--
 .../core/ml/utils/MlIndexAndAliasTests.java   |  8 ++---
 .../ml/integration/DeleteExpiredDataIT.java   |  3 ++
 .../xpack/ml/integration/MlJobIT.java         | 35 +++++++++++--------
 .../ml/action/TransportDeleteJobAction.java   |  2 +-
 .../job/persistence/JobResultsProvider.java   | 13 +++++--
 .../ml/integration/AnnotationIndexIT.java     |  5 +++
 .../ml/integration/JobResultsProviderIT.java  | 22 +++++++-----
 .../test/ml/custom_all_field.yml              |  1 +
 .../rest-api-spec/test/ml/index_layout.yml    | 21 +++++++++++
 .../rest-api-spec/test/ml/jobs_crud.yml       |  2 +-
 12 files changed, 87 insertions(+), 37 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
index 2ef467562929b..131a1053519a7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
@@ -44,9 +44,11 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
                                                          final ActionListener<Boolean> finalListener) {
 
         final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
-            final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
-                .addAlias(INDEX_NAME, READ_ALIAS_NAME)
-                .addAlias(INDEX_NAME, WRITE_ALIAS_NAME).request();
+            final IndicesAliasesRequest request =
+                client.admin().indices().prepareAliases()
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
+                    .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
                 ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
                 client.admin().indices()::aliases);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
index 2c3aba673c6d0..9a246183cfa18 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
@@ -136,7 +136,7 @@ private static void createFirstConcreteIndex(Client client,
             .indices()
             .prepareCreate(index);
         if (addAlias) {
-            requestBuilder.addAlias(new Alias(alias));
+            requestBuilder.addAlias(new Alias(alias).isHidden(true));
         }
         CreateIndexRequest request = requestBuilder.request();
 
@@ -166,7 +166,7 @@ private static void updateWriteAlias(Client client,
         IndicesAliasesRequestBuilder requestBuilder = client.admin()
             .indices()
             .prepareAliases()
-            .addAlias(newIndex, alias);
+            .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias).isHidden(true));
         if (currentIndex != null) {
             requestBuilder.removeAlias(currentIndex, alias);
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
index 850d197aa6d8b..eb745423177d8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
@@ -114,7 +114,7 @@ public void testCreateStateIndexAndAliasIfNecessary_CleanState() {
 
         CreateIndexRequest createRequest = createRequestCaptor.getValue();
         assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
-        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
+        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true))));
     }
 
     private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexName) {
@@ -157,7 +157,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPo
         assertThat(
             indicesAliasesRequest.getAliasActions(),
             contains(
-                AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX),
+                AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX).isHidden(true),
                 AliasActions.remove().alias(TEST_INDEX_ALIAS).index(LEGACY_INDEX_WITHOUT_SUFFIX)));
     }
 
@@ -175,7 +175,7 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> e
         IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
         assertThat(
             indicesAliasesRequest.getAliasActions(),
-            contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName)));
+            contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName).isHidden(true)));
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() {
@@ -205,7 +205,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLeg
 
         CreateIndexRequest createRequest = createRequestCaptor.getValue();
         assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
-        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
+        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true))));
     }
 
     public void testIndexNameComparator() {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
index ca0c99eb4b421..017c11f4b3b10 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -183,6 +184,7 @@ public void testDeleteExpiredData() throws Exception {
         retainAllSnapshots("snapshots-retention-with-retain");
 
         long totalModelSizeStatsBeforeDelete = client().prepareSearch("*")
+                .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
                 .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
                 .get().getHits().getTotalHits().value;
         long totalNotificationsCountBeforeDelete =
@@ -231,6 +233,7 @@ public void testDeleteExpiredData() throws Exception {
         assertThat(getModelSnapshots("results-and-snapshots-retention").size(), equalTo(1));
 
         long totalModelSizeStatsAfterDelete = client().prepareSearch("*")
+                .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
                 .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
                 .get().getHits().getTotalHits().value;
         long totalNotificationsCountAfterDelete =
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
index bc22667ef9418..62b05bbcd81f4 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
@@ -194,15 +194,17 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         // appear immediately so wait here.
         assertBusy(() -> {
             try {
-                String aliasesResponse = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
-                assertThat(aliasesResponse,
-                        containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{"));
+                String aliasesResponse = getAliases();
+                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName)
+                        + "\":{\"aliases\":{"));
                 assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)
-                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}"));
-                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}"));
+                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}},\"is_hidden\":true}"));
+                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1)
+                        + "\":{\"is_hidden\":true}"));
                 assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)
-                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}"));
-                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}"));
+                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}},\"is_hidden\":true}"));
+                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2)
+                        + "\":{\"is_hidden\":true}"));
             } catch (ResponseException e) {
                 throw new AssertionError(e);
             }
@@ -270,7 +272,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1));
 
         // check that indices still exist, but no longer have job1 entries and aliases are gone
-        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
+        responseAsString = getAliases();
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
         assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
 
@@ -286,7 +288,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
 
         // Delete the second job and verify aliases are gone, and original concrete/custom index is gone
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2));
-        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
+        responseAsString = getAliases();
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
 
         refreshAllIndices();
@@ -626,6 +628,7 @@ public void testMultiIndexDelete() throws Exception {
         extraIndex1.setJsonEntity("{\n" +
             "    \"aliases\" : {\n" +
             "        \"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)+ "\" : {\n" +
+            "            \"is_hidden\" : true,\n" +
             "            \"filter\" : {\n" +
             "                \"term\" : {\"" + Job.ID + "\" : \"" + jobId + "\" }\n" +
             "            }\n" +
@@ -637,6 +640,7 @@ public void testMultiIndexDelete() throws Exception {
         extraIndex2.setJsonEntity("{\n" +
             "    \"aliases\" : {\n" +
             "        \"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)+ "\" : {\n" +
+            "            \"is_hidden\" : true,\n" +
             "            \"filter\" : {\n" +
             "                \"term\" : {\"" + Job.ID + "\" : \"" + jobId + "\" }\n" +
             "            }\n" +
@@ -784,6 +788,9 @@ public void testDelete_multipleRequest() throws Exception {
             assertNull(recreationException.get().getMessage(), recreationException.get());
         }
 
+        String expectedReadAliasString = "\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
+            + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}},\"is_hidden\":true}";
+        String expectedWriteAliasString = "\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{\"is_hidden\":true}";
         try {
             // The idea of the code above is that the deletion is sufficiently time-consuming that
             // all threads enter the deletion call before the first one exits it.  Usually this happens,
@@ -796,9 +803,8 @@ public void testDelete_multipleRequest() throws Exception {
             // if there's been a race between deletion and recreation these are what will be missing.
             String aliases = getAliases();
 
-            assertThat(aliases, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
-                    + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}}}"));
-            assertThat(aliases, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{}"));
+            assertThat(aliases, containsString(expectedReadAliasString));
+            assertThat(aliases, containsString(expectedWriteAliasString));
 
 
         } catch (ResponseException missingJobException) {
@@ -807,9 +813,8 @@ public void testDelete_multipleRequest() throws Exception {
 
             // The job aliases should be deleted
             String aliases = getAliases();
-            assertThat(aliases, not(containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
-                    + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}}}")));
-            assertThat(aliases, not(containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{}")));
+            assertThat(aliases, not(containsString(expectedReadAliasString)));
+            assertThat(aliases, not(containsString(expectedWriteAliasString)));
         }
 
         assertEquals(numThreads, recreationGuard.get());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
index bb47caed2349d..2a5d35ce7f26a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
@@ -489,7 +489,7 @@ private void deleteAliases(ParentTaskAssigningClient parentTaskClient, String jo
 
         // first find the concrete indices associated with the aliases
         GetAliasesRequest aliasesRequest = new GetAliasesRequest().aliases(readAliasName, writeAliasName)
-                .indicesOptions(IndicesOptions.lenientExpandOpen());
+                .indicesOptions(IndicesOptions.lenientExpandOpenHidden());
         executeAsyncWithOrigin(parentTaskClient.threadPool().getThreadContext(), ML_ORIGIN, aliasesRequest,
                 ActionListener.<GetAliasesResponse>wrap(
                         getAliasesResponse -> {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index 48e8fc9ecdaa2..1f332b5d538d1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -282,9 +282,16 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen
         final String indexName = tempIndexName;
 
         ActionListener<Boolean> indexAndMappingsListener = ActionListener.wrap(success -> {
-            final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
-                    .addAlias(indexName, readAliasName, QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId()))
-                    .addAlias(indexName, writeAliasName).request();
+            final IndicesAliasesRequest request =
+                client.admin().indices().prepareAliases()
+                    .addAliasAction(
+                        IndicesAliasesRequest.AliasActions.add()
+                            .index(indexName)
+                            .alias(readAliasName)
+                            .isHidden(true)
+                            .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())))
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(writeAliasName).isHidden(true))
+                    .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
                     ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(true), finalListener::onFailure),
                     client.admin().indices()::aliases);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
index 585bfd8545678..40494ce71d885 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
@@ -19,6 +19,8 @@
 
 import java.util.List;
 
+import static org.hamcrest.Matchers.is;
+
 public class AnnotationIndexIT extends MlSingleNodeTestCase {
 
     @Override
@@ -72,6 +74,9 @@ private int numberOfAnnotationsAliases() {
             .getAliases();
         if (aliases != null) {
             for (ObjectObjectCursor<String, List<AliasMetaData>> entry : aliases) {
+                for (AliasMetaData aliasMetaData : entry.value) {
+                    assertThat("Annotations aliases should be hidden but are not: " + aliases, aliasMetaData.isHidden(), is(true));
+                }
                 count += entry.value.size();
             }
         }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
index f7806f1be9758..5e0fb7ce03e70 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
@@ -142,8 +142,10 @@ public void testPutJob_CreatesResultsIndex() {
         assertThat(mappingProperties.keySet(), hasItem("by_field_1"));
 
         // Check aliases have been created
-        assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
-            AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())));
+        assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(
+            AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
+            AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())
+        ));
 
         // Now let's create a second job to test things work when the index exists already
         assertThat(mappingProperties.keySet(), not(hasItem("by_field_2")));
@@ -187,8 +189,10 @@ public void testPutJob_WithCustomResultsIndex() {
         assertThat(mappingProperties.keySet(), hasItem("by_field"));
 
         // Check aliases have been created
-        assertThat(getAliases(customIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
-            AnomalyDetectorsIndex.resultsWriteAlias(job.getId())));
+        assertThat(getAliases(customIndex), containsInAnyOrder(
+            AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
+            AnomalyDetectorsIndex.resultsWriteAlias(job.getId())
+        ));
     }
 
     @AwaitsFix(bugUrl ="https://github.com/elastic/elasticsearch/issues/40134")
@@ -366,12 +370,14 @@ private Map<String, Object> getIndexMappingProperties(String index) {
     }
 
     private Set<String> getAliases(String index) {
-        GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(
-            new GetAliasesRequest().indices(index)).actionGet();
+        GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(new GetAliasesRequest().indices(index)).actionGet();
         ImmutableOpenMap<String, List<AliasMetaData>> aliases = getAliasesResponse.getAliases();
         assertThat(aliases.containsKey(index), is(true));
-        List<AliasMetaData> aliasMetaData = aliases.get(index);
-        return aliasMetaData.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
+        List<AliasMetaData> aliasMetaDataList = aliases.get(index);
+        for (AliasMetaData aliasMetaData : aliasMetaDataList) {
+            assertThat("Anomalies aliases should be hidden but are not: " + aliases, aliasMetaData.isHidden(), is(true));
+        }
+        return aliasMetaDataList.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
     }
 
     private List<Calendar> getCalendars(String jobId) throws Exception {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
index bb8245a8e7eb4..f6bf53fb289db 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
@@ -148,6 +148,7 @@ setup:
 
   - do:
       search:
+        expand_wildcards: all
         rest_total_hits_as_int: true
         body: { query: { bool: { must: [
               { query_string: { query: "result_type:record"} },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
index 6774ae8b455cd..2b8a9063b4ad3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
@@ -100,6 +100,13 @@ setup:
         index: ".ml-state-000001"
   - is_true: ''
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.exists_alias:
+        name: ".ml-state-write"
+  - is_true: ''
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -439,6 +446,13 @@ setup:
         index: ".ml-state-000001"
   - is_true: ''
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.exists_alias:
+        name: ".ml-state-write"
+  - is_true: ''
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -681,6 +695,13 @@ setup:
         index: ".ml-state-000001"
   - is_true: ''
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.exists_alias:
+        name: ".ml-state-write"
+  - is_true: ''
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
index 445ab11b04288..7b365dbe92dc2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -70,7 +70,7 @@
   - do:
       indices.get_alias:
         name: ".ml-anomalies-.write-job-crud-test-apis"
-  - match: { \.ml-anomalies-shared.aliases.\.ml-anomalies-\.write-job-crud-test-apis: {} }
+  - match: { \.ml-anomalies-shared.aliases.\.ml-anomalies-\.write-job-crud-test-apis: { is_hidden: true } }
 
   - do:
       ml.delete_job:

From baccbec5f5a02611da6f7af56d2f09735dcfc25e Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Wed, 18 Mar 2020 10:36:16 +0200
Subject: [PATCH 090/285] Mute failing test (#53709)

see https://github.com/elastic/elasticsearch/issues/53708
---
 .../test/cluster.component_template/10_basic.yml              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
index 1ce79e3ca18c7..177678a3da81d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
@@ -1,8 +1,8 @@
 ---
 "Basic CRUD":
   - skip:
-      version: " - 7.99.99"
-      reason: not backported yet
+      version: "all"
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/53708"
 
   - do:
       cluster.put_component_template:

From 3d2417c2eaa871714895af712fd4a484350ef748 Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Wed, 18 Mar 2020 09:55:47 +0100
Subject: [PATCH 091/285] [Transform] re-enable bwc (#53707)

re-enable bwc after #53695 has been merged
---
 build.gradle                                                  | 4 ++--
 .../xpack/core/transform/action/PreviewTransformAction.java   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.gradle b/build.gradle
index a151683b4847d..2dbf94ae7dabb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53695" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
index 2515429d6af0a..ba38938236637 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
@@ -180,7 +180,7 @@ public Response(StreamInput in) throws IOException {
             for (int i = 0; i < size; i++) {
                 this.docs.add(in.readMap());
             }
-            if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_7_0
+            if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
                 this.generatedDestIndexSettings = new TransformDestIndexSettings(in);
             } else if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
                 Map<String, Object> objectMap = in.readMap();
@@ -204,7 +204,7 @@ public void writeTo(StreamOutput out) throws IOException {
             for (Map<String, Object> doc : docs) {
                 out.writeMapWithConsistentOrder(doc);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_7_0
+            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                 generatedDestIndexSettings.writeTo(out);
             } else if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
                 out.writeMap(generatedDestIndexSettings.getMappings());

From f9cbc76908aa8a7464ed1fea155625da1c6140f4 Mon Sep 17 00:00:00 2001
From: Yash Joshi <jyash97@gmail.com>
Date: Wed, 18 Mar 2020 14:42:57 +0530
Subject: [PATCH 092/285] [Docs] Fix typo in range query (#53656)

---
 docs/reference/query-dsl/range-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index ea7e7e9529435..f98eef9ee0534 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -203,7 +203,7 @@ the entire month.
 `lte`::
 +
 --
-Rounds up to the lastest millisecond.
+Rounds up to the latest millisecond.
 
 For example, `2014-11-18||/M` rounds up to `2014-11-30T23:59:59.999`, including
 the entire month.

From fe5092ae24558ed5a6ef3688a32dca3f705a4a1b Mon Sep 17 00:00:00 2001
From: Tianlun Li <shawnli1014@gmail.com>
Date: Wed, 18 Mar 2020 03:03:21 -0700
Subject: [PATCH 093/285] Deprecate delaying state recovery for master nodes
 (#53646)

It is useful to be able to delay state recovery until enough data nodes have
joined the cluster, since this gives the shard allocator a decent opportunity
to re-use as much existing data as possible. However we also have the option to
delay state recovery until a certain number of master-eligible nodes have
joined, and this is unnecessary: we require a majority of master-eligible nodes
for state recovery, and there is no advantage in waiting for more.

This commit deprecates the unnecessary settings in preparation for their
removal.

Relates #51806
---
 docs/reference/modules/gateway.asciidoc       | 12 ++++++---
 .../elasticsearch/gateway/GatewayService.java |  8 +++---
 .../gateway/GatewayServiceTests.java          | 27 ++++++++++++-------
 3 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/docs/reference/modules/gateway.asciidoc b/docs/reference/modules/gateway.asciidoc
index 2b0783c9de0b0..c5bb5d1579c4f 100644
--- a/docs/reference/modules/gateway.asciidoc
+++ b/docs/reference/modules/gateway.asciidoc
@@ -10,12 +10,14 @@ recover the cluster state and the cluster's data:
 
 `gateway.expected_nodes`::
 
+    deprecated:[7.7.0, This setting will be removed in 8.0. You should use `gateway.expected_data_nodes` instead.]
     The number of (data or master) nodes that are expected to be in the cluster.
     Recovery of local shards will start as soon as the expected number of
     nodes have joined the cluster. Defaults to `0`
 
 `gateway.expected_master_nodes`::
 
+    deprecated:[7.7.0, This setting will be removed in 8.0. You should use `gateway.expected_data_nodes` instead.]
     The number of master nodes that are expected to be in the cluster.
     Recovery of local shards will start as soon as the expected number of
     master nodes have joined the cluster. Defaults to `0`
@@ -37,10 +39,12 @@ as long as the following conditions are met:
 
 `gateway.recover_after_nodes`::
 
+    deprecated:[7.7.0, This setting will be removed in 8.0. You should use `gateway.recover_after_data_nodes` instead.]
     Recover as long as this many data or master nodes have joined the cluster.
 
 `gateway.recover_after_master_nodes`::
 
+    deprecated:[7.7.0, This setting will be removed in 8.0. You should use `gateway.recover_after_data_nodes` instead.]
     Recover as long as this many master nodes have joined the cluster.
 
 `gateway.recover_after_data_nodes`::
@@ -53,8 +57,8 @@ NOTE: These settings only take effect on a full cluster restart.
 === Dangling indices
 
 When a node joins the cluster, any shards stored in its local data
-directory which do not already exist in the cluster will be imported into the 
-cluster. This functionality is intended as a best effort to help users who 
-lose all master nodes. If a new master node is started which is unaware of 
-the other indices in the cluster, adding the old nodes will cause the old 
+directory which do not already exist in the cluster will be imported into the
+cluster. This functionality is intended as a best effort to help users who
+lose all master nodes. If a new master node is started which is unaware of
+the other indices in the cluster, adding the old nodes will cause the old
 indices to be imported, instead of being deleted.
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 86f05f398d1f8..5219988237850 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -51,19 +51,19 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
     private static final Logger logger = LogManager.getLogger(GatewayService.class);
 
     public static final Setting<Integer> EXPECTED_NODES_SETTING =
-        Setting.intSetting("gateway.expected_nodes", -1, -1, Property.NodeScope);
+        Setting.intSetting("gateway.expected_nodes", -1, -1, Property.NodeScope, Property.Deprecated);
     public static final Setting<Integer> EXPECTED_DATA_NODES_SETTING =
         Setting.intSetting("gateway.expected_data_nodes", -1, -1, Property.NodeScope);
     public static final Setting<Integer> EXPECTED_MASTER_NODES_SETTING =
-        Setting.intSetting("gateway.expected_master_nodes", -1, -1, Property.NodeScope);
+        Setting.intSetting("gateway.expected_master_nodes", -1, -1, Property.NodeScope, Property.Deprecated);
     public static final Setting<TimeValue> RECOVER_AFTER_TIME_SETTING =
         Setting.positiveTimeSetting("gateway.recover_after_time", TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final Setting<Integer> RECOVER_AFTER_NODES_SETTING =
-        Setting.intSetting("gateway.recover_after_nodes", -1, -1, Property.NodeScope);
+        Setting.intSetting("gateway.recover_after_nodes", -1, -1, Property.NodeScope, Property.Deprecated);
     public static final Setting<Integer> RECOVER_AFTER_DATA_NODES_SETTING =
         Setting.intSetting("gateway.recover_after_data_nodes", -1, -1, Property.NodeScope);
     public static final Setting<Integer> RECOVER_AFTER_MASTER_NODES_SETTING =
-        Setting.intSetting("gateway.recover_after_master_nodes", 0, 0, Property.NodeScope);
+        Setting.intSetting("gateway.recover_after_master_nodes", 0, 0, Property.NodeScope, Property.Deprecated);
 
     public static final ClusterBlock STATE_NOT_RECOVERED_BLOCK = new ClusterBlock(1, "state not recovered / initialized", true, true,
         false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 4049cec796102..41c6dc6f72ae7 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
@@ -40,24 +41,32 @@ public void testDefaultRecoverAfterTime() {
         GatewayService service = createService(Settings.builder());
         assertNull(service.recoverAfterTime());
 
-        // ensure default is set when setting expected_nodes
-        service = createService(Settings.builder().put("gateway.expected_nodes", 1));
-        assertThat(service.recoverAfterTime(), Matchers.equalTo(GatewayService.DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET));
-
         // ensure default is set when setting expected_data_nodes
         service = createService(Settings.builder().put("gateway.expected_data_nodes", 1));
         assertThat(service.recoverAfterTime(), Matchers.equalTo(GatewayService.DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET));
 
-        // ensure default is set when setting expected_master_nodes
-        service = createService(Settings.builder().put("gateway.expected_master_nodes", 1));
-        assertThat(service.recoverAfterTime(), Matchers.equalTo(GatewayService.DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET));
-
         // ensure settings override default
         final TimeValue timeValue = TimeValue.timeValueHours(3);
         // ensure default is set when setting expected_nodes
-        service = createService(Settings.builder().put("gateway.expected_nodes", 1).put("gateway.recover_after_time",
+        service = createService(Settings.builder().put("gateway.recover_after_time",
             timeValue.toString()));
         assertThat(service.recoverAfterTime().millis(), Matchers.equalTo(timeValue.millis()));
     }
 
+    public void testDeprecatedSettings() {
+        GatewayService service = createService(Settings.builder());
+
+        service = createService(Settings.builder().put("gateway.expected_nodes", 1));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {GatewayService.EXPECTED_NODES_SETTING });
+
+        service = createService(Settings.builder().put("gateway.expected_master_nodes", 1));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {GatewayService.EXPECTED_MASTER_NODES_SETTING });
+
+        service = createService(Settings.builder().put("gateway.recover_after_nodes", 1));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {GatewayService.RECOVER_AFTER_NODES_SETTING });
+
+        service = createService(Settings.builder().put("gateway.recover_after_master_nodes", 1));
+        assertSettingDeprecationsAndWarnings(new Setting<?>[] {GatewayService.RECOVER_AFTER_MASTER_NODES_SETTING });
+    }
+
 }

From 2a44c2922262870cb7067eda33ebfa47e80b94c5 Mon Sep 17 00:00:00 2001
From: Ralph Ursprung <39383228+rursprung@users.noreply.github.com>
Date: Wed, 18 Mar 2020 11:52:50 +0100
Subject: [PATCH 094/285] [Docs] Fix highlighting in match-query example
 (#52426)

---
 docs/reference/query-dsl/match-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index a2c912bf8aa48..ecd8d3dbca0c1 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -258,7 +258,7 @@ GET /_search
 
 The `match` query supports multi-terms synonym expansion with the <<analysis-synonym-graph-tokenfilter,
 synonym_graph>> token filter. When this filter is used, the parser creates a phrase query for each multi-terms synonyms.
-For example, the following synonym: `"ny, new york" would produce:`
+For example, the following synonym: `"ny, new york"` would produce:
 
 `(ny OR ("new york"))`
 

From 0411627efa2da64e30c775edc74458a49e409bab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Wed, 18 Mar 2020 11:55:14 +0100
Subject: [PATCH 095/285] Fix ReloadSynonymAnalyzerIT failure (#53663)

There is an assertion in ReloadAnalyzersResponse.merge that compares index names
of merged responses that was falsely using object equality instead of
String.equals(). In the past this didn't seem to matter but with changes in the
test setup we started to see failures. Correcting this and also simplifying test
a bit to be able to run it repeatedly if needed.

Closes #53443
---
 .../xpack/core/action/ReloadAnalyzersResponse.java          | 2 +-
 .../xpack/core/rest/action/ReloadSynonymAnalyzerIT.java     | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java
index fd84ba2d8df77..378aed90eed04 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java
@@ -164,7 +164,7 @@ public Set<String> getReloadedAnalyzers() {
         }
 
         void merge(ReloadResult other) {
-            assert this.indexName == other.index;
+            assert this.indexName.equals(other.index);
             this.reloadedAnalyzers.addAll(other.reloadedSearchAnalyzers);
             this.reloadedIndicesNodes.add(other.nodeId);
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
index 37cd500bf31b2..1d5dd3b212432 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.rest.action;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.Response;
 import org.elasticsearch.action.search.SearchResponse;
@@ -40,7 +39,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53443")
 public class ReloadSynonymAnalyzerIT extends ESIntegTestCase {
 
     @Override
@@ -66,10 +64,8 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException,
         Path config = internalCluster().getInstance(Environment.class).configFile();
         String synonymsFileName = "synonyms.txt";
         Path synonymsFile = config.resolve(synonymsFileName);
-        Files.createFile(synonymsFile);
-        assertTrue(Files.exists(synonymsFile));
         try (PrintWriter out = new PrintWriter(
-                new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.CREATE), StandardCharsets.UTF_8))) {
+                new OutputStreamWriter(Files.newOutputStream(synonymsFile), StandardCharsets.UTF_8))) {
             out.println("foo, baz");
         }
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()

From 795a92707f178ec7b7047fa8af2711eda663d89a Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Wed, 18 Mar 2020 11:50:36 +0000
Subject: [PATCH 096/285] Remove deprecation warning when doc scripts refer to
 '_type' field (#53605)

We currently emit a warning in 8x when a script refers to the _type field for
a document. However, in 8x this field no longer exists, so the deprecation
warning is not required.

Relates to #41059
---
 docs/reference/mapping/fields/type-field.asciidoc    |  1 -
 .../elasticsearch/search/lookup/LeafDocLookup.java   | 12 ------------
 2 files changed, 13 deletions(-)

diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index 2c5dc7195d643..044729b84eece 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -57,7 +57,6 @@ GET my_index/_search
 
 --------------------------
 // TEST[warning:[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead.]
-// TEST[warning:[types removal] Looking up doc types [_type] in scripts is deprecated.]
 
 <1> Querying on the `_type` field
 <2> Aggregating on the `_type` field
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
index c6f6cc8a0620e..9ec969b85c39e 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
@@ -18,10 +18,8 @@
  */
 package org.elasticsearch.search.lookup;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -38,12 +36,6 @@
 
 public class LeafDocLookup implements Map<String, ScriptDocValues<?>> {
 
-    private static final DeprecationLogger DEPRECATION_LOGGER
-            = new DeprecationLogger(LogManager.getLogger(LeafDocLookup.class));
-    static final String TYPES_DEPRECATION_KEY = "type-field-doc-lookup";
-    static final String TYPES_DEPRECATION_MESSAGE =
-            "[types removal] Looking up doc types [_type] in scripts is deprecated.";
-
     private final Map<String, ScriptDocValues<?>> localCacheFieldData = new HashMap<>(4);
 
     private final MapperService mapperService;
@@ -74,10 +66,6 @@ public void setDocument(int docId) {
 
     @Override
     public ScriptDocValues<?> get(Object key) {
-        // deprecate _type
-        if ("_type".equals(key)) {
-            DEPRECATION_LOGGER.deprecatedAndMaybeLog(TYPES_DEPRECATION_KEY, TYPES_DEPRECATION_MESSAGE);
-        }
         // assume its a string...
         String fieldName = key.toString();
         ScriptDocValues<?> scriptValues = localCacheFieldData.get(fieldName);

From e1096b9457cd2adfc3992b1533bf9eef3cad5de7 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Wed, 18 Mar 2020 13:23:45 +0100
Subject: [PATCH 097/285] Restore off-heap loading for term dictionary in
 ReadOnlyEngine (#53713)

This is a partial restore of #43158, following decision taken in #51247

Closes #51247
---
 .../java/org/elasticsearch/index/engine/ReadOnlyEngine.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index 62957e2c1c2c3..b675ba24629c8 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -65,10 +65,10 @@ public class ReadOnlyEngine extends Engine {
 
     /**
      * Reader attributes used for read only engines. These attributes prevent loading term dictionaries on-heap even if the field is an
-     * ID field if we are reading form memory maps.
+     * ID field.
      */
     private static final Map<String, String> OFF_HEAP_READER_ATTRIBUTES = Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY,
-        BlockTreeTermsReader.FSTLoadMode.AUTO.name());
+        BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name());
     private final SegmentInfos lastCommittedSegmentInfos;
     private final SeqNoStats seqNoStats;
     private final ElasticsearchReaderManager readerManager;

From 3a39ed0055c82d3fc7bd343912b435840163c70d Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Wed, 18 Mar 2020 08:33:20 -0400
Subject: [PATCH 098/285] [DOCS] Remove `light_bengali` stemmer (#53697)

Only the `bengali` stemmer is available in Lucene and surfaced through
Elasticsearch. This removes the incorrect `light_bengali` link in our
docs.
---
 .../reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
index 4e98e24d08ef0..957aad084619f 100644
--- a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
@@ -54,7 +54,6 @@ http://snowball.tartarus.org/algorithms/basque/stemmer.html[*`basque`*]
 
 Bengali::
 http://www.tandfonline.com/doi/abs/10.1080/02564602.1993.11437284[*`bengali`*]
-http://members.unine.ch/jacques.savoy/clef/BengaliStemmerLight.java.txt[*`light_bengali`*]
 
 Brazilian Portuguese::
 

From dfb779ec8d94ab0aec078995f5dfa6ce82695384 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Wed, 18 Mar 2020 14:50:10 +0200
Subject: [PATCH 099/285] Mute testHistoryIsWrittenWithDeletion (#53719)

see #53718
---
 .../elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 3c53b78e4523b..14c99d471e983 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -1412,6 +1412,7 @@ public void testHistoryIsWrittenWithFailure() throws Exception {
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-1", false, "ERROR"), 30, TimeUnit.SECONDS);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53718")
     public void testHistoryIsWrittenWithDeletion() throws Exception {
         // Index should be created and then deleted by ILM
         createIndexWithSettings(index, Settings.builder(), false);

From de9e44fb681522314c6d030a1a74850019205c05 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 18 Mar 2020 14:14:54 +0100
Subject: [PATCH 100/285] Adapt serialization version checks in
 ShardSearchRequest (#53660)

This change adapts the serialization checks to 7.7.0 in order to cope with #53659.
Note that this commit also disables the bwc tests temporarily in order to be able to
merge #53659 first.

Relates #51852
---
 build.gradle                                               | 4 ++--
 .../elasticsearch/search/internal/ShardSearchRequest.java  | 7 +------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/build.gradle b/build.gradle
index 2dbf94ae7dabb..b83f19efbba42 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53659" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 6c1fc7e959b40..96d0d67812d3c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -179,12 +179,9 @@ public ShardSearchRequest(StreamInput in) throws IOException {
         preference = in.readOptionalString();
         if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
             canReturnNullResponseIfMatchNoDocs = in.readBoolean();
-        } else {
-            canReturnNullResponseIfMatchNoDocs = false;
-        }
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
             bottomSortValues = in.readOptionalWriteable(SearchSortValuesAndFormats::new);
         } else {
+            canReturnNullResponseIfMatchNoDocs = false;
             bottomSortValues = null;
         }
         originalIndices = OriginalIndices.readOriginalIndices(in);
@@ -223,8 +220,6 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
         }
         if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
             out.writeBoolean(canReturnNullResponseIfMatchNoDocs);
-        }
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
             out.writeOptionalWriteable(bottomSortValues);
         }
     }

From 110ff6cdd1bc65aa7f9005879711a2c5edef4fc4 Mon Sep 17 00:00:00 2001
From: zacharymorn <zacharymorn@gmail.com>
Date: Wed, 18 Mar 2020 06:16:26 -0700
Subject: [PATCH 101/285] Create GET _cat/transforms API Issue (#53643)

Adds new` _cat/transform` and `_cat/transform/{transform_id}` endpoints.
---
 .../table}/TableColumnAttributeBuilder.java   |   2 +-
 .../cat/RestCatDataFrameAnalyticsAction.java  |   1 +
 .../ml/rest/cat/RestCatDatafeedsAction.java   |   1 +
 .../xpack/ml/rest/cat/RestCatJobsAction.java  |   1 +
 .../rest/cat/RestCatTrainedModelsAction.java  |   1 +
 .../api/transform.cat_transform.json          |  82 ++++++
 .../test/transform/transforms_cat_apis.yml    | 138 ++++++++++
 .../xpack/transform/Transform.java            |   2 +
 .../rest/action/RestCatTransformAction.java   | 254 ++++++++++++++++++
 9 files changed, 481 insertions(+), 1 deletion(-)
 rename x-pack/plugin/{ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat => core/src/main/java/org/elasticsearch/xpack/core/common/table}/TableColumnAttributeBuilder.java (98%)
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
 create mode 100644 x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/TableColumnAttributeBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/table/TableColumnAttributeBuilder.java
similarity index 98%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/TableColumnAttributeBuilder.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/table/TableColumnAttributeBuilder.java
index 96a7c8be8bf36..f50ebbb9a1515 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/TableColumnAttributeBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/table/TableColumnAttributeBuilder.java
@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.rest.cat;
 
+package org.elasticsearch.xpack.core.common.table;
 
 import org.elasticsearch.common.Strings;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDataFrameAnalyticsAction.java
index 9d2294cea03fa..dadeaa0b7be9f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDataFrameAnalyticsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestActionListener;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java
index e29098cb49031..a78dc6f6cfb42 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java
index 9d67a93af5ce8..888f9df6a6e2c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestRequest;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
index 79bb08c25d0ba..8649d0a42e86b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestRequest;
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
new file mode 100644
index 0000000000000..26a7be322aa3c
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
@@ -0,0 +1,82 @@
+{
+  "transform.cat_transform":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-transform.html"
+    },
+    "stability":"stable",
+    "url":{
+      "paths":[
+        {
+          "path":"/_cat/transform",
+          "methods":[
+            "GET"
+          ]
+        },
+        {
+          "path":"/_cat/transform/{transform_id}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "transform_id":{
+              "type":"string",
+              "description":"The id of the transform for which to get stats. '_all' or '*' implies all transforms"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "from":{
+        "type":"int",
+        "required":false,
+        "description":"skips a number of transform configs, defaults to 0"
+      },
+      "size":{
+        "type":"int",
+        "required":false,
+        "description":"specifies a max number of transforms to get, defaults to 100"
+      },
+      "allow_no_match":{
+        "type":"boolean",
+        "required":false,
+        "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)"
+      },
+      "format":{
+        "type":"string",
+        "description":"a short version of the Accept header, e.g. json, yaml"
+      },
+      "h":{
+        "type":"list",
+        "description":"Comma-separated list of column names to display"
+      },
+      "help":{
+        "type":"boolean",
+        "description":"Return help information",
+        "default":false
+      },
+      "s":{
+        "type":"list",
+        "description":"Comma-separated list of column names or column aliases to sort by"
+      },
+      "time":{
+        "type":"enum",
+        "description":"The unit in which to display time values",
+        "options":[
+          "d (Days)",
+          "h (Hours)",
+          "m (Minutes)",
+          "s (Seconds)",
+          "ms (Milliseconds)",
+          "micros (Microseconds)",
+          "nanos (Nanoseconds)"
+        ]
+      },
+      "v":{
+        "type":"boolean",
+        "description":"Verbose mode. Display column headers",
+        "default":false
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
new file mode 100644
index 0000000000000..37517b2c822f6
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
@@ -0,0 +1,138 @@
+setup:
+  - skip:
+      features: headers
+  - do:
+      indices.create:
+        index: airline-data
+        body:
+          mappings:
+            properties:
+              time:
+                type: date
+              airline:
+                type: keyword
+              responsetime:
+                type: float
+              event_rate:
+                type: integer
+  - do:
+      indices.create:
+        index: airline-data-other
+        body:
+          mappings:
+            properties:
+              time:
+                type: date
+              airline:
+                type: keyword
+              responsetime:
+                type: float
+              event_rate:
+                type: integer
+  - do:
+      transform.put_transform:
+        transform_id: "airline-transform-stats"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-by-airline" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+
+---
+teardown:
+  - do:
+      transform.delete_transform:
+        transform_id: "airline-transform-stats"
+
+---
+"Test cat transform stats hiding headers":
+  - do:
+      transform.cat_transform:
+        transform_id: "airline-transform-stats"
+  - match:
+      $body: |
+        /^  #id                           \s+ create_time \s+ version \s+ source_index \s+ dest_index              \s+ pipeline \s+ transform_type \s+ frequency \s+ max_page_search_size \s+ state    \n
+            (airline\-transform\-stats    \s+ [^\s]+      \s+ [^\s]+  \s+ airline-data \s+ airline-data-by-airline \s+          \s+ batch          \s+ 1m        \s+ 500                  \s+ STOPPED \n)+  $/
+
+---
+"Test cat transform stats with column selection":
+  - do:
+      transform.cat_transform:
+        transform_id: "airline-transform-stats"
+        v: true
+        h: id,version,source_index,dest_index,search_total,index_total,dt,cdtea,indexed_documents_exp_avg
+  - match:
+      $body: |
+        /^   id                       \s+ version \s+ source_index \s+ dest_index              \s+ search_total \s+ index_total \s+ dt \s+ cdtea  \s+ indexed_documents_exp_avg \n
+            (airline\-transform-stats \s+ [^\s]+  \s+ airline-data \s+ airline-data-by-airline \s+ 0            \s+ 0           \s+ 0  \s+ 0.0    \s+ 0.0 \n)+  $/
+
+
+---
+"Test cat transform stats with batch transform":
+  - do:
+      transform.put_transform:
+        transform_id: "airline-transform-batch"
+        body: >
+          {
+            "source": {
+              "index": ["airline-data", "airline-data-other"],
+              "query": {"bool":{"filter":{"term":{"airline":"foo"}}}}
+            },
+            "dest": { "index": "airline-data-by-airline-batch" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "description": "description"
+          }
+  - do:
+      transform.cat_transform:
+        transform_id: "airline-transform-batch"
+        v: true
+  - match:
+      $body: |
+        /^   id                        \s+ create_time \s+ version \s+ source_index                    \s+ dest_index                    \s+ pipeline \s+ description \s+ transform_type \s+ frequency \s+ max_page_search_size \s+ state   \n
+            (airline\-transform\-batch \s+ [^\s]+      \s+ [^\s]+  \s+ airline-data,airline-data-other \s+ airline-data-by-airline-batch \s+          \s+ description \s+ batch          \s+ 1m        \s+ 500                  \s+ STOPPED \n)+  $/
+  - do:
+      transform.delete_transform:
+        transform_id: "airline-transform-batch"
+
+---
+"Test cat transform stats with continuous transform":
+  - do:
+      transform.put_transform:
+        transform_id: "airline-transform-continuous"
+        body: >
+          {
+            "source": {
+              "index": ["airline-data", "airline-data-other"],
+              "query": {"bool":{"filter":{"term":{"airline":"foo"}}}}
+            },
+            "dest": { "index": "airline-data-by-airline-continuous" },
+            "frequency": "10s",
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "description": "description",
+            "sync": {
+              "time": {
+                "field": "time"
+              }
+            }
+          }
+  - do:
+      transform.cat_transform:
+        transform_id: "airline-transform-continuous"
+        v: true
+  - match:
+      $body: |
+        /^   id                             \s+ create_time \s+ version \s+ source_index                    \s+ dest_index                         \s+ pipeline \s+ description \s+ transform_type \s+ frequency \s+ max_page_search_size \s+ state   \n
+            (airline\-transform\-continuous \s+ [^\s]+      \s+ [^\s]+  \s+ airline-data,airline-data-other \s+ airline-data-by-airline-continuous \s+          \s+ description \s+ continuous     \s+ 10s       \s+ 500                  \s+ STOPPED \n)+  $/
+  - do:
+      transform.delete_transform:
+        transform_id: "airline-transform-continuous"
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
index e92bc80f338f1..f94237c9a5753 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
@@ -88,6 +88,7 @@
 import org.elasticsearch.xpack.transform.persistence.IndexBasedTransformConfigManager;
 import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 import org.elasticsearch.xpack.transform.persistence.TransformInternalIndex;
+import org.elasticsearch.xpack.transform.rest.action.RestCatTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestDeleteTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestGetTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestGetTransformStatsAction;
@@ -201,6 +202,7 @@ public List<RestHandler> getRestHandlers(
             new RestGetTransformStatsAction(),
             new RestPreviewTransformAction(),
             new RestUpdateTransformAction(),
+            new RestCatTransformAction(),
 
             // deprecated endpoints, to be removed for 8.0.0
             new RestPutTransformActionDeprecated(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
new file mode 100644
index 0000000000000..7a5f669bbcb4d
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
@@ -0,0 +1,254 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.transform.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.common.table.TableColumnAttributeBuilder;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.action.RestActionListener;
+import org.elasticsearch.rest.action.RestResponseListener;
+import org.elasticsearch.rest.action.cat.AbstractCatAction;
+import org.elasticsearch.rest.action.cat.RestTable;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.action.GetTransformAction;
+import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction;
+import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
+import org.elasticsearch.xpack.core.transform.transforms.TransformStats;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH;
+
+public class RestCatTransformAction extends AbstractCatAction {
+
+    private static final Integer DEFAULT_MAX_PAGE_SEARCH_SIZE = Integer.valueOf(500);
+    private static final TimeValue DEFAULT_TRANSFORM_FREQUENCY = TimeValue.timeValueMillis(60000);
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(GET, "_cat/transform"),
+            new Route(GET, "_cat/transform/{" + TransformField.TRANSFORM_ID + "}"));
+    }
+
+    @Override
+    public String getName() {
+        return "cat_transform_action";
+    }
+
+    @Override
+    protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient client) {
+        String id = restRequest.param(TransformField.TRANSFORM_ID);
+        if (Strings.isNullOrEmpty(id)) {
+            id = MetaData.ALL;
+        }
+
+        GetTransformAction.Request request = new GetTransformAction.Request(id);
+        request.setAllowNoResources(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
+
+        GetTransformStatsAction.Request statsRequest = new GetTransformStatsAction.Request(id);
+        statsRequest.setAllowNoMatch(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
+
+        if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
+            PageParams pageParams = new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
+                                                    restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE));
+            request.setPageParams(pageParams);
+            statsRequest.setPageParams(pageParams);
+        }
+
+        return channel -> client.execute(GetTransformAction.INSTANCE, request, new RestActionListener<>(channel) {
+            @Override
+            public void processResponse(GetTransformAction.Response response) {
+                client.execute(GetTransformStatsAction.INSTANCE, statsRequest, new RestResponseListener<>(channel) {
+                    @Override
+                    public RestResponse buildResponse(GetTransformStatsAction.Response statsResponse) throws Exception {
+                        return RestTable.buildResponse(buildTable(response, statsResponse), channel);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    protected void documentation(StringBuilder sb) {
+        sb.append("/_cat/transform\n");
+        sb.append("/_cat/transform/{" + TransformField.TRANSFORM_ID + "}\n");
+    }
+
+    @Override
+    protected Table getTableWithHeader(RestRequest unused) {
+        return getTableWithHeader();
+    }
+
+    private static Table getTableWithHeader() {
+        return new Table()
+            .startHeaders()
+            // Transform config info
+            .addCell("id", TableColumnAttributeBuilder.builder("the id").build())
+            .addCell("create_time",
+                TableColumnAttributeBuilder.builder("transform creation time")
+                    .setAliases("ct", "createTime")
+                    .build())
+            .addCell("version",
+                TableColumnAttributeBuilder.builder("the version of Elasticsearch when the transform was created")
+                    .setAliases("v")
+                    .build())
+            .addCell("source_index",
+                TableColumnAttributeBuilder.builder("source index")
+                    .setAliases("si", "sourceIndex")
+                    .build())
+            .addCell("dest_index",
+                TableColumnAttributeBuilder.builder("destination index")
+                    .setAliases("di", "destIndex")
+                    .build())
+            .addCell("pipeline",
+                TableColumnAttributeBuilder.builder("transform pipeline")
+                    .setAliases("p")
+                    .build())
+            .addCell("description",
+                TableColumnAttributeBuilder.builder("description")
+                    .setAliases("d")
+                    .build())
+            .addCell("transform_type",
+                TableColumnAttributeBuilder.builder("batch or continuous transform")
+                    .setAliases("tt")
+                    .build())
+            .addCell("frequency",
+                TableColumnAttributeBuilder.builder("frequency of transform")
+                    .setAliases("f")
+                    .build())
+            .addCell("max_page_search_size",
+                TableColumnAttributeBuilder.builder("max page search size ")
+                    .setAliases("mpsz")
+                    .build())
+
+            // Transform stats info
+            .addCell("state",
+                TableColumnAttributeBuilder.builder("transform state")
+                    .setAliases("s")
+                    .setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
+                    .build())
+            .addCell("reason",
+                TableColumnAttributeBuilder.builder("reason", false)
+                    .setAliases("r", "reason")
+                    .build())
+            .addCell("changes_last_detection_time",
+                TableColumnAttributeBuilder.builder("changes last detected time", false)
+                    .setAliases("cldt")
+                    .build())
+            .addCell("search_total",
+                TableColumnAttributeBuilder.builder("total number of searches", false)
+                    .setAliases("st")
+                    .build())
+            .addCell("search_failure",
+                TableColumnAttributeBuilder.builder("total number of search failures", false)
+                    .setAliases("sf")
+                    .build())
+            .addCell("search_time",
+                TableColumnAttributeBuilder.builder("search time", false)
+                    .setAliases("stime")
+                    .build())
+            .addCell("index_total",
+                TableColumnAttributeBuilder.builder("total number of indices", false)
+                    .setAliases("it")
+                    .build())
+            .addCell("index_failure",
+                TableColumnAttributeBuilder.builder("total number of index failures", false)
+                    .setAliases("if")
+                    .build())
+            .addCell("index_time",
+                TableColumnAttributeBuilder.builder("index time", false)
+                    .setAliases("itime")
+                    .build())
+            .addCell("document_total",
+                TableColumnAttributeBuilder.builder("total number of documents", false)
+                    .setAliases("dt")
+                    .build())
+            .addCell("invocation_total",
+                TableColumnAttributeBuilder.builder("total number of invocations", false)
+                    .setAliases("itotal")
+                    .build())
+            .addCell("page_total",
+                TableColumnAttributeBuilder.builder("total number of pages", false)
+                    .setAliases("pt")
+                    .build())
+            .addCell("checkpoint_duration_time_exp_avg",
+                TableColumnAttributeBuilder.builder("exponential average checkpoint processing time (milliseconds)", false)
+                    .setAliases("cdtea", "checkpointTimeExpAvg")
+                    .build())
+            .addCell("indexed_documents_exp_avg",
+                TableColumnAttributeBuilder.builder("exponential average number of documents indexed", false)
+                    .setAliases("idea")
+                    .build())
+            .addCell("processed_documents_exp_avg",
+                TableColumnAttributeBuilder.builder("exponential average number of documents processed", false)
+                    .setAliases("pdea")
+                    .build())
+            .endHeaders();
+    }
+
+    private Table buildTable(GetTransformAction.Response response, GetTransformStatsAction.Response statsResponse) {
+        Table table = getTableWithHeader();
+        Map<String, TransformStats> statsById = statsResponse.getTransformsStats().stream()
+                                                                .collect(Collectors.toMap(TransformStats::getId, Function.identity()));
+        response.getTransformConfigurations().forEach(config -> {
+            TransformStats stats = statsById.get(config.getId());
+            TransformCheckpointingInfo checkpointingInfo = null;
+            TransformIndexerStats transformIndexerStats = null;
+
+            if(stats != null) {
+                checkpointingInfo = stats.getCheckpointingInfo();
+                transformIndexerStats = stats.getIndexerStats();
+            }
+
+            table
+                .startRow()
+                .addCell(config.getId())
+                .addCell(config.getCreateTime())
+                .addCell(config.getVersion())
+                .addCell(String.join(",", config.getSource().getIndex()))
+                .addCell(config.getDestination().getIndex())
+                .addCell(config.getDestination().getPipeline())
+                .addCell(config.getDescription())
+                .addCell(config.getSyncConfig() == null ? "batch" : "continuous")
+                .addCell(config.getFrequency() == null ? DEFAULT_TRANSFORM_FREQUENCY : config.getFrequency())
+                .addCell(config.getPivotConfig() == null || config.getPivotConfig().getMaxPageSearchSize() == null ?
+                            DEFAULT_MAX_PAGE_SEARCH_SIZE : config.getPivotConfig().getMaxPageSearchSize())
+                .addCell(stats == null ? null : stats.getState())
+                .addCell(stats == null ? null : stats.getReason())
+                .addCell(checkpointingInfo == null ? null : checkpointingInfo.getChangesLastDetectedAt())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getSearchTotal())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getSearchFailures())
+                .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getSearchTime()))
+
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getIndexTotal())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getIndexFailures())
+                .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getIndexTime()))
+
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumDocuments())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumInvocations())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumPages())
+
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getExpAvgCheckpointDurationMs())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getExpAvgDocumentsIndexed())
+                .addCell(transformIndexerStats == null ? null : transformIndexerStats.getExpAvgDocumentsProcessed())
+                .endRow();
+        });
+
+        return table;
+    }
+}

From f2a2dcbb8a595546e0935428996fcf54245f7a1d Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Wed, 18 Mar 2020 09:42:25 -0400
Subject: [PATCH 102/285] [DOCS] Streamline `analyzer` mapping parm def
 (#51874)

Simplifies the `analyzer` mapping parameter definition to remove
duplicated analysis content and examples.
---
 .../mapping/params/analyzer.asciidoc          | 92 ++++---------------
 1 file changed, 17 insertions(+), 75 deletions(-)

diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index 02f10363c44ea..47f0a5d68ec34 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -1,81 +1,23 @@
 [[analyzer]]
 === `analyzer`
 
-The values of <<text,`text`>> fields are passed through an
-<<analysis,analyzer>> to convert the string into a stream of _tokens_ or
-_terms_.  For instance, the string `"The quick Brown Foxes."` may, depending
-on which analyzer is used,  be analyzed to the tokens: `quick`, `brown`,
-`fox`.  These are the actual terms that are indexed for the field, which makes
-it possible to search efficiently for individual words _within_  big blobs of
-text.
-
-This analysis process needs to happen not just at index time, but also at
-query time: the query string needs to be passed through the same (or a
-similar) analyzer so that the terms that it tries to find are in the same
-format as those that exist in the index.
-
-Elasticsearch ships with a number of <<analysis-analyzers,pre-defined analyzers>>,
-which can be used without further configuration.  It also ships with many
-<<analysis-charfilters,character filters>>, <<analysis-tokenizers,tokenizers>>,
-and <<analysis-tokenfilters>> which can be combined to configure
-custom analyzers per index.
-
-Analyzers can be specified per-query, per-field or per-index. At index time,
-Elasticsearch will look for an analyzer in this order:
-
-* The `analyzer` defined in the field mapping.
-* An analyzer named `default` in the index settings.
-* The <<analysis-standard-analyzer,`standard`>> analyzer.
-
-At query time, there are a few more layers:
-
-* The `analyzer` defined in a <<full-text-queries,full-text query>>.
-* The `search_analyzer` defined in the field mapping.
-* The `analyzer` defined in the field mapping.
-* An analyzer named `default_search` in the index settings.
-* An analyzer named `default` in the index settings.
-* The <<analysis-standard-analyzer,`standard`>> analyzer.
-
-The easiest way to specify an analyzer for a particular field is to define it
-in the field mapping, as follows:
-
-[source,console]
---------------------------------------------------
-PUT /my_index
-{
-  "mappings": {
-    "properties": {
-      "text": { <1>
-        "type": "text",
-        "fields": {
-          "english": { <2>
-            "type":     "text",
-            "analyzer": "english"
-          }
-        }
-      }
-    }
-  }
-}
-
-GET my_index/_analyze <3>
-{
-  "field": "text",
-  "text": "The quick Brown Foxes."
-}
-
-GET my_index/_analyze <4>
-{
-  "field": "text.english",
-  "text": "The quick Brown Foxes."
-}
---------------------------------------------------
-
-<1> The `text` field uses the default `standard` analyzer`.
-<2> The `text.english` <<multi-fields,multi-field>> uses the `english` analyzer, which removes stop words and applies stemming.
-<3> This returns the tokens: [ `the`, `quick`, `brown`, `foxes` ].
-<4> This returns the tokens: [ `quick`, `brown`, `fox` ].
-
+[IMPORTANT]
+====
+Only <<text,`text`>> fields support the `analyzer` mapping parameter.
+====
+
+The `analyzer` parameter specifies the <<analyzer-anatomy,analyzer>> used for
+<<analysis,text analysis>> when indexing or searching a `text` field.
+
+Unless overridden with the <<search-analyzer,`search_analyzer`>> mapping
+parameter, this analyzer is used for both <<analysis-index-search-time,index and
+search analysis>>. See <<specify-analyzer>>.
+
+[TIP]
+====
+We recommend testing analyzers before using them in production. See
+<<test-analyzer>>.
+====
 
 [[search-quote-analyzer]]
 ==== `search_quote_analyzer`

From 6b5fc35e4458e60a7ca5822584ec6a60562f2c01 Mon Sep 17 00:00:00 2001
From: Marios Trivyzas <matriv@gmail.com>
Date: Wed, 18 Mar 2020 14:52:03 +0100
Subject: [PATCH 103/285] Increase step between checks for cancellation
 (#53712)

The introduction of the ExitableDirectoryReader showed increase of
latencies for range queries using pointvalues.

Check for cancellation every 1024 docs instead of every 15 to lower
the impact of the check in query's performance.

Follows: #52822
Fixes: #53496
---
 .../elasticsearch/search/internal/ExitableDirectoryReader.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
index b66532bbd093e..51a94f4023b65 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
@@ -245,7 +245,7 @@ public int getDocCount() {
 
     private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor {
 
-        private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15
+        private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 10) - 1; // 1023
 
         private final PointValues.IntersectVisitor in;
         private final QueryCancellation queryCancellation;

From 9f0562be67254606fa748f313240c959785fe370 Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Wed, 18 Mar 2020 14:01:25 +0000
Subject: [PATCH 104/285] Fix highlighter support in PinnedQuery and added test
 (#53716)

Closes #53699
---
 .../lucene/search/CappedScoreQuery.java       |  5 ++++
 .../PinnedQueryBuilderIT.java                 | 27 +++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java
index 9a8f4fbe38bd1..ed8ede57b4d52 100644
--- a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java
@@ -33,6 +33,11 @@ public Query getQuery() {
         return query;
     }
 
+    @Override
+    public void visit(QueryVisitor visitor) {
+        query.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this));
+    }
+
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = query.rewrite(reader);
diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
index 6aa69a2325a8f..6867cddaf1530 100644
--- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
+++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
@@ -17,12 +17,15 @@
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -196,5 +199,29 @@ public void testExplain() throws Exception {
 
 
     }
+    
+    public void testHighlight() throws Exception {
+        // Issue raised in https://github.com/elastic/elasticsearch/issues/53699
+        assertAcked(prepareCreate("test").setMapping(
+                jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field1")
+                        .field("analyzer", "whitespace").field("type", "text").endObject().endObject().endObject().endObject()));
+        ensureGreen();
+        client().prepareIndex("test").setId("1").setSource("field1", "the quick brown fox").get();
+        refresh();
+
+        PinnedQueryBuilder pqb = new PinnedQueryBuilder(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR), "2");
+        
+        HighlightBuilder testHighlighter = new HighlightBuilder();
+        testHighlighter.field("field1");
 
+        SearchResponse searchResponse = client().prepareSearch().setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(pqb)
+                .highlighter(testHighlighter)
+                .setExplain(true).get();
+        assertHitCount(searchResponse, 1);
+        Map<String, HighlightField> highlights = searchResponse.getHits().getHits()[0].getHighlightFields();
+        assertThat(highlights.size(), equalTo(1));
+        HighlightField highlight = highlights.get("field1");
+        assertThat(highlight.fragments()[0].toString(), equalTo("<em>the</em> <em>quick</em> <em>brown</em> fox"));
+    }  
 }
+

From 3ef3cc571f3b1ef3f1a3a797b0147e2f571b8fac Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Wed, 18 Mar 2020 09:08:13 -0500
Subject: [PATCH 105/285] Add Watcher to available rest resources (#53620)

Prior to this commit Watcher explicitly copied test between two
projects with a copy task. This commit removes the explicit copy in favor
of adding the Watcher tests to the available restResources that may be
copied between projects.

This is how inter-project dependencies should be modeled. However, only
Watcher is included here since it is (currently) the only project with
inter-project test dependencies.

Note - this re-introduces: commit: 4f48e053f92b2ae7677a2e2ba0e5464ede7128d1
with some additional fixes.
---
 .../gradle/test/rest/CopyRestTestsTask.java   |  4 +--
 .../gradle/test/rest/RestResourcesPlugin.java | 27 ++++++++++++++-----
 rest-api-spec/build.gradle                    |  6 +----
 x-pack/plugin/build.gradle                    |  2 --
 x-pack/plugin/watcher/qa/rest/build.gradle    |  1 +
 .../watcher/qa/with-security/build.gradle     | 12 +++------
 6 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 9e9d1e0d9e795..4c86b86b56f5b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -114,7 +114,7 @@ void copy() {
             if (BuildParams.isInternal()) {
                 getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
                 project.copy(c -> {
-                    c.from(coreConfig.getSingleFile());
+                    c.from(coreConfig.getAsFileTree());
                     c.into(getOutputDir());
                     c.include(corePatternSet.getIncludes());
                 });
@@ -138,7 +138,7 @@ void copy() {
         if (includeXpack.get().isEmpty() == false) {
             getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
-                c.from(xpackConfig.getSingleFile());
+                c.from(xpackConfig.getAsFileTree());
                 c.into(getOutputDir());
                 c.include(xpackPatternSet.getIncludes());
             });
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
index a512b9b1fc025..3cdc4d24dd3c1 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.gradle.info.BuildParams;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.provider.Provider;
 
@@ -86,21 +87,31 @@ public class RestResourcesPlugin implements Plugin<Project> {
     public void apply(Project project) {
         RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class);
 
+        // tests
+        Configuration testConfig = project.getConfigurations().create("restTestConfig");
+        Configuration xpackTestConfig = project.getConfigurations().create("restXpackTest");
+        project.getConfigurations().create("restTests");
+        project.getConfigurations().create("restXpackTests");
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
-                task.coreConfig = project.getConfigurations().create("restTest");
+                task.coreConfig = testConfig;
                 if (BuildParams.isInternal()) {
+                    // core
                     Dependency restTestdependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
                     project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
-
-                    task.xpackConfig = project.getConfigurations().create("restXpackTest");
+                    // x-pack
+                    task.xpackConfig = xpackTestConfig;
                     Dependency restXPackTestdependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency);
                     task.dependsOn(task.xpackConfig);
+                    // watcher
+                    Dependency restWatcherTests = project.getDependencies()
+                        .project(Map.of("path", ":x-pack:plugin:watcher:qa:rest", "configuration", "restXpackTests"));
+                    project.getDependencies().add(task.xpackConfig.getName(), restWatcherTests);
                 } else {
                     Dependency dependency = project.getDependencies()
                         .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
@@ -109,18 +120,22 @@ public void apply(Project project) {
                 task.dependsOn(task.coreConfig);
             });
 
+        // api
+        Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
+        Configuration xpackSpecConfig = project.getConfigurations().create("restXpackSpec");
+        project.getConfigurations().create("restSpecs");
+        project.getConfigurations().create("restXpackSpecs");
         Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.includeXpack.set(extension.restApi.getIncludeXpack());
                 task.dependsOn(copyRestYamlTestTask);
-                task.coreConfig = project.getConfigurations().create("restSpec");
+                task.coreConfig = specConfig;
                 if (BuildParams.isInternal()) {
                     Dependency restSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
                     project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
-
-                    task.xpackConfig = project.getConfigurations().create("restXpackSpec");
+                    task.xpackConfig = xpackSpecConfig;
                     Dependency restXpackSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency);
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index fa29345e0ff6b..8902c57211ef2 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -1,15 +1,11 @@
 apply plugin: 'elasticsearch.build'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
+apply plugin: 'elasticsearch.rest-resources'
 
 test.enabled = false
 jarHell.enabled = false
 
-configurations {
-  restSpecs
-  restTests
-}
-
 artifacts {
   restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api"))
   restTests(new File(projectDir, "src/main/resources/rest-api-spec/test"))
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index e2867b1d715a8..e20dd66323320 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -12,8 +12,6 @@ dependencies {
 
 configurations {
   testArtifacts.extendsFrom testRuntime
-  restXpackSpecs
-  restXpackTests
 }
 
 artifacts {
diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle
index df648262e9973..8949fad1f9b96 100644
--- a/x-pack/plugin/watcher/qa/rest/build.gradle
+++ b/x-pack/plugin/watcher/qa/rest/build.gradle
@@ -17,6 +17,7 @@ task testJar(type: Jar) {
 
 artifacts {
   testArtifacts testJar
+  restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
 }
 
 restResources {
diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle
index c80b1894d0c73..5a8b03ca68d37 100644
--- a/x-pack/plugin/watcher/qa/with-security/build.gradle
+++ b/x-pack/plugin/watcher/qa/with-security/build.gradle
@@ -7,21 +7,15 @@ dependencies {
   testCompile project(path: ':x-pack:plugin:watcher:qa:rest', configuration: 'testArtifacts')
 }
 
-
-// bring in watcher rest test suite from the rest project
-task copyWatcherRestTests(type: Copy) {
-  into project.sourceSets.test.output.resourcesDir
-  from project(xpackProject('plugin:watcher:qa:rest').path).sourceSets.test.resources.srcDirs
-  include 'rest-api-spec/test/watcher/**'
-}
-
 restResources {
   restApi {
     includeXpack 'watcher', 'security', 'xpack'
   }
+  restTests {
+    includeXpack 'watcher'
+  }
 }
 
-integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'

From afc2383b72b1d095450581a79b6512684376e243 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Wed, 18 Mar 2020 09:09:29 -0500
Subject: [PATCH 106/285] Optimize which Rest resources are used by the Rest
 tests. (#53299)

This should help with Gradle's incremental compile such that projects
only depend upon the resources they use.

related #52114
---
 modules/aggs-matrix-stats/build.gradle                    | 6 ++++++
 modules/analysis-common/build.gradle                      | 6 ++++++
 modules/ingest-common/build.gradle                        | 6 ++++++
 modules/ingest-geoip/build.gradle                         | 6 ++++++
 modules/ingest-user-agent/build.gradle                    | 6 ++++++
 modules/lang-expression/build.gradle                      | 5 +++++
 modules/lang-mustache/build.gradle                        | 6 ++++++
 modules/lang-painless/build.gradle                        | 7 +++++++
 modules/mapper-extras/build.gradle                        | 6 ++++++
 modules/parent-join/build.gradle                          | 6 ++++++
 modules/percolator/build.gradle                           | 5 +++++
 modules/rank-eval/build.gradle                            | 6 ++++++
 modules/reindex/build.gradle                              | 7 +++++++
 modules/repository-url/build.gradle                       | 8 +++++++-
 modules/transport-netty4/build.gradle                     | 6 ++++++
 plugins/analysis-icu/build.gradle                         | 6 ++++++
 plugins/analysis-kuromoji/build.gradle                    | 6 ++++++
 plugins/analysis-nori/build.gradle                        | 5 +++++
 plugins/analysis-phonetic/build.gradle                    | 6 ++++++
 plugins/analysis-smartcn/build.gradle                     | 6 ++++++
 plugins/analysis-stempel/build.gradle                     | 6 ++++++
 plugins/analysis-ukrainian/build.gradle                   | 6 ++++++
 plugins/discovery-azure-classic/build.gradle              | 5 +++++
 plugins/discovery-ec2/build.gradle                        | 6 ++++++
 plugins/discovery-ec2/qa/amazon-ec2/build.gradle          | 6 ++++++
 plugins/discovery-gce/build.gradle                        | 6 ++++++
 plugins/discovery-gce/qa/gce/build.gradle                 | 6 ++++++
 plugins/ingest-attachment/build.gradle                    | 6 ++++++
 plugins/mapper-annotated-text/build.gradle                | 6 ++++++
 plugins/mapper-murmur3/build.gradle                       | 6 ++++++
 plugins/mapper-size/build.gradle                          | 6 ++++++
 plugins/repository-azure/build.gradle                     | 6 ++++++
 .../qa/microsoft-azure-storage/build.gradle               | 6 ++++++
 plugins/repository-gcs/build.gradle                       | 6 ++++++
 .../repository-gcs/qa/google-cloud-storage/build.gradle   | 6 ++++++
 plugins/repository-hdfs/build.gradle                      | 6 ++++++
 plugins/store-smb/build.gradle                            | 6 +++++-
 x-pack/plugin/autoscaling/qa/rest/build.gradle            | 1 +
 x-pack/plugin/ccr/qa/rest/build.gradle                    | 3 ++-
 x-pack/plugin/enrich/qa/rest/build.gradle                 | 1 +
 x-pack/plugin/eql/qa/rest/build.gradle                    | 1 +
 x-pack/plugin/graph/qa/with-security/build.gradle         | 1 +
 x-pack/plugin/ilm/qa/rest/build.gradle                    | 1 +
 x-pack/plugin/ml/qa/ml-with-security/build.gradle         | 1 +
 x-pack/plugin/watcher/qa/rest/build.gradle                | 2 ++
 45 files changed, 230 insertions(+), 3 deletions(-)

diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
index b397edf2cbb5c..ffe8145eba341 100644
--- a/modules/aggs-matrix-stats/build.gradle
+++ b/modules/aggs-matrix-stats/build.gradle
@@ -21,3 +21,9 @@ esplugin {
   description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'
   classname 'org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin'
 }
+
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'cluster', 'index', 'search', 'nodes'
+  }
+}
diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle
index 773c645f698cc..24e6e651c3fd9 100644
--- a/modules/analysis-common/build.gradle
+++ b/modules/analysis-common/build.gradle
@@ -23,6 +23,12 @@ esplugin {
   extendedPlugins = ['lang-painless']
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'cluster', 'search', 'nodes', 'bulk', 'termvectors', 'explain', 'count'
+  }
+}
+
 dependencies {
   compileOnly project(':modules:lang-painless')
 }
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index 6590bdc1c52ef..2ff50f1543ed9 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -29,6 +29,12 @@ dependencies {
   compile project(':libs:elasticsearch-dissect')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget'
+  }
+}
+
 testClusters.integTest {
   // Needed in order to test ingest pipeline templating:
   // (this is because the integTest node is not using default distribution, but only the minimal number of required modules)
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index dfa165e566170..5d200bf87b55a 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -34,6 +34,12 @@ dependencies {
   testCompile 'org.elasticsearch:geolite2-databases:20191119'
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest'
+  }
+}
+
 task copyDefaultGeoIp2DatabaseFiles(type: Copy) {
   from { zipTree(configurations.testCompile.files.find { it.name.contains('geolite2-databases') }) }
   into "${project.buildDir}/ingest-geoip"
diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle
index b3e17d58360d4..d45b30c9f1daf 100644
--- a/modules/ingest-user-agent/build.gradle
+++ b/modules/ingest-user-agent/build.gradle
@@ -22,6 +22,12 @@ esplugin {
   classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin'
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest'
+  }
+}
+
 testClusters.integTest {
   extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml')
 }
diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle
index 2fd6e53effa34..d3a165c13b947 100644
--- a/modules/lang-expression/build.gradle
+++ b/modules/lang-expression/build.gradle
@@ -29,6 +29,11 @@ dependencies {
   compile 'org.ow2.asm:asm-commons:5.0.4'
   compile 'org.ow2.asm:asm-tree:5.0.4'
 }
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'search'
+  }
+}
 
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 5adc409c8bcc1..ed0641b89fa1a 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -26,3 +26,9 @@ dependencies {
   compile "com.github.spullara.mustache.java:compiler:0.9.6"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'bulk',
+      'put_script', 'render_search_template', 'search_template', 'msearch_template', 'lang_mustache'
+  }
+}
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 75e41dcf2e2aa..68a990a3dda9a 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -44,6 +44,13 @@ dependencyLicenses {
   mapping from: /asm-.*/, to: 'asm'
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', 
+                'scripts_painless_execute', 'put_script', 'delete_script'
+  }
+}
+
 test {
   // in WhenThingsGoWrongTests we intentionally generate an out of memory error, this prevents the heap from being dumped to disk
   jvmArgs '-XX:-OmitStackTraceInFastThrow', '-XX:-HeapDumpOnOutOfMemoryError'
diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle
index d4f3b6871a631..4d2e4b7d70ffe 100644
--- a/modules/mapper-extras/build.gradle
+++ b/modules/mapper-extras/build.gradle
@@ -21,3 +21,9 @@ esplugin {
   description 'Adds advanced field mappers'
   classname 'org.elasticsearch.index.mapper.MapperExtrasPlugin'
 }
+
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get' 
+  }
+}
diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle
index 756a65a371a9b..d6cb0c3c3a515 100644
--- a/modules/parent-join/build.gradle
+++ b/modules/parent-join/build.gradle
@@ -21,3 +21,9 @@ esplugin {
   description 'This module adds the support parent-child queries and aggregations'
   classname 'org.elasticsearch.join.ParentJoinPlugin'
 }
+
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search' 
+  }
+}
diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
index b0cb4390b02bb..ae19823db1542 100644
--- a/modules/percolator/build.gradle
+++ b/modules/percolator/build.gradle
@@ -26,6 +26,11 @@ dependencies {
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search', 'msearch'  
+  }
+}
 dependencyLicenses {
   // Don't check the client's license. We know it.
   dependencies = project.configurations.runtime.fileCollection {
diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle
index 35f8fef5176a5..1adcd3016e50d 100644
--- a/modules/rank-eval/build.gradle
+++ b/modules/rank-eval/build.gradle
@@ -22,6 +22,12 @@ esplugin {
   classname 'org.elasticsearch.index.rankeval.RankEvalPlugin'
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'rank_eval'  
+  }
+}
+
 testClusters.integTest {
   // Modules who's integration is explicitly tested in integration tests
   module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile)
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index e255c2567cc9a..a6b60166c343f 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -55,6 +55,13 @@ dependencies {
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', 
+                'update_by_query', 'delete_by_query', 'reindex_rethrottle', 'tasks', 'reindex', 'put_script'  
+  }
+}
+
 thirdPartyAudit.ignoreMissingClasses(
   // Commons logging
   'javax.servlet.ServletContextEvent',
diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle
index 31fba2964f0c5..cec28b1d00fc8 100644
--- a/modules/repository-url/build.gradle
+++ b/modules/repository-url/build.gradle
@@ -26,7 +26,13 @@ esplugin {
   classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
 }
 
-// This directory is shared between two URL repositories and one FS repository in YAML integration tests
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'bulk', 'count', 'snapshot' 
+  }
+}
+
+ // This directory is shared between two URL repositories and one FS repository in YAML integration tests
 File repositoryDir = new File(project.buildDir, "shared-repository")
 
 /** A task to start the URLFixture which exposes the repositoryDir over HTTP **/
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index 9db16d00e7d61..02386665abb41 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -43,6 +43,12 @@ dependencies {
   compile "io.netty:netty-transport:${versions.netty}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 dependencyLicenses {
   mapping from: /netty-.*/, to: 'netty'
 }
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index 3fbf7f09b726e..3d1e2909c2291 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -35,6 +35,12 @@ dependencies {
   compile "com.ibm.icu:icu4j:${versions.icu4j}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
+
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle
index 333818a8d61b7..85c8b472aceea 100644
--- a/plugins/analysis-kuromoji/build.gradle
+++ b/plugins/analysis-kuromoji/build.gradle
@@ -26,6 +26,12 @@ dependencies {
   compile "org.apache.lucene:lucene-analyzers-kuromoji:${versions.lucene}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
+
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle
index a9d3a1126dc58..c732ec9037db9 100644
--- a/plugins/analysis-nori/build.gradle
+++ b/plugins/analysis-nori/build.gradle
@@ -26,6 +26,11 @@ dependencies {
   compile "org.apache.lucene:lucene-analyzers-nori:${versions.lucene}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle
index 61c4fdbd58394..82091b91d9ead 100644
--- a/plugins/analysis-phonetic/build.gradle
+++ b/plugins/analysis-phonetic/build.gradle
@@ -27,6 +27,12 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
+
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle
index ebe44850d00a7..a6d287ef690de 100644
--- a/plugins/analysis-smartcn/build.gradle
+++ b/plugins/analysis-smartcn/build.gradle
@@ -26,6 +26,12 @@ dependencies {
   compile "org.apache.lucene:lucene-analyzers-smartcn:${versions.lucene}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
+
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle
index 488e99ec912bd..7f90e10817364 100644
--- a/plugins/analysis-stempel/build.gradle
+++ b/plugins/analysis-stempel/build.gradle
@@ -26,6 +26,12 @@ dependencies {
   compile "org.apache.lucene:lucene-analyzers-stempel:${versions.lucene}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
+
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle
index 0e254e5f05026..cec538d7d79a8 100644
--- a/plugins/analysis-ukrainian/build.gradle
+++ b/plugins/analysis-ukrainian/build.gradle
@@ -29,6 +29,12 @@ dependencies {
   compile "ua.net.nlp:morfologik-ukrainian-search:3.7.5"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
+
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /morfologik-.*/, to: 'lucene'
diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index 257777526c92b..bbad69232d0fa 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -57,6 +57,11 @@ dependencies {
   compile 'javax.xml.bind:jaxb-api:2.2.2'
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
 // needed to be consistent with ssl host checking
 String host = InetAddress.getLoopbackAddress().getHostAddress()
 
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index bc507dc5d9742..72d3c5b294bd3 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -40,6 +40,12 @@ dependencies {
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 dependencyLicenses {
   mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
   mapping from: /jackson-.*/, to: 'jackson'
diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
index 023c864d62e7c..a10d9feccfdce 100644
--- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
+++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
@@ -32,6 +32,12 @@ dependencies {
   testCompile project(path: ':plugins:discovery-ec2', configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 final int ec2NumberOfNodes = 3
 
 Map<String, Object> expansions = [
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index 7e5c29ff5c33c..275eb9312a30b 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -21,6 +21,12 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 dependencyLicenses {
   mapping from: /google-.*/, to: 'google'
 }
diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle
index 3a5bf84c7ba93..7eb2fe13aae8f 100644
--- a/plugins/discovery-gce/qa/gce/build.gradle
+++ b/plugins/discovery-gce/qa/gce/build.gradle
@@ -33,6 +33,12 @@ dependencies {
   testCompile project(path: ':plugins:discovery-gce', configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 /** A task to start the GCEFixture which emulates a GCE service **/
 task gceFixture(type: AntFixture) {
   dependsOn compileTestJava
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index 0c5d5deaf3f04..668c4c397e814 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -72,6 +72,12 @@ dependencies {
   compile 'org.apache.commons:commons-lang3:3.9'
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'ingest', 'index', 'get' 
+  }
+}
+
 dependencyLicenses {
   mapping from: /apache-mime4j-.*/, to: 'apache-mime4j'
 }
diff --git a/plugins/mapper-annotated-text/build.gradle b/plugins/mapper-annotated-text/build.gradle
index 8ce1ca2a416fe..2ae9a9278b0a5 100644
--- a/plugins/mapper-annotated-text/build.gradle
+++ b/plugins/mapper-annotated-text/build.gradle
@@ -21,3 +21,9 @@ esplugin {
   description 'The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.'
   classname 'org.elasticsearch.plugin.mapper.AnnotatedTextPlugin'
 }
+
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle
index 5b985d9138f52..f556e82db3867 100644
--- a/plugins/mapper-murmur3/build.gradle
+++ b/plugins/mapper-murmur3/build.gradle
@@ -21,3 +21,9 @@ esplugin {
   description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.'
   classname 'org.elasticsearch.plugin.mapper.MapperMurmur3Plugin'
 }
+
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+}
diff --git a/plugins/mapper-size/build.gradle b/plugins/mapper-size/build.gradle
index 5a49ce5d04b78..0c757b4966bd3 100644
--- a/plugins/mapper-size/build.gradle
+++ b/plugins/mapper-size/build.gradle
@@ -21,3 +21,9 @@ esplugin {
   description 'The Mapper Size plugin allows document to record their uncompressed size at index time.'
   classname 'org.elasticsearch.plugin.mapper.MapperSizePlugin'
 }
+
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'get'
+  }
+}
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 5a11f8b02bf84..33ab95e6ea405 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -32,6 +32,12 @@ dependencies {
   testCompile project(':test:fixtures:azure-fixture')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 dependencyLicenses {
   mapping from: /azure-.*/, to: 'azure'
   mapping from: /jackson-.*/, to: 'jackson'
diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
index 529ee899fc8c4..b594ff3b172f2 100644
--- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
+++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
@@ -29,6 +29,12 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
+restResources {
+  restApi {
+    includeCore '_common', 'snapshot', 'bulk', 'count', 'indices'
+  }
+}
+
 testFixtures.useFixture ":test:fixtures:azure-fixture", "azure-fixture"
 
 boolean useFixture = false
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index bd20c36922297..9af1ae73327ef 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -56,6 +56,12 @@ dependencies {
   testCompile project(':test:fixtures:gcs-fixture')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
 dependencyLicenses {
   mapping from: /google-cloud-.*/, to: 'google-cloud'
   mapping from: /google-auth-.*/, to: 'google-auth'
diff --git a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle
index aaa5a4ddc553b..10105d2f2348c 100644
--- a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle
+++ b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle
@@ -36,6 +36,12 @@ dependencies {
   testCompile project(path: ':plugins:repository-gcs')
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'snapshot','indices', 'index', 'bulk', 'count'
+  }
+}
+
 testFixtures.useFixture(':test:fixtures:gcs-fixture')
 boolean useFixture = false
 
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 83e3735300a0f..bf570b8e0abb6 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -74,6 +74,12 @@ dependencies {
   }
 }
 
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'snapshot'
+  }
+}
+
 normalization {
   runtimeClasspath {
     // ignore generated keytab files for the purposes of build avoidance
diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle
index f2238e0a49c90..87aad3e59769b 100644
--- a/plugins/store-smb/build.gradle
+++ b/plugins/store-smb/build.gradle
@@ -21,4 +21,8 @@ esplugin {
   description 'The Store SMB plugin adds support for SMB stores.'
   classname 'org.elasticsearch.plugin.store.smb.SMBStorePlugin'
 }
-
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes', 'index', 'indices', 'get'
+  }
+}
diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle
index 631c6e950ee1a..d7d66eeffc27f 100644
--- a/x-pack/plugin/autoscaling/qa/rest/build.gradle
+++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle
@@ -11,6 +11,7 @@ dependencies {
 
 restResources {
   restApi {
+    includeCore '_common'
     includeXpack 'autoscaling'
   }
 }
diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle
index d95ac271df519..fd16660178c57 100644
--- a/x-pack/plugin/ccr/qa/rest/build.gradle
+++ b/x-pack/plugin/ccr/qa/rest/build.gradle
@@ -5,8 +5,9 @@ apply plugin: 'elasticsearch.standalone-test'
 
 restResources {
   restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'info' 
     includeXpack 'ccr'
-  }
+  } 
 }
 
 dependencies {
diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle
index 209154dd448b0..9344134741778 100644
--- a/x-pack/plugin/enrich/qa/rest/build.gradle
+++ b/x-pack/plugin/enrich/qa/rest/build.gradle
@@ -4,6 +4,7 @@ apply plugin: 'elasticsearch.rest-test'
 
 restResources {
   restApi {
+    includeCore '_common', 'indices', 'index'
     includeXpack 'enrich'
   }
 }
diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle
index a7e94fcba6d6d..43077d1d5c5d8 100644
--- a/x-pack/plugin/eql/qa/rest/build.gradle
+++ b/x-pack/plugin/eql/qa/rest/build.gradle
@@ -6,6 +6,7 @@ apply plugin: 'elasticsearch.rest-test'
 
 restResources {
   restApi {
+    includeCore '_common', 'bulk'
     includeXpack 'eql'
   }
 }
diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle
index 869e926102dcb..2f0461dd3f189 100644
--- a/x-pack/plugin/graph/qa/with-security/build.gradle
+++ b/x-pack/plugin/graph/qa/with-security/build.gradle
@@ -9,6 +9,7 @@ dependencies {
 // bring in graph rest test suite
 restResources {
   restApi {
+    includeCore '_common', 'cluster', 'indices', 'index'
     includeXpack 'graph'
   }
   restTests {
diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle
index 6c5ea98a0b46d..365810123252b 100644
--- a/x-pack/plugin/ilm/qa/rest/build.gradle
+++ b/x-pack/plugin/ilm/qa/rest/build.gradle
@@ -10,6 +10,7 @@ dependencies {
 
 restResources {
   restApi {
+    includeCore '_common', 'cluster', 'indices', 'index', 'snapshot'
     includeXpack 'ilm', 'slm'
   }
 }
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index 516e5c626fca2..1146db74510bb 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -11,6 +11,7 @@ dependencies {
 // bring in machine learning rest test suite
 restResources {
   restApi {
+    includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'count', 'ingest'
     includeXpack 'ml', 'cat'
   }
   restTests {
diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle
index 8949fad1f9b96..798b906ca4e1a 100644
--- a/x-pack/plugin/watcher/qa/rest/build.gradle
+++ b/x-pack/plugin/watcher/qa/rest/build.gradle
@@ -22,6 +22,8 @@ artifacts {
 
 restResources {
   restApi {
+    includeCore '_common', 'cluster', 'nodes', 'search', 'get', 'indices', 
+                'index', 'put_script', 'ingest', 'count'
     includeXpack 'watcher', 'xpack'
   }
 }

From 534a4a9b320fbdecbebf6cf30505915ee55e8e88 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Wed, 18 Mar 2020 14:13:00 +0000
Subject: [PATCH 107/285] Remove [removal-of-types] docs page, and point to 7x
 docs (#53670)

Given that types have been removed entirely in 8.0, we don't need a detailed page
explaining what they are and why they are going away. This commit replaces the
page with a short paragraph saying that types are no longer supported, and a link
to the [removal-of-types] page in 7x

Relates to #41059
---
 .../mapping/removal_of_types.asciidoc         | 674 +-----------------
 1 file changed, 3 insertions(+), 671 deletions(-)

diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc
index ec101080e1b31..ca028355de6f7 100644
--- a/docs/reference/mapping/removal_of_types.asciidoc
+++ b/docs/reference/mapping/removal_of_types.asciidoc
@@ -1,674 +1,6 @@
 [[removal-of-types]]
 == Removal of mapping types
 
-IMPORTANT: Indices created in Elasticsearch 7.0.0 or later no longer accept a
-`_default_` mapping. Indices created in 6.x will continue to function as before
-in Elasticsearch 6.x. Types are deprecated in APIs in 7.0, with breaking changes
-to the index creation, put mapping, get mapping, put template, get template and
-get field mappings APIs.
-
-[float]
-=== What are mapping types?
-
-Since the first release of Elasticsearch, each document has been stored in a
-single index and assigned a single mapping type.  A mapping type was used to
-represent the type of document or entity being indexed, for instance a
-`twitter` index might have a `user` type and a `tweet` type.
-
-Each mapping type could have its own fields, so the `user` type might have a
-`full_name` field, a `user_name` field, and an `email` field, while the
-`tweet` type could have a `content` field, a `tweeted_at` field and, like the
-`user` type, a `user_name` field.
-
-Each document had a `_type` meta-field containing the type name, and searches
-could be limited to one or more types by specifying the type name(s) in the
-URL:
-
-[source,js]
-----
-GET twitter/user,tweet/_search
-{
-  "query": {
-    "match": {
-      "user_name": "kimchy"
-    }
-  }
-}
-----
-// NOTCONSOLE
-
-The `_type` field was combined with the document's `_id` to generate a `_uid`
-field, so documents of different types with the same `_id` could exist in a
-single index.
-
-Mapping types were also used to establish a
-<<parent-join,parent-child relationship>>
-between documents, so documents of type `question` could be parents to
-documents of type `answer`.
-
-[float]
-=== Why are mapping types being removed?
-
-Initially, we spoke about an ``index'' being similar to a ``database'' in an
-SQL database, and a ``type'' being equivalent to a
-``table''.
-
-This was a bad analogy that led to incorrect assumptions. In an SQL database,
-tables are independent of each other.  The columns in one table have no
-bearing on columns with the same name in another table.  This is not the case
-for fields in a mapping type.
-
-In an Elasticsearch index, fields that have the same name in different mapping
-types are backed by the same Lucene field internally.  In other words, using
-the example above, the `user_name` field in the `user` type is stored in
-exactly the same field as the `user_name` field in the `tweet` type, and both
-`user_name` fields must have the same mapping (definition) in both types.
-
-This can lead to frustration when, for example, you want `deleted` to be a
-`date` field in one type and a `boolean` field in another type in the same
-index.
-
-On top of that, storing different entities that have few or no fields in
-common in the same index leads to sparse data and interferes with Lucene's
-ability to compress documents efficiently.
-
-For these reasons, we have decided to remove the concept of mapping types from
-Elasticsearch.
-
-[float]
-=== Alternatives to mapping types
-
-[float]
-==== Index per document type
-
-The first alternative is to have an index per document type.  Instead of
-storing tweets and users in a single `twitter` index, you could store tweets
-in the `tweets` index and users in the `user` index. Indices are completely
-independent of each other and so there will be no conflict of field types
-between indices.
-
-This approach has two benefits:
-
-* Data is more likely to be dense and so benefit from compression techniques
-  used in Lucene.
-
-* The term statistics used for scoring in full text search are more likely to
-  be accurate because all documents in the same index represent a single
-  entity.
-
-Each index can be sized appropriately for the number of documents it will
-contain: you can use a smaller number of primary shards for `users` and a
-larger number of primary shards for `tweets`.
-
-[float]
-==== Custom type field
-
-Of course, there is a limit to how many primary shards can exist in a cluster
-so you may not want to waste an entire shard for a collection of only a few
-thousand documents.  In this case, you can implement your own custom `type`
-field which will work in a similar way to the old `_type`.
-
-Let's take the `user`/`tweet` example above.  Originally, the workflow would
-have looked something like this:
-
-[source,js]
-----
-PUT twitter
-{
-  "mappings": {
-    "user": {
-      "properties": {
-        "name": { "type": "text" },
-        "user_name": { "type": "keyword" },
-        "email": { "type": "keyword" }
-      }
-    },
-    "tweet": {
-      "properties": {
-        "content": { "type": "text" },
-        "user_name": { "type": "keyword" },
-        "tweeted_at": { "type": "date" }
-      }
-    }
-  }
-}
-
-PUT twitter/user/kimchy
-{
-  "name": "Shay Banon",
-  "user_name": "kimchy",
-  "email": "shay@kimchy.com"
-}
-
-PUT twitter/tweet/1
-{
-  "user_name": "kimchy",
-  "tweeted_at": "2017-10-24T09:00:00Z",
-  "content": "Types are going away"
-}
-
-GET twitter/tweet/_search
-{
-  "query": {
-    "match": {
-      "user_name": "kimchy"
-    }
-  }
-}
-----
-// NOTCONSOLE
-
-You can achieve the same thing by adding a custom `type` field as follows:
-
-[source,js]
-----
-PUT twitter
-{
-  "mappings": {
-    "_doc": {
-      "properties": {
-        "type": { "type": "keyword" }, <1>
-        "name": { "type": "text" },
-        "user_name": { "type": "keyword" },
-        "email": { "type": "keyword" },
-        "content": { "type": "text" },
-        "tweeted_at": { "type": "date" }
-      }
-    }
-  }
-}
-
-PUT twitter/_doc/user-kimchy
-{
-  "type": "user", <1>
-  "name": "Shay Banon",
-  "user_name": "kimchy",
-  "email": "shay@kimchy.com"
-}
-
-PUT twitter/_doc/tweet-1
-{
-  "type": "tweet", <1>
-  "user_name": "kimchy",
-  "tweeted_at": "2017-10-24T09:00:00Z",
-  "content": "Types are going away"
-}
-
-GET twitter/_search
-{
-  "query": {
-    "bool": {
-      "must": {
-        "match": {
-          "user_name": "kimchy"
-        }
-      },
-      "filter": {
-        "match": {
-          "type": "tweet" <1>
-        }
-      }
-    }
-  }
-}
-----
-// NOTCONSOLE
-<1> The explicit `type` field takes the place of the implicit `_type` field.
-
-[float]
-[[parent-child-mapping-types]]
-==== Parent/Child without mapping types
-
-Previously, a parent-child relationship was represented by making one mapping
-type the parent, and one or more other mapping types the children.  Without
-types, we can no longer use this syntax.  The parent-child feature will
-continue to function as before, except that the way of expressing the
-relationship between documents has been changed to use the new
-<<parent-join,`join` field>>.
-
-
-[float]
-=== Schedule for removal of mapping types
-
-This is a big change for our users, so we have tried to make it as painless as
-possible.  The change will roll out as follows:
-
-Elasticsearch 5.6.0::
-
-* Setting `index.mapping.single_type: true` on an index will enable the
-  single-type-per-index behaviour which will be enforced in 6.0.
-
-* The <<parent-join,`join` field>> replacement for parent-child is available
-  on indices created in 5.6.
-
-Elasticsearch 6.x::
-
-* Indices created in 5.x will continue to function in 6.x as they did in 5.x.
-
-* Indices created in 6.x only allow a single-type per index.  Any name
-  can be used for the type, but there can be only one. The preferred type name
-  is `_doc`, so that index APIs have the same path as they will have in 7.0:
-  `PUT {index}/_doc/{id}` and `POST {index}/_doc`
-
-* The `_type` name can no longer be combined with the `_id` to form the `_uid`
-  field. The `_uid` field has become an alias for the `_id` field.
-
-* New indices no longer support the old-style of parent/child and should
-  use the <<parent-join,`join` field>> instead.
-
-* The `_default_` mapping type is deprecated.
-
-* In 6.8, the index creation, index template, and mapping APIs support a query
-  string parameter (`include_type_name`) which indicates whether requests and
-  responses should include a type name. It defaults to `true`, and should be set
-  to an explicit value to prepare to upgrade to 7.0. Not setting `include_type_name`
-  will result in a deprecation warning. Indices which don't have an explicit type will
-  use the dummy type name `_doc`.
-
-Elasticsearch 7.x::
-
-* Specifying types in requests is deprecated. For instance, indexing a
-  document no longer requires a document `type`.  The new index APIs
-  are `PUT {index}/_doc/{id}` in case of explicit ids and `POST {index}/_doc`
-  for auto-generated ids. Note that in 7.0, `_doc` is a permanent part of the
-  path, and represents the endpoint name rather than the document type.
-
-* The `include_type_name` parameter in the index creation, index template,
-  and mapping APIs will default to `false`. Setting the parameter at all will
-  result in a deprecation warning.
-
-* The `_default_` mapping type is removed.
-
-Elasticsearch 8.x::
-
-* Specifying types in requests is no longer supported.
-
-* The `include_type_name` parameter is removed.
-
-[float]
-=== Migrating multi-type indices to single-type
-
-The <<docs-reindex,Reindex API>> can be used to convert multi-type indices to
-single-type indices. The following examples can be used in Elasticsearch 5.6
-or Elasticsearch 6.x.  In 6.x, there is no need to specify
-`index.mapping.single_type` as that is the default.
-
-[float]
-==== Index per document type
-
-This first example splits our `twitter` index into a `tweets` index and a
-`users` index:
-
-[source,js]
-----
-PUT users
-{
-  "settings": {
-    "index.mapping.single_type": true
-  },
-  "mappings": {
-    "_doc": {
-      "properties": {
-        "name": {
-          "type": "text"
-        },
-        "user_name": {
-          "type": "keyword"
-        },
-        "email": {
-          "type": "keyword"
-        }
-      }
-    }
-  }
-}
-
-PUT tweets
-{
-  "settings": {
-    "index.mapping.single_type": true
-  },
-  "mappings": {
-    "_doc": {
-      "properties": {
-        "content": {
-          "type": "text"
-        },
-        "user_name": {
-          "type": "keyword"
-        },
-        "tweeted_at": {
-          "type": "date"
-        }
-      }
-    }
-  }
-}
-
-POST _reindex
-{
-  "source": {
-    "index": "twitter",
-    "type": "user"
-  },
-  "dest": {
-    "index": "users",
-    "type": "_doc"
-  }
-}
-
-POST _reindex
-{
-  "source": {
-    "index": "twitter",
-    "type": "tweet"
-  },
-  "dest": {
-    "index": "tweets",
-    "type": "_doc"
-  }
-}
-----
-// NOTCONSOLE
-
-[float]
-==== Custom type field
-
-This next example adds a custom `type` field and sets it to the value of the
-original `_type`.  It also adds the type to the `_id` in case there are any
-documents of different types which have conflicting IDs:
-
-[source,js]
-----
-PUT new_twitter
-{
-  "mappings": {
-    "_doc": {
-      "properties": {
-        "type": {
-          "type": "keyword"
-        },
-        "name": {
-          "type": "text"
-        },
-        "user_name": {
-          "type": "keyword"
-        },
-        "email": {
-          "type": "keyword"
-        },
-        "content": {
-          "type": "text"
-        },
-        "tweeted_at": {
-          "type": "date"
-        }
-      }
-    }
-  }
-}
-
-
-POST _reindex
-{
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter"
-  },
-  "script": {
-    "source": """
-      ctx._source.type = ctx._type;
-      ctx._id = ctx._type + '-' + ctx._id;
-      ctx._type = '_doc';
-    """
-  }
-}
-----
-// NOTCONSOLE
-
-[float]
-=== Typeless APIs in 7.0
-
-In Elasticsearch 7.0, each API will support typeless requests,
-and specifying a type will produce a deprecation warning.
-
-NOTE: Typeless APIs work even if the target index contains a custom type.
-For example, if an index has the custom type name `my_type`, we can add
-documents to it using typeless `index` calls, and load documents with typeless
-`get` calls.
-
-[float]
-==== Index APIs
-
-Index creation, index template, and mapping APIs support a new `include_type_name`
-URL parameter that specifies whether mapping definitions in requests and responses
-should contain the type name. The parameter defaults to `true` in version 6.8 to
-match the pre-7.0 behavior of using type names in mappings. It defaults to `false`
-in version 7.0 and will be removed in version 8.0.
-
-It should be set explicitly in 6.8 to prepare to upgrade to 7.0. To avoid deprecation
-warnings in 6.8, the parameter can be set to either `true` or `false`. In 7.0, setting
-`include_type_name` at all will result in a deprecation warning.
-
-See some examples of interactions with Elasticsearch with this option set to `false`:
-
-[source,console]
---------------------------------------------------
-PUT index
-{
-  "mappings": {
-    "properties": { <1>
-      "foo": {
-        "type": "keyword"
-      }
-    }
-  }
-}
---------------------------------------------------
-
-<1> Mappings are included directly under the `mappings` key, without a type name.
-
-[source,console]
---------------------------------------------------
-PUT index/_mappings
-{
-  "properties": { <1>
-    "bar": {
-      "type": "text"
-    }
-  }
-}
---------------------------------------------------
-// TEST[continued]
-
-<1> Mappings are included directly under the `mappings` key, without a type name.
-
-[source,console]
---------------------------------------------------
-GET index/_mappings
---------------------------------------------------
-// TEST[continued]
-
-The above call returns
-
-[source,console-result]
---------------------------------------------------
-{
-  "index": {
-    "mappings": {
-      "properties": { <1>
-        "foo": {
-          "type": "keyword"
-        },
-        "bar": {
-          "type": "text"
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-
-<1> Mappings are included directly under the `mappings` key, without a type name.
-
-[float]
-==== Document APIs
-
-In 7.0, index APIs must be called with the `{index}/_doc` path for automatic
-generation of the `_id` and `{index}/_doc/{id}` with explicit ids.
-
-[source,console]
---------------------------------------------------
-PUT index/_doc/1
-{
-  "foo": "baz"
-}
---------------------------------------------------
-
-[source,console-result]
---------------------------------------------------
-{
-  "_index": "index",
-  "_id": "1",
-  "_version": 1,
-  "result": "created",
-  "_shards": {
-    "total": 2,
-    "successful": 1,
-    "failed": 0
-  },
-  "_seq_no": 0,
-  "_primary_term": 1
-}
---------------------------------------------------
-
-Similarly, the `get` and `delete` APIs use the path `{index}/_doc/{id}`:
-
-[source,console]
---------------------------------------------------
-GET index/_doc/1
---------------------------------------------------
-// TEST[continued]
-
-NOTE: In 7.0, `_doc` represents the endpoint name instead of the document type.
-The `_doc` component is a permanent part of the path for the document `index`,
-`get`, and `delete` APIs going forward, and will not be removed in 8.0.
-
-For API paths that contain both a type and endpoint name like `_update`,
-in 7.0 the endpoint will immediately follow the index name:
-
-[source,console]
---------------------------------------------------
-POST index/_update/1
-{
-    "doc" : {
-        "foo" : "qux"
-    }
-}
-
-GET /index/_source/1
---------------------------------------------------
-// TEST[continued]
-
-Types should also no longer appear in the body of requests. The following
-example of bulk indexing omits the type both in the URL, and in the individual
-bulk commands:
-
-[source,console]
---------------------------------------------------
-POST _bulk
-{ "index" : { "_index" : "index", "_id" : "3" } }
-{ "foo" : "baz" }
-{ "index" : { "_index" : "index", "_id" : "4" } }
-{ "foo" : "qux" }
---------------------------------------------------
-
-[float]
-==== Search APIs
-
-When calling a search API such `_search`, `_msearch`, or `_explain`, types
-should not be included in the URL. Additionally, the `_type` field should not
-be used in queries, aggregations, or scripts.
-
-[float]
-==== Index templates
-
-It is recommended to make index templates typeless by re-adding them with
-`include_type_name` set to `false`. Under the hood, typeless templates will use
-the dummy type `_doc` when creating indices.
-
-In case typeless templates are used with typed index creation calls or typed
-templates are used with typeless index creation calls, the template will still
-be applied but the index creation call decides whether there should be a type
-or not. For instance in the below example, `index-1-01` will have a type in
-spite of the fact that it matches a template that is typeless, and `index-2-01`
-will be typeless in spite of the fact that it matches a template that defines
-a type. Both `index-1-01` and `index-2-01` will inherit the `foo` field from
-the template that they match.
-
-[source,console]
---------------------------------------------------
-PUT _template/template1
-{
-  "index_patterns":[ "index-1-*" ],
-  "mappings": {
-    "properties": {
-      "foo": {
-        "type": "keyword"
-      }
-    }
-  }
-}
-
-PUT index-1-01
-{
-  "mappings": {
-    "properties": {
-      "bar": {
-        "type": "long"
-      }
-    }
-  }
-}
-
-PUT index-2-01
-{
-  "mappings": {
-    "properties": {
-      "bar": {
-        "type": "long"
-      }
-    }
-  }
-}
---------------------------------------------------
-
-//////////////////////////
-
-[source,console]
---------------------------------------------------
-DELETE /_template/template1
---------------------------------------------------
-// TEST[continued]
-
-//////////////////////////
-
-In case of implicit index creation, because of documents that get indexed in
-an index that doesn't exist yet, the template is always honored. This is
-usually not a problem due to the fact that typeless index calls work on typed
-indices.
-
-[float]
-==== Mixed-version clusters
-
-In a cluster composed of both 6.8 and 7.0 nodes, the parameter
-`include_type_name` should be specified in index APIs like index
-creation. This is because the parameter has a different default between
-6.8 and 7.0, so the same mapping definition will not be valid for both
-node versions.
-
-Typeless document APIs such as `bulk` and `update` are only available as of
-7.0, and will not work with 6.8 nodes. This also holds true for the typeless
-versions of queries that perform document lookups, such as `terms`.
+Elasticsearch 8.0.0 no longer supports mapping types.  For details on how to
+migrate your clusters away from mapping types, see the
+{ref-7x}/removal-of-types.html[removal of types] documentation for the 7.x release

From e23311ce516b99519935b822b885af176670dee4 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Wed, 18 Mar 2020 14:15:30 +0000
Subject: [PATCH 108/285] Make it possible to deprecate all variants of a
 ParseField with no replacement (#53722)

Sometimes we want to deprecate and remove a ParseField entirely, without replacement;
for example, the various places where we specify a _type field in 7x. Currently we can
tell users only that a particular field name should not be used, and that another name should
be used in its place. This commit adds the ability to say that a field should not be used at
all.
---
 .../client/RestHighLevelClient.java           |  7 +--
 .../client/ml/job/config/RuleScope.java       | 11 +---
 .../DeleteRoleMappingResponseTests.java       | 10 +---
 .../security/ExpressionRoleMappingTests.java  | 34 ++++-------
 .../security/GetPrivilegesResponseTests.java  | 10 +---
 .../GetRoleMappingsResponseTests.java         | 60 ++++++++-----------
 .../security/GetRolesResponseTests.java       | 10 +---
 .../RoleMapperExpressionParserTests.java      | 10 +---
 .../privileges/ApplicationPrivilegeTests.java | 10 +---
 .../org/elasticsearch/common/ParseField.java  | 16 ++++-
 .../common/xcontent/DeprecationHandler.java   | 35 ++++++++++-
 .../elasticsearch/common/ParseFieldTests.java | 15 +++++
 .../xcontent/LoggingDeprecationHandler.java   |  7 ++-
 ...LoggingDeprecationAccumulationHandler.java |  7 +++
 .../xpack/security/authc/ApiKeyService.java   |  6 ++
 15 files changed, 128 insertions(+), 120 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index b464c2166f865..e1ca36f552819 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -1888,12 +1888,7 @@ protected static boolean convertExistsResponse(Response response) {
      * emitted there just mean that you are talking to an old version of
      * Elasticsearch. There isn't anything you can do about the deprecation.
      */
-    private static final DeprecationHandler DEPRECATION_HANDLER = new DeprecationHandler() {
-        @Override
-        public void usedDeprecatedName(String usedName, String modernName) {}
-        @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {}
-    };
+    private static final DeprecationHandler DEPRECATION_HANDLER = DeprecationHandler.IGNORE_DEPRECATIONS;
 
     static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/RuleScope.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/RuleScope.java
index 8b6886d582524..95e727b818abe 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/RuleScope.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/RuleScope.java
@@ -50,7 +50,7 @@ public static ContextParser<Void, RuleScope> parser() {
                     Map<String, ?> value = (Map<String, ?>) entry.getValue();
                     builder.map(value);
                     try (XContentParser scopeParser = XContentFactory.xContent(builder.contentType()).createParser(
-                            NamedXContentRegistry.EMPTY, DEPRECATION_HANDLER, Strings.toString(builder))) {
+                            NamedXContentRegistry.EMPTY, DeprecationHandler.IGNORE_DEPRECATIONS, Strings.toString(builder))) {
                         scope.put(entry.getKey(), FilterRef.PARSER.parse(scopeParser, null));
                     }
                 }
@@ -59,15 +59,6 @@ public static ContextParser<Void, RuleScope> parser() {
         };
     }
 
-    private static final DeprecationHandler DEPRECATION_HANDLER = new DeprecationHandler() {
-
-        @Override
-        public void usedDeprecatedName(String usedName, String modernName) {}
-
-        @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {}
-    };
-
     private final Map<String, FilterRef> scope;
 
     public RuleScope() {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/DeleteRoleMappingResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/DeleteRoleMappingResponseTests.java
index d89deb44e9f68..1eee41bc23685 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/DeleteRoleMappingResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/DeleteRoleMappingResponseTests.java
@@ -35,15 +35,7 @@ public class DeleteRoleMappingResponseTests extends ESTestCase {
     public void testFromXContent() throws IOException {
         final String json = "{ \"found\" : \"true\" }";
         final DeleteRoleMappingResponse response = DeleteRoleMappingResponse.fromXContent(XContentType.JSON.xContent().createParser(
-                new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                    @Override
-                    public void usedDeprecatedName(String usedName, String modernName) {
-                    }
-
-                    @Override
-                    public void usedDeprecatedField(String usedName, String replacedWith) {
-                    }
-                }, json));
+                new NamedXContentRegistry(Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json));
         final DeleteRoleMappingResponse expectedResponse = new DeleteRoleMappingResponse(true);
         assertThat(response, equalTo(expectedResponse));
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ExpressionRoleMappingTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ExpressionRoleMappingTests.java
index f30307ebde51a..3dd9d3ca5ee53 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ExpressionRoleMappingTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ExpressionRoleMappingTests.java
@@ -37,29 +37,21 @@
 public class ExpressionRoleMappingTests extends ESTestCase {
 
     public void testExpressionRoleMappingParser() throws IOException {
-        final String json = 
-                "{\n" + 
-                "   \"enabled\" : true,\n" + 
-                "   \"roles\" : [\n" + 
-                "     \"superuser\"\n" + 
-                "   ],\n" + 
-                "   \"rules\" : {\n" + 
-                "     \"field\" : {\n" + 
-                "       \"realm.name\" : \"kerb1\"\n" + 
-                "     }\n" + 
-                "   },\n" + 
-                "   \"metadata\" : { }\n" + 
+        final String json =
+                "{\n" +
+                "   \"enabled\" : true,\n" +
+                "   \"roles\" : [\n" +
+                "     \"superuser\"\n" +
+                "   ],\n" +
+                "   \"rules\" : {\n" +
+                "     \"field\" : {\n" +
+                "       \"realm.name\" : \"kerb1\"\n" +
+                "     }\n" +
+                "   },\n" +
+                "   \"metadata\" : { }\n" +
                 " }";
         final ExpressionRoleMapping expressionRoleMapping = ExpressionRoleMapping.PARSER.parse(XContentType.JSON.xContent().createParser(
-                new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                    @Override
-                    public void usedDeprecatedName(String usedName, String modernName) {
-                    }
-
-                    @Override
-                    public void usedDeprecatedField(String usedName, String replacedWith) {
-                    }
-                }, json), "example-role-mapping");
+                new NamedXContentRegistry(Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json), "example-role-mapping");
         final ExpressionRoleMapping expectedRoleMapping = new ExpressionRoleMapping("example-role-mapping",
             FieldRoleMapperExpression.ofKeyValues("realm.name", "kerb1"),
             singletonList("superuser"), Collections.emptyList(),
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetPrivilegesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetPrivilegesResponseTests.java
index bf55e224095fe..ce17c9b1105d8 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetPrivilegesResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetPrivilegesResponseTests.java
@@ -81,15 +81,7 @@ public void testFromXContent() throws IOException {
             "}";
 
         final GetPrivilegesResponse response = GetPrivilegesResponse.fromXContent(XContentType.JSON.xContent().createParser(
-            new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                @Override
-                public void usedDeprecatedName(String usedName, String modernName) {
-                }
-
-                @Override
-                public void usedDeprecatedField(String usedName, String replacedWith) {
-                }
-            }, json));
+            new NamedXContentRegistry(Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json));
 
         final ApplicationPrivilege readTestappPrivilege =
             new ApplicationPrivilege("testapp", "read", Arrays.asList("action:login", "data:read/*"), null);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRoleMappingsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRoleMappingsResponseTests.java
index 20883b859f9ae..4fda57f2ff631 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRoleMappingsResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRoleMappingsResponseTests.java
@@ -36,42 +36,34 @@
 public class GetRoleMappingsResponseTests extends ESTestCase {
 
     public void testFromXContent() throws IOException {
-        final String json = "{\n" + 
-                " \"kerberosmapping\" : {\n" + 
-                "   \"enabled\" : true,\n" + 
-                "   \"roles\" : [\n" + 
-                "     \"superuser\"\n" + 
-                "   ],\n" + 
-                "   \"rules\" : {\n" + 
-                "     \"field\" : {\n" + 
-                "       \"realm.name\" : \"kerb1\"\n" + 
-                "     }\n" + 
-                "   },\n" + 
-                "   \"metadata\" : { }\n" + 
-                " },\n" + 
-                " \"ldapmapping\" : {\n" + 
-                "   \"enabled\" : false,\n" + 
-                "   \"roles\" : [\n" + 
-                "     \"monitoring\"\n" + 
-                "   ],\n" + 
-                "   \"rules\" : {\n" + 
-                "     \"field\" : {\n" + 
-                "       \"groups\" : \"cn=ipausers,cn=groups,cn=accounts,dc=ipademo,dc=local\"\n" + 
-                "     }\n" + 
-                "   },\n" + 
-                "   \"metadata\" : { }\n" + 
-                " }\n" + 
+        final String json = "{\n" +
+                " \"kerberosmapping\" : {\n" +
+                "   \"enabled\" : true,\n" +
+                "   \"roles\" : [\n" +
+                "     \"superuser\"\n" +
+                "   ],\n" +
+                "   \"rules\" : {\n" +
+                "     \"field\" : {\n" +
+                "       \"realm.name\" : \"kerb1\"\n" +
+                "     }\n" +
+                "   },\n" +
+                "   \"metadata\" : { }\n" +
+                " },\n" +
+                " \"ldapmapping\" : {\n" +
+                "   \"enabled\" : false,\n" +
+                "   \"roles\" : [\n" +
+                "     \"monitoring\"\n" +
+                "   ],\n" +
+                "   \"rules\" : {\n" +
+                "     \"field\" : {\n" +
+                "       \"groups\" : \"cn=ipausers,cn=groups,cn=accounts,dc=ipademo,dc=local\"\n" +
+                "     }\n" +
+                "   },\n" +
+                "   \"metadata\" : { }\n" +
+                " }\n" +
                 "}";
         final GetRoleMappingsResponse response = GetRoleMappingsResponse.fromXContent(XContentType.JSON.xContent().createParser(
-                new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                    @Override
-                    public void usedDeprecatedName(String usedName, String modernName) {
-                    }
-
-                    @Override
-                    public void usedDeprecatedField(String usedName, String replacedWith) {
-                    }
-                }, json));
+                new NamedXContentRegistry(Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json));
         final List<ExpressionRoleMapping> expectedRoleMappingsList = new ArrayList<>();
         expectedRoleMappingsList.add(new ExpressionRoleMapping("kerberosmapping", FieldRoleMapperExpression.ofKeyValues("realm.name",
                 "kerb1"), Collections.singletonList("superuser"), Collections.emptyList(), null, true));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRolesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRolesResponseTests.java
index c4620fa1a2f3d..8be4320e4de7c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRolesResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRolesResponseTests.java
@@ -64,15 +64,7 @@ public void testFromXContent() throws IOException {
                 "  }\n" +
                 "}";
         final GetRolesResponse response = GetRolesResponse.fromXContent((XContentType.JSON.xContent().createParser(
-            new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                @Override
-                public void usedDeprecatedName(String usedName, String modernName) {
-                }
-
-                @Override
-                public void usedDeprecatedField(String usedName, String replacedWith) {
-                }
-            }, json)));
+            new NamedXContentRegistry(Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json)));
         assertThat(response.getRoles().size(), equalTo(1));
         assertThat(response.getTransientMetadataMap().size(), equalTo(1));
         final Role role = response.getRoles().get(0);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/support/expressiondsl/parser/RoleMapperExpressionParserTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/support/expressiondsl/parser/RoleMapperExpressionParserTests.java
index 24ed5684fa856..1c9ccaa2f7490 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/support/expressiondsl/parser/RoleMapperExpressionParserTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/support/expressiondsl/parser/RoleMapperExpressionParserTests.java
@@ -115,15 +115,7 @@ private <T> T checkExpressionType(RoleMapperExpression expr, Class<T> type) {
 
     private RoleMapperExpression parse(String json) throws IOException {
         return new RoleMapperExpressionParser().parse("rules", XContentType.JSON.xContent().createParser(new NamedXContentRegistry(
-                Collections.emptyList()), new DeprecationHandler() {
-                    @Override
-                    public void usedDeprecatedName(String usedName, String modernName) {
-                    }
-
-                    @Override
-                    public void usedDeprecatedField(String usedName, String replacedWith) {
-                    }
-                }, json));
+                Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json));
     }
 
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/user/privileges/ApplicationPrivilegeTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/user/privileges/ApplicationPrivilegeTests.java
index b720187673023..532ff612fedc1 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/user/privileges/ApplicationPrivilegeTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/user/privileges/ApplicationPrivilegeTests.java
@@ -54,15 +54,7 @@ public void testFromXContentAndToXContent() throws IOException {
                 + "  }\n"
                 + "}";
         final ApplicationPrivilege privilege = ApplicationPrivilege.fromXContent(XContentType.JSON.xContent().createParser(
-            new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                @Override
-                public void usedDeprecatedName(String usedName, String modernName) {
-                }
-
-                @Override
-                public void usedDeprecatedField(String usedName, String replacedWith) {
-                }
-            }, json));
+            new NamedXContentRegistry(Collections.emptyList()), DeprecationHandler.IGNORE_DEPRECATIONS, json));
         final Map<String, Object> metadata = new HashMap<>();
         metadata.put("description", "Read access to myapp");
         final ApplicationPrivilege expectedPrivilege =
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
index 084d82372c0ce..f4f12ed4f4c81 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
@@ -34,6 +34,7 @@ public class ParseField {
     private final String[] deprecatedNames;
     private String allReplacedWith = null;
     private final String[] allNames;
+    private boolean fullyDeprecated = false;
 
     private static final String[] EMPTY = new String[0];
 
@@ -96,6 +97,15 @@ public ParseField withAllDeprecated(String allReplacedWith) {
         return parseField;
     }
 
+    /**
+     * Return a new ParseField where all field names are deprecated with no replacement
+     */
+    public ParseField withAllDeprecated() {
+        ParseField parseField = this.withDeprecation(getAllNamesIncludedDeprecated());
+        parseField.fullyDeprecated = true;
+        return parseField;
+    }
+
     /**
      * Does {@code fieldName} match this field?
      * @param fieldName
@@ -108,7 +118,7 @@ public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
         Objects.requireNonNull(fieldName, "fieldName cannot be null");
         // if this parse field has not been completely deprecated then try to
         // match the preferred name
-        if (allReplacedWith == null && fieldName.equals(name)) {
+        if (fullyDeprecated == false && allReplacedWith == null && fieldName.equals(name)) {
             return true;
         }
         // Now try to match against one of the deprecated names. Note that if
@@ -116,7 +126,9 @@ public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
         // fields will be in the deprecatedNames array
         for (String depName : deprecatedNames) {
             if (fieldName.equals(depName)) {
-                if (allReplacedWith == null) {
+                if (fullyDeprecated) {
+                    deprecationHandler.usedDeprecatedField(fieldName);
+                } else if (allReplacedWith == null) {
                     deprecationHandler.usedDeprecatedName(fieldName, name);
                 } else {
                     deprecationHandler.usedDeprecatedField(fieldName, allReplacedWith);
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
index 1b0dcf4568086..2cfda82eb4c48 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
@@ -41,6 +41,32 @@ public void usedDeprecatedName(String usedName, String modernName) {
             throw new UnsupportedOperationException("deprecated fields not supported here but got ["
                 + usedName + "] which has been replaced with [" + modernName + "]");
         }
+
+        @Override
+        public void usedDeprecatedField(String usedName) {
+            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                + usedName + "] which has been deprecated entirely");
+        }
+    };
+
+    /**
+     * Ignores all deprecations
+     */
+    DeprecationHandler IGNORE_DEPRECATIONS = new DeprecationHandler() {
+        @Override
+        public void usedDeprecatedName(String usedName, String modernName) {
+
+        }
+
+        @Override
+        public void usedDeprecatedField(String usedName, String replacedWith) {
+
+        }
+
+        @Override
+        public void usedDeprecatedField(String usedName) {
+
+        }
     };
 
     /**
@@ -52,9 +78,16 @@ public void usedDeprecatedName(String usedName, String modernName) {
 
     /**
      * Called when the provided field name matches the current field but the entire
-     * field has been marked as deprecated.
+     * field has been marked as deprecated and another field should be used
      * @param usedName the provided field name
      * @param replacedWith the name of the field that replaced this field
      */
     void usedDeprecatedField(String usedName, String replacedWith);
+
+    /**
+     * Called when the provided field name matches the current field but the entire
+     * field has been marked as deprecated with no replacement
+     * @param usedName the provided field name
+     */
+    void usedDeprecatedField(String usedName);
 }
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
index 72ba5578a49cb..381c7c3e959be 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
@@ -60,6 +60,21 @@ public void testAllDeprecated() {
         assertWarnings("Deprecated field [like_text] used, replaced by [like]");
     }
 
+    public void testDeprecatedWithNoReplacement() {
+        String name = "dep";
+        String[] alternatives = new String[]{"old_dep", "new_dep"};
+        ParseField field = new ParseField(name).withDeprecation(alternatives).withAllDeprecated();
+        assertFalse(field.match("not a field name", LoggingDeprecationHandler.INSTANCE));
+        assertTrue(field.match("dep", LoggingDeprecationHandler.INSTANCE));
+        assertWarnings("Deprecated field [dep] used, which has been removed entirely");
+        assertTrue(field.match("old_dep", LoggingDeprecationHandler.INSTANCE));
+        assertWarnings("Deprecated field [old_dep] used, which has been removed entirely");
+        assertTrue(field.match("new_dep", LoggingDeprecationHandler.INSTANCE));
+        assertWarnings("Deprecated field [new_dep] used, which has been removed entirely");
+
+
+    }
+
     public void testGetAllNamesIncludedDeprecated() {
         ParseField parseField = new ParseField("terms", "in");
         assertThat(parseField.getAllNamesIncludedDeprecated(), arrayContainingInAnyOrder("terms", "in"));
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
index 097093ccce5a2..9a885b55d146c 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
@@ -33,7 +33,7 @@
  * though the user sent them.
  */
 public class LoggingDeprecationHandler implements DeprecationHandler {
-    public static LoggingDeprecationHandler INSTANCE = new LoggingDeprecationHandler();
+    public static final LoggingDeprecationHandler INSTANCE = new LoggingDeprecationHandler();
     /**
      * The logger to which to send deprecation messages.
      *
@@ -57,4 +57,9 @@ public void usedDeprecatedName(String usedName, String modernName) {
     public void usedDeprecatedField(String usedName, String replacedWith) {
         deprecationLogger.deprecated("Deprecated field [{}] used, replaced by [{}]", usedName, replacedWith);
     }
+
+    @Override
+    public void usedDeprecatedField(String usedName) {
+        deprecationLogger.deprecated("Deprecated field [{}] used, which has been removed entirely", usedName);
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
index 2322b3d90262b..5e2a4d7a25302 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
@@ -39,6 +39,13 @@ public void usedDeprecatedField(String usedName, String replacedWith) {
             new Object[] {usedName, replacedWith}));
     }
 
+    @Override
+    public void usedDeprecatedField(String usedName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName);
+        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, which has been deprecated entirely",
+            new Object[]{ usedName }));
+    }
+
     /**
      * The collected deprecation warnings
      */
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 021984d073a23..f36ce90660a23 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -628,6 +628,12 @@ public void usedDeprecatedField(String usedName, String replacedWith) {
             deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], replaced by [{}]",
                 usedName, apiKeyId, replacedWith);
         }
+
+        @Override
+        public void usedDeprecatedField(String usedName) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], which has been removed entirely",
+                usedName);
+        }
     }
 
     /**

From 97cf64f6aa6e04df3bf9bedc32f32a4fe64fce71 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Wed, 18 Mar 2020 16:15:35 +0100
Subject: [PATCH 109/285] [TEST] Replace agg key in async search yaml test
 (#53727)

Some clients have problems running this test as a numeric key is treated like an array index by default.
We can work around this by renaming the aggregation key to not be a numeric.
---
 .../rest-api-spec/test/async_search/10_basic.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
index ef457a569ad0e..292e3bd6ccec6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
@@ -48,7 +48,7 @@
           query:
             match_all: {}
           aggs:
-            1:
+            max:
               max:
                 field: max
           sort: max
@@ -58,7 +58,7 @@
   - match:  { is_partial:                   false }
   - length: { response.hits.hits:               3 }
   - match:  { response.hits.hits.0._source.max: 1 }
-  - match:  { response.aggregations.1.value:  3.0 }
+  - match:  { response.aggregations.max.value:  3.0 }
 
   - do:
       async_search.submit:
@@ -68,7 +68,7 @@
         clean_on_completion: false
         body:
           aggs:
-            1:
+            max:
               max:
                 field: max
           sort: max
@@ -78,7 +78,7 @@
   - match:  { is_partial:                   false }
   - length: { response.hits.hits:               3 }
   - match:  { response.hits.hits.0._source.max: 1 }
-  - match:  { response.aggregations.1.value:  3.0 }
+  - match:  { response.aggregations.max.value:  3.0 }
 
   # test with typed_keys:
   - do:
@@ -90,7 +90,7 @@
         typed_keys: true
         body:
           aggs:
-            1:
+            max:
               max:
                 field: max
           sort: max
@@ -100,7 +100,7 @@
   - match:  { is_partial:                       false }
   - length: { response.hits.hits:                   3 }
   - match:  { response.hits.hits.0._source.max:     1 }
-  - match:  { response.aggregations.max#1.value:  3.0 }
+  - match:  { response.aggregations.max#max.value:  3.0 }
 
   - do:
       async_search.get:
@@ -110,7 +110,7 @@
   - match:  { is_partial:                     false }
   - length: { response.hits.hits:                 3 }
   - match:  { response.hits.hits.0._source.max:   1 }
-  - match:  { response.aggregations.1.value:    3.0 }
+  - match:  { response.aggregations.max.value:    3.0 }
 
   # test with typed_keys:
   - do:
@@ -122,7 +122,7 @@
   - match:  { is_partial:                       false }
   - length: { response.hits.hits:                   3 }
   - match:  { response.hits.hits.0._source.max:     1 }
-  - match:  { response.aggregations.max#1.value:  3.0 }
+  - match:  { response.aggregations.max#max.value:  3.0 }
 
   - do:
       async_search.delete:

From ddaf3534d908630900f4814806aea1de8f1ebecc Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Wed, 18 Mar 2020 15:18:49 +0000
Subject: [PATCH 110/285] Use QueryVisitor when extracting PercolatorQuery list
 for highlighting (#53728)

The highlighting phase for percolator queries currently uses some custom query
traversal logic to find all instances of PercolatorQuery in the query tree for the
current search context. This commit converts things to instead use a QueryVisitor,
which future-proofs us against new wrapper queries or queries from custom
plugins that the percolator module doesn't know about.
---
 .../PercolatorHighlightSubFetchPhase.java     | 44 +++++--------------
 ...PercolatorHighlightSubFetchPhaseTests.java |  8 +++-
 2 files changed, 18 insertions(+), 34 deletions(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index d13a54cba0467..cf76f531b6d66 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -21,16 +21,11 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.document.DocumentField;
-import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -139,33 +134,18 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
     }
 
     static List<PercolateQuery> locatePercolatorQuery(Query query) {
-        if (query instanceof PercolateQuery) {
-            return Collections.singletonList((PercolateQuery) query);
-        } else if (query instanceof BooleanQuery) {
-            List<PercolateQuery> percolateQueries = new ArrayList<>();
-            for (BooleanClause clause : ((BooleanQuery) query).clauses()) {
-                List<PercolateQuery> result = locatePercolatorQuery(clause.getQuery());
-                if (result.isEmpty() == false) {
-                    percolateQueries.addAll(result);
-                }
-            }
-            return percolateQueries;
-        } else if (query instanceof DisjunctionMaxQuery) {
-            List<PercolateQuery> percolateQueries = new ArrayList<>();
-            for (Query disjunct : ((DisjunctionMaxQuery) query).getDisjuncts()) {
-                List<PercolateQuery> result = locatePercolatorQuery(disjunct);
-                if (result.isEmpty() == false) {
-                    percolateQueries.addAll(result);
+        if (query == null) {
+            return Collections.emptyList();
+        }
+        List<PercolateQuery> queries = new ArrayList<>();
+        query.visit(new QueryVisitor() {
+            @Override
+            public void visitLeaf(Query query) {
+                if (query instanceof PercolateQuery) {
+                    queries.add((PercolateQuery)query);
                 }
             }
-            return  percolateQueries;
-        } else if (query instanceof ConstantScoreQuery) {
-            return locatePercolatorQuery(((ConstantScoreQuery) query).getQuery());
-        } else if (query instanceof BoostQuery) {
-            return locatePercolatorQuery(((BoostQuery) query).getQuery());
-        } else if (query instanceof FunctionScoreQuery) {
-            return locatePercolatorQuery(((FunctionScoreQuery) query).getSubQuery());
-        }
-        return Collections.emptyList();
+        });
+        return queries;
     }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index 291a42c14665f..09dd2e67e9901 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -37,6 +37,7 @@
 import java.util.Collections;
 
 import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
@@ -99,8 +100,11 @@ public void testLocatePercolatorQuery() {
         bq.add(percolateQuery, BooleanClause.Occur.FILTER);
         bq.add(percolateQuery2, BooleanClause.Occur.FILTER);
         assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(bq.build()).size(), equalTo(2));
-        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(bq.build()).get(0), sameInstance(percolateQuery));
-        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(bq.build()).get(1), sameInstance(percolateQuery2));
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(bq.build()),
+            containsInAnyOrder(sameInstance(percolateQuery), sameInstance(percolateQuery2)));
+
+        assertNotNull(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(null));
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(null).size(), equalTo(0));
     }
 
 }

From c89ba8a0963047812ccbd662bcd033fe5fcdec19 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Wed, 18 Mar 2020 11:20:55 -0400
Subject: [PATCH 111/285] [DOCS] Add temporary redirect for missing Component
 Template API docs (#53735)

The following API spec files contain a link to a not-yet-created
doc page for the component template APIs:

* [cluster.delete_component_template.json][0]
* [cluster.get_component_template.json][1]
* [cluster.put_component_template.json][2]

The Elaticsearch-js client uses these spec files to create their docs.
This created a broken link in the Elaticsearch-js, which has broken
the docs build.

This PR adds a temporary redirect for the docs page. This redirect
should be removed when the actual API docs are added.

[0]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
[1]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
[2]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
---
 docs/reference/redirects.asciidoc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 9ab175ee4d8e4..065450560d0a2 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -380,3 +380,8 @@ See <<ccs-gateway-seed-nodes>> and <<ccs-min-roundtrips>>.
 === Asynchronous search
 
 coming::[7.x]
+
+[role="exclude",id="indices-component-templates"]
+=== Component template APIs
+
+coming::[7.x]

From f605ec2d3c212fe740d8ac18c47cbf92090248a7 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Wed, 18 Mar 2020 08:29:18 -0700
Subject: [PATCH 112/285] [DOCS] Adds stub for cat transform API (#53737)

---
 docs/reference/cat.asciidoc           | 14 ++++++++------
 docs/reference/cat/transform.asciidoc | 28 +++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 6 deletions(-)
 create mode 100644 docs/reference/cat/transform.asciidoc

diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc
index d557a8c930a20..62d5165984aa5 100644
--- a/docs/reference/cat.asciidoc
+++ b/docs/reference/cat.asciidoc
@@ -255,16 +255,18 @@ include::cat/recovery.asciidoc[]
 
 include::cat/repositories.asciidoc[]
 
-include::cat/tasks.asciidoc[]
-
-include::cat/thread_pool.asciidoc[]
-
-include::cat/trainedmodel.asciidoc[]
-
 include::cat/shards.asciidoc[]
 
 include::cat/segments.asciidoc[]
 
 include::cat/snapshots.asciidoc[]
 
+include::cat/tasks.asciidoc[]
+
 include::cat/templates.asciidoc[]
+
+include::cat/thread_pool.asciidoc[]
+
+include::cat/trainedmodel.asciidoc[]
+
+include::cat/transform.asciidoc[]
diff --git a/docs/reference/cat/transform.asciidoc b/docs/reference/cat/transform.asciidoc
new file mode 100644
index 0000000000000..835106c08b496
--- /dev/null
+++ b/docs/reference/cat/transform.asciidoc
@@ -0,0 +1,28 @@
+[[cat-transform]]
+=== cat transform API
+++++
+<titleabbrev>cat transform</titleabbrev>
+++++
+
+Returns configuration and usage information about {transforms}.
+
+
+[[cat-transform-api-request]]
+==== {api-request-title}
+
+`GET /_cat/transform`
+
+
+//[[cat-transform-api-desc]]
+//==== {api-description-title}
+
+
+//[[cat-transform-api-query-params]]
+//==== {api-query-parms-title}
+
+
+//[[cat-transform-api-response-codes]]
+//==== {api-response-codes-title}
+
+//[[cat-transform-api-examples]]
+//==== {api-examples-title}

From a29a83e29cbb25f16e61223bc418491097744821 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Wed, 18 Mar 2020 17:31:36 +0200
Subject: [PATCH 113/285] Mute TimeSeriesLifecycleActionsIT (#53740)

see #53738
---
 .../xpack/ilm/TimeSeriesLifecycleActionsIT.java    | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 14c99d471e983..979fc447db172 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -10,6 +10,7 @@
 import org.apache.http.entity.StringEntity;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
@@ -75,6 +76,7 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53738")
 public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
     private static final Logger logger = LogManager.getLogger(TimeSeriesLifecycleActionsIT.class);
     private static final String FAILED_STEP_RETRY_COUNT_FIELD = "failed_step_retry_count";
@@ -94,7 +96,7 @@ public void refreshIndex() {
     public static void updatePolicy(String indexName, String policy) throws IOException {
         Request changePolicyRequest = new Request("PUT", "/" + indexName + "/_settings");
         final StringEntity changePolicyEntity = new StringEntity("{ \"index.lifecycle.name\": \"" + policy + "\" }",
-                ContentType.APPLICATION_JSON);
+            ContentType.APPLICATION_JSON);
         changePolicyRequest.setEntity(changePolicyEntity);
         assertOK(client().performRequest(changePolicyRequest));
     }
@@ -649,9 +651,9 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception
         createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
 
-       ensureGreen(index);
+        ensureGreen(index);
 
-       // unallocate all index shards
+        // unallocate all index shards
         Request setAllocationToMissingAttribute = new Request("PUT", "/" + index + "/_settings");
         setAllocationToMissingAttribute.setJsonEntity("{\n" +
             "  \"settings\": {\n" +
@@ -1326,8 +1328,8 @@ public void testWaitForActiveShardsStep() throws Exception {
         String originalIndex = index + "-000001";
         String secondIndex = index + "-000002";
         createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias),
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias),
             true);
 
         // create policy
@@ -1807,7 +1809,7 @@ private void createSlmPolicy(String smlPolicy, String repo) throws IOException {
                 .field("repository", repo)
                 .field("name", "snap" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT))
                 .startObject("config")
-                    .field("include_global_state", false)
+                .field("include_global_state", false)
                 .endObject()
                 .endObject()));
 

From 024deb41a432f351199ecc89dd3ac5932c2599be Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Wed, 18 Mar 2020 15:38:00 +0000
Subject: [PATCH 114/285] Wildcard field docs formatting fix

Bullet points weren't rendering correctly
---
 docs/reference/mapping/types/wildcard.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
index 51d10ff53ca92..9d01f4820a894 100644
--- a/docs/reference/mapping/types/wildcard.asciidoc
+++ b/docs/reference/mapping/types/wildcard.asciidoc
@@ -8,6 +8,7 @@
 
 A `wildcard` field stores values optimised for wildcard grep-like queries.
 Wildcard queries are possible on other field types but suffer from constraints:
+
 * `text` fields limit matching of any wildcard expressions to individual tokens rather than the original whole value held in a field
 * `keyword` fields are untokenized but slow at performing wildcard queries (especially patterns with leading wildcards).
 

From d1cbdfb753045a0eafebefe22cf7aa6b235a3da3 Mon Sep 17 00:00:00 2001
From: Dominic Page <11043991+djptek@users.noreply.github.com>
Date: Wed, 18 Mar 2020 17:03:52 +0100
Subject: [PATCH 115/285] Geo shape query vs geo point (#52382)

Enable geo_shape query to work on geo_point fields for shapes: circle, polygon, multipolygon, rectangle

see: #48928

Co-Authored-By:  @iverase
---
 docs/reference/query-dsl/geo-queries.asciidoc |   8 +-
 .../query-dsl/geo-shape-query.asciidoc        |  20 +-
 .../elasticsearch/geometry/LinearRing.java    |   2 +-
 .../mapper/AbstractGeometryFieldMapper.java   |  28 +-
 .../AbstractSearchableGeometryFieldType.java  |  63 +++
 .../index/mapper/GeoPointFieldMapper.java     |  14 +-
 .../index/mapper/GeoShapeIndexer.java         |   3 +-
 .../query/AbstractGeometryQueryBuilder.java   |   8 +-
 .../index/query/GeoShapeQueryBuilder.java     |  28 +-
 .../query/LegacyGeoShapeQueryProcessor.java   |   3 +-
 .../VectorGeoPointShapeQueryProcessor.java    | 191 +++++++++
 .../query/VectorGeoShapeQueryProcessor.java   |  11 +-
 .../common/geo/GeometryIndexerTests.java      |   3 +-
 .../GeoShapeQueryBuilderGeoPointTests.java    |  65 +++
 .../GeoShapeQueryBuilderGeoShapeTests.java    |  84 ++++
 .../query/GeoShapeQueryBuilderTests.java      |  63 +--
 .../search/geo/GeoPointShapeQueryTests.java   | 141 ++++++-
 .../search/geo/GeoQueryTests.java             | 397 ++++++++++++++++--
 .../search/geo/GeoShapeQueryTests.java        |   4 -
 .../index/query/ShapeQueryBuilder.java        |  19 +-
 .../index/query/ShapeQueryProcessor.java      |   4 +-
 21 files changed, 1002 insertions(+), 157 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/AbstractSearchableGeometryFieldType.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java

diff --git a/docs/reference/query-dsl/geo-queries.asciidoc b/docs/reference/query-dsl/geo-queries.asciidoc
index b3cc9112576cc..b4eb86763e702 100644
--- a/docs/reference/query-dsl/geo-queries.asciidoc
+++ b/docs/reference/query-dsl/geo-queries.asciidoc
@@ -18,9 +18,11 @@ Finds documents with geo-points within the specified distance of a central point
 Find documents with geo-points within the specified polygon.
 
 <<query-dsl-geo-shape-query,`geo_shape`>> query::
-Finds documents with geo-shapes which either intersect, are contained by, or do not intersect with the specified
-geo-shape.
-
+Finds documents with:
+* `geo-shapes` which either intersect, are contained by, or do not intersect
+with the specified geo-shape
+* `geo-points` which intersect the specified
+geo-shape
 
 include::geo-bounding-box-query.asciidoc[]
 
diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc
index 9706b90d82845..966267d4e3537 100644
--- a/docs/reference/query-dsl/geo-shape-query.asciidoc
+++ b/docs/reference/query-dsl/geo-shape-query.asciidoc
@@ -4,9 +4,9 @@
 <titleabbrev>Geo-shape</titleabbrev>
 ++++
 
-Filter documents indexed using the `geo_shape` type.
+Filter documents indexed using the `geo_shape` or `geo_point` type.
 
-Requires the <<geo-shape,`geo_shape` Mapping>>.
+Requires the <<geo-shape,`geo_shape` Mapping>> or the <<geo-point,`geo_point` Mapping>>.
 
 The `geo_shape` query uses the same grid square representation as the
 `geo_shape` mapping to find documents that have a shape that intersects
@@ -142,7 +142,7 @@ GET /example/_search
 The <<spatial-strategy, geo_shape strategy>> mapping parameter determines
 which spatial relation operators may be used at search time.
 
-The following is a complete list of spatial relation operators available:
+The following is a complete list of spatial relation operators available when searching a field of type `geo_shape`:
 
 * `INTERSECTS` - (default) Return all documents whose `geo_shape` field
 intersects the query geometry.
@@ -153,6 +153,11 @@ is within the query geometry.
 * `CONTAINS` - Return all documents whose `geo_shape` field
 contains the query geometry.
 
+When searching a field of type `geo_point` there is a single supported spatial relation operator:
+
+* `INTERSECTS` - (default) Return all documents whose `geo_point` field
+intersects the query geometry.
+
 [float]
 ==== Ignore Unmapped
 
@@ -162,6 +167,15 @@ querying multiple indexes which might have different mappings. When set to
 `false` (the default value) the query will throw an exception if the field
 is not mapped.
 
+==== Shape Types supported for Geo-Point
+
+When searching a field of type `geo_point` the following shape types are not supported:
+
+* `POINT`
+* `LINE`
+* `MULTIPOINT`
+* `MULTILINE`
+
 ==== Notes
 Geo-shape queries on geo-shapes implemented with <<prefix-trees, `PrefixTrees`>> will not be executed if
 <<query-dsl-allow-expensive-queries, `search.allow_expensive_queries`>> is set to false.
diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java b/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java
index c0d0150c3023e..babe02bdf55af 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java
@@ -64,6 +64,6 @@ public <T, E extends Exception> T visit(GeometryVisitor<T, E> visitor) throws E
     public String toString() {
         return "linearring(x=" + Arrays.toString(getX()) +
             ", y=" + Arrays.toString(getY()) +
-            (hasZ() ? ", z=" + Arrays.toString(getZ()) : "");
+            (hasZ() ? ", z=" + Arrays.toString(getZ()) : "") + ")";
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
index 006af39b8f59b..d065e8121dc05 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
@@ -26,8 +26,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation;
 import org.elasticsearch.common.settings.Settings;
@@ -35,7 +33,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.DeprecatedParameters;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
@@ -91,19 +88,6 @@ public interface Parser<Parsed> {
 
     }
 
-    /**
-     * interface representing a query builder that generates a query from the given shape
-     */
-    public interface QueryProcessor {
-        Query process(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context);
-
-        @Deprecated
-        default Query process(Geometry shape, String fieldName, SpatialStrategy strategy, ShapeRelation relation,
-                              QueryShardContext context) {
-            return process(shape, fieldName, relation, context);
-        }
-    }
-
     public abstract static class Builder<T extends Builder, Y extends AbstractGeometryFieldMapper>
             extends FieldMapper.Builder<T, Y> {
         protected Boolean coerce;
@@ -272,14 +256,14 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
         }
     }
 
-    public abstract static class AbstractGeometryFieldType<Parsed, Processed> extends MappedFieldType {
+    public abstract static class AbstractGeometryFieldType<Parsed, Processed> extends AbstractSearchableGeometryFieldType {
         protected Orientation orientation = Defaults.ORIENTATION.value();
 
         protected Indexer<Parsed, Processed> geometryIndexer;
 
         protected Parser<Parsed> geometryParser;
 
-        protected QueryProcessor geometryQueryBuilder;
+
 
         protected AbstractGeometryFieldType() {
             setIndexOptions(IndexOptions.DOCS);
@@ -339,14 +323,6 @@ public void setGeometryParser(Parser<Parsed> geometryParser)  {
         protected Parser<Parsed> geometryParser() {
             return geometryParser;
         }
-
-        public void setGeometryQueryBuilder(QueryProcessor geometryQueryBuilder)  {
-            this.geometryQueryBuilder = geometryQueryBuilder;
-        }
-
-        public QueryProcessor geometryQueryBuilder() {
-            return geometryQueryBuilder;
-        }
     }
 
     protected Explicit<Boolean> coerce;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractSearchableGeometryFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractSearchableGeometryFieldType.java
new file mode 100644
index 0000000000000..29154bfc36548
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractSearchableGeometryFieldType.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.SpatialStrategy;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.index.query.QueryShardContext;
+
+/**
+ * a base class for geometry types that support shape query builder
+ */
+public abstract class AbstractSearchableGeometryFieldType extends MappedFieldType {
+
+    protected QueryProcessor geometryQueryBuilder;
+
+    protected AbstractSearchableGeometryFieldType() {
+    }
+
+    protected AbstractSearchableGeometryFieldType(AbstractSearchableGeometryFieldType ref) {
+        super(ref);
+    }
+
+    public void setGeometryQueryBuilder(QueryProcessor geometryQueryBuilder)  {
+        this.geometryQueryBuilder = geometryQueryBuilder;
+    }
+
+    public QueryProcessor geometryQueryBuilder() {
+        return geometryQueryBuilder;
+    }
+
+    /**
+     * interface representing a query builder that generates a query from the given shape
+     */
+    public interface QueryProcessor {
+        Query process(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context);
+
+        @Deprecated
+        default Query process(Geometry shape, String fieldName, SpatialStrategy strategy, ShapeRelation relation,
+                              QueryShardContext context) {
+            return process(shape, fieldName, relation, context);
+        }
+    }
+}
+
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
index 93d8215f21cac..5239331819be3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointDVIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.VectorGeoPointShapeQueryProcessor;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
@@ -123,6 +124,14 @@ public GeoPointFieldMapper build(BuilderContext context, String simpleName, Mapp
                 ignoreMalformed, ignoreZValue, copyTo);
         }
 
+        @Override
+        protected void setupFieldType(BuilderContext context) {
+            super.setupFieldType(context);
+
+            GeoPointFieldType fieldType = (GeoPointFieldType)fieldType();
+            fieldType.setGeometryQueryBuilder(new VectorGeoPointShapeQueryProcessor());
+        }
+
         @Override
         public GeoPointFieldMapper build(BuilderContext context) {
             return build(context, name, fieldType, defaultFieldType, context.indexSettings(),
@@ -213,7 +222,7 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
         throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called");
     }
 
-    public static class GeoPointFieldType extends MappedFieldType {
+    public static class GeoPointFieldType extends AbstractSearchableGeometryFieldType {
         public GeoPointFieldType() {
         }
 
@@ -253,7 +262,8 @@ public Query existsQuery(QueryShardContext context) {
 
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
-            throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: ["
+            throw new QueryShardException(context,
+                "Geo fields do not support exact searching, use dedicated geo queries instead: ["
                 + name() + "]");
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
index 558b4068ffa90..0b7458de58b16 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.geo.GeoLineDecomposer;
 import org.elasticsearch.common.geo.GeoPolygonDecomposer;
+import org.elasticsearch.common.geo.GeoShapeType;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
@@ -64,7 +65,7 @@ public Geometry prepareForIndexing(Geometry geometry) {
         return geometry.visit(new GeometryVisitor<>() {
             @Override
             public Geometry visit(Circle circle) {
-                throw new UnsupportedOperationException("CIRCLE geometry is not supported");
+                throw new UnsupportedOperationException(GeoShapeType.CIRCLE + " geometry is not supported");
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
index 3a996bde56e52..d9c3464f1ba26 100644
--- a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
@@ -337,11 +337,9 @@ public boolean ignoreUnmapped() {
     }
 
     /** list of content types this shape query is compatible with */
-    protected abstract List validContentTypes();
+    protected abstract List<String> validContentTypes();
     /** builds the appropriate lucene shape query */
     protected abstract Query buildShapeQuery(QueryShardContext context, MappedFieldType fieldType);
-    /** returns expected content type for this query */
-    protected abstract String queryFieldType();
     /** writes the xcontent specific to this shape query */
     protected abstract void doShapeQueryXContent(XContentBuilder builder, Params params) throws IOException;
     /** creates a new ShapeQueryBuilder from the provided field name and shape builder */
@@ -365,7 +363,9 @@ protected Query doToQuery(QueryShardContext context) {
             if (ignoreUnmapped) {
                 return new MatchNoDocsQuery();
             } else {
-                throw new QueryShardException(context, "failed to find " + queryFieldType() + " field [" + fieldName + "]");
+                throw new QueryShardException(context, "failed to find "
+                    + String.join(" or ", validContentTypes())
+                    + " field [" + fieldName + "]");
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 78c727a90212b..d5d940b353f90 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -34,12 +34,14 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
+import org.elasticsearch.index.mapper.AbstractSearchableGeometryFieldType;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Supplier;
@@ -56,6 +58,12 @@ public class GeoShapeQueryBuilder extends AbstractGeometryQueryBuilder<GeoShapeQ
 
     private SpatialStrategy strategy;
 
+    protected static final List<String> validContentTypes =
+        Collections.unmodifiableList(
+            Arrays.asList(
+                GeoShapeFieldMapper.CONTENT_TYPE,
+                GeoPointFieldMapper.CONTENT_TYPE));
+
     /**
      * Creates a new GeoShapeQueryBuilder whose Query will be against the given
      * field name using the given Shape
@@ -168,13 +176,8 @@ public SpatialStrategy strategy() {
     }
 
     @Override
-    protected List validContentTypes() {
-        return Arrays.asList(GeoShapeFieldMapper.CONTENT_TYPE);
-    }
-
-    @Override
-    public String queryFieldType() {
-        return GeoShapeFieldMapper.CONTENT_TYPE;
+    protected List<String> validContentTypes() {
+        return validContentTypes;
     }
 
     @Override
@@ -197,12 +200,15 @@ protected GeoShapeQueryBuilder newShapeQueryBuilder(String fieldName, Supplier<G
 
     @Override
     public Query buildShapeQuery(QueryShardContext context, MappedFieldType fieldType) {
-        if (fieldType.typeName().equals(GeoShapeFieldMapper.CONTENT_TYPE) == false) {
+        if (validContentTypes().contains(fieldType.typeName()) == false) {
             throw new QueryShardException(context,
-                "Field [" + fieldName + "] is not of type [" + queryFieldType() + "] but of type [" + fieldType.typeName() + "]");
+                "Field [" + fieldName + "] is of unsupported type [" + fieldType.typeName() + "]. ["
+                + NAME + "] query supports the following types ["
+                + String.join(",", validContentTypes()) +  "]");
         }
 
-        final AbstractGeometryFieldMapper.AbstractGeometryFieldType ft = (AbstractGeometryFieldMapper.AbstractGeometryFieldType) fieldType;
+        final AbstractSearchableGeometryFieldType ft =
+            (AbstractSearchableGeometryFieldType) fieldType;
         return new ConstantScoreQuery(ft.geometryQueryBuilder().process(shape, fieldName, strategy, relation, context));
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/LegacyGeoShapeQueryProcessor.java b/server/src/main/java/org/elasticsearch/index/query/LegacyGeoShapeQueryProcessor.java
index 20bbb65f34f39..018e3443687d4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/LegacyGeoShapeQueryProcessor.java
+++ b/server/src/main/java/org/elasticsearch/index/query/LegacyGeoShapeQueryProcessor.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.geometry.Polygon;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
+import org.elasticsearch.index.mapper.AbstractSearchableGeometryFieldType;
 import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;
 import org.locationtech.jts.geom.Coordinate;
 import org.locationtech.spatial4j.shape.Shape;
@@ -62,7 +63,7 @@
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 
-public class LegacyGeoShapeQueryProcessor implements AbstractGeometryFieldMapper.QueryProcessor {
+public class LegacyGeoShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
 
     private AbstractGeometryFieldMapper.AbstractGeometryFieldType ft;
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java b/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java
new file mode 100644
index 0000000000000..d9b8d5a28b1ab
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.document.LatLonDocValuesField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.geo.GeoPolygonDecomposer;
+import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.geometry.Circle;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.GeometryCollection;
+import org.elasticsearch.geometry.GeometryVisitor;
+import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiLine;
+import org.elasticsearch.geometry.MultiPoint;
+import org.elasticsearch.geometry.MultiPolygon;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.ShapeType;
+import org.elasticsearch.index.mapper.AbstractSearchableGeometryFieldType;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+import java.util.ArrayList;
+
+import static org.elasticsearch.index.mapper.GeoShapeIndexer.toLucenePolygon;
+
+public class VectorGeoPointShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
+
+    @Override
+    public Query process(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+        validateIsGeoPointFieldType(fieldName, context);
+        // geo points only support intersects
+        if (relation != ShapeRelation.INTERSECTS) {
+            throw new QueryShardException(context,
+                relation+ " query relation not supported for Field [" + fieldName + "].");
+        }
+        // wrap geoQuery as a ConstantScoreQuery
+        return getVectorQueryFromShape(shape, fieldName, relation, context);
+    }
+
+    private void validateIsGeoPointFieldType(String fieldName, QueryShardContext context) {
+        MappedFieldType fieldType = context.fieldMapper(fieldName);
+        if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType == false) {
+            throw new QueryShardException(context, "Expected " + GeoPointFieldMapper.CONTENT_TYPE
+                + " field type for Field [" + fieldName + "] but found " + fieldType.typeName());
+        }
+    }
+
+    protected Query getVectorQueryFromShape(
+        Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+        ShapeVisitor shapeVisitor = new ShapeVisitor(context, fieldName, relation);
+        return queryShape.visit(shapeVisitor);
+    }
+
+    private class ShapeVisitor implements GeometryVisitor<Query, RuntimeException> {
+        QueryShardContext context;
+        MappedFieldType fieldType;
+        String fieldName;
+        ShapeRelation relation;
+
+        ShapeVisitor(QueryShardContext context, String fieldName, ShapeRelation relation) {
+            this.context = context;
+            this.fieldType = context.fieldMapper(fieldName);
+            this.fieldName = fieldName;
+            this.relation = relation;
+        }
+
+        @Override
+        public Query visit(Circle circle) {
+            Query query = LatLonPoint.newDistanceQuery(
+                fieldName, circle.getLat(), circle.getLon(), circle.getRadiusMeters());
+            if (fieldType.hasDocValues()) {
+                Query dvQuery = LatLonDocValuesField.newSlowDistanceQuery(
+                    fieldName, circle.getLat(), circle.getLon(), circle.getRadiusMeters());
+                query = new IndexOrDocValuesQuery(query, dvQuery);
+            }
+            return query;
+        }
+
+        @Override
+        public Query visit(GeometryCollection<?> collection) {
+            BooleanQuery.Builder bqb = new BooleanQuery.Builder();
+            visit(bqb, collection);
+            return bqb.build();
+        }
+
+        private void visit(BooleanQuery.Builder bqb, GeometryCollection<?> collection) {
+            BooleanClause.Occur occur = BooleanClause.Occur.FILTER;
+            for (Geometry shape : collection) {
+                bqb.add(shape.visit(this), occur);
+            }
+        }
+
+        @Override
+        public Query visit(org.elasticsearch.geometry.Line line) {
+            throw new QueryShardException(context, "Field [" + fieldName + "] does not support "
+                + GeoShapeType.LINESTRING + " queries");
+        }
+
+        @Override
+        // don't think this is called directly
+        public Query visit(LinearRing ring) {
+            throw new QueryShardException(context, "Field [" + fieldName + "] does not support "
+                + ShapeType.LINEARRING + " queries");
+        }
+
+        @Override
+        public Query visit(MultiLine multiLine) {
+            throw new QueryShardException(context, "Field [" + fieldName + "] does not support "
+                + GeoShapeType.MULTILINESTRING + " queries");
+        }
+
+        @Override
+        public Query visit(MultiPoint multiPoint) {
+            throw new QueryShardException(context, "Field [" + fieldName + "] does not support "
+                + GeoShapeType.MULTIPOINT + " queries");
+        }
+
+        // helper for visit(MultiPolygon multiPolygon) and visit(Polygon polygon)
+        private Query visit(ArrayList<Polygon> collector) {
+            org.apache.lucene.geo.Polygon[] lucenePolygons =
+                new org.apache.lucene.geo.Polygon[collector.size()];
+            for (int i = 0; i < collector.size(); i++) {
+                lucenePolygons[i] = toLucenePolygon(collector.get(i));
+            }
+            Query query = LatLonPoint.newPolygonQuery(fieldName, lucenePolygons);
+            if (fieldType.hasDocValues()) {
+                Query dvQuery = LatLonDocValuesField.newSlowPolygonQuery(fieldName, lucenePolygons);
+                query = new IndexOrDocValuesQuery(query, dvQuery);
+            }
+            return query;
+        }
+
+        @Override
+        public Query visit(MultiPolygon multiPolygon) {
+            ArrayList<org.elasticsearch.geometry.Polygon> collector = new ArrayList<>();
+            GeoPolygonDecomposer.decomposeMultiPolygon(multiPolygon, true, collector);
+            return visit(collector);
+        }
+
+        @Override
+        public Query visit(Point point) {
+            // not currently supported
+            throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.POINT +
+                " queries");
+        }
+
+        @Override
+        public Query visit(Polygon polygon) {
+            ArrayList<org.elasticsearch.geometry.Polygon> collector = new ArrayList<>();
+            GeoPolygonDecomposer.decomposePolygon(polygon, true, collector);
+            return visit(collector);
+        }
+
+        @Override
+        public Query visit(Rectangle r) {
+            Query query = LatLonPoint.newBoxQuery(fieldName, r.getMinY(), r.getMaxY(), r.getMinX(), r.getMaxX());
+            if (fieldType.hasDocValues()) {
+                Query dvQuery = LatLonDocValuesField.newSlowBoxQuery(
+                    fieldName, r.getMinY(), r.getMaxY(), r.getMinX(), r.getMaxX());
+                query = new IndexOrDocValuesQuery(query, dvQuery);
+            }
+            return query;
+        }
+    }
+}
+
diff --git a/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java b/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
index 55ef5098bc4ca..4d79437bf239a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
+++ b/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
@@ -39,14 +39,14 @@
 import org.elasticsearch.geometry.MultiPolygon;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
+import org.elasticsearch.index.mapper.AbstractSearchableGeometryFieldType;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeIndexer;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import static org.elasticsearch.index.mapper.GeoShapeIndexer.toLucenePolygon;
 
-public class VectorGeoShapeQueryProcessor implements AbstractGeometryFieldMapper.QueryProcessor {
+public class VectorGeoShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
 
     @Override
     public Query process(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) {
@@ -59,7 +59,8 @@ public Query process(Geometry shape, String fieldName, ShapeRelation relation, Q
         return getVectorQueryFromShape(shape, fieldName, relation, context);
     }
 
-    protected Query getVectorQueryFromShape(Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+    protected Query getVectorQueryFromShape(
+        Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
         GeoShapeIndexer geometryIndexer = new GeoShapeIndexer(true, fieldName);
 
         Geometry processedShape = geometryIndexer.prepareForIndexing(queryShape);
@@ -85,7 +86,7 @@ private class ShapeVisitor implements GeometryVisitor<Query, RuntimeException> {
 
         @Override
         public Query visit(Circle circle) {
-            throw new QueryShardException(context, "Field [" + fieldName + "] found and unknown shape Circle");
+            throw new QueryShardException(context, "Field [" + fieldName + "] found an unknown shape Circle");
         }
 
         @Override
@@ -117,7 +118,7 @@ public Query visit(org.elasticsearch.geometry.Line line) {
 
         @Override
         public Query visit(LinearRing ring) {
-            throw new QueryShardException(context, "Field [" + fieldName + "] found and unsupported shape LinearRing");
+            throw new QueryShardException(context, "Field [" + fieldName + "] found an unsupported shape LinearRing");
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
index b00b53ec18271..f5828aad13ee1 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.geometry.Polygon;
 import org.elasticsearch.index.mapper.GeoShapeIndexer;
 import org.elasticsearch.test.ESTestCase;
-
 import java.io.IOException;
 import java.text.ParseException;
 import java.util.ArrayList;
@@ -52,7 +51,7 @@ public class GeometryIndexerTests extends ESTestCase {
     public void testCircle() {
         UnsupportedOperationException ex =
             expectThrows(UnsupportedOperationException.class, () -> indexer.prepareForIndexing(new Circle(2, 1, 3)));
-        assertEquals("CIRCLE geometry is not supported", ex.getMessage());
+        assertEquals(GeoShapeType.CIRCLE + " geometry is not supported", ex.getMessage());
     }
 
     public void testCollection() {
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java
new file mode 100644
index 0000000000000..e7fb3a0ece67d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.test.geo.RandomShapeGenerator;
+
+public class GeoShapeQueryBuilderGeoPointTests extends GeoShapeQueryBuilderTests {
+
+    protected String fieldName() {
+        return GEO_POINT_FIELD_NAME;
+    }
+
+    protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) {
+        RandomShapeGenerator.ShapeType shapeType = RandomShapeGenerator.ShapeType.POLYGON;
+        ShapeBuilder<?, ?, ?> shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType);
+        GeoShapeQueryBuilder builder;
+        clearShapeFields();
+        if (indexedShape == false) {
+            builder = new GeoShapeQueryBuilder(fieldName(), shape);
+        } else {
+            indexedShapeToReturn = shape;
+            indexedShapeId = randomAlphaOfLengthBetween(3, 20);
+            builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId);
+            if (randomBoolean()) {
+                indexedShapeIndex = randomAlphaOfLengthBetween(3, 20);
+                builder.indexedShapeIndex(indexedShapeIndex);
+            }
+            if (randomBoolean()) {
+                indexedShapePath = randomAlphaOfLengthBetween(3, 20);
+                builder.indexedShapePath(indexedShapePath);
+            }
+            if (randomBoolean()) {
+                indexedShapeRouting = randomAlphaOfLengthBetween(3, 20);
+                builder.indexedShapeRouting(indexedShapeRouting);
+            }
+        }
+        if (randomBoolean()) {
+            builder.relation(randomFrom(ShapeRelation.INTERSECTS));
+        }
+
+        if (randomBoolean()) {
+            builder.ignoreUnmapped(randomBoolean());
+        }
+        return builder;
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java
new file mode 100644
index 0000000000000..36b438994c076
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.test.geo.RandomShapeGenerator;
+
+public class GeoShapeQueryBuilderGeoShapeTests extends GeoShapeQueryBuilderTests {
+
+    protected String fieldName() {
+        return GEO_SHAPE_FIELD_NAME;
+    }
+
+    protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) {
+        RandomShapeGenerator.ShapeType shapeType = randomFrom(
+            RandomShapeGenerator.ShapeType.POINT,
+            RandomShapeGenerator.ShapeType.MULTIPOINT,
+            RandomShapeGenerator.ShapeType.LINESTRING,
+            RandomShapeGenerator.ShapeType.MULTILINESTRING,
+            RandomShapeGenerator.ShapeType.POLYGON);
+        ShapeBuilder<?, ?, ?> shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType);
+        GeoShapeQueryBuilder builder;
+        clearShapeFields();
+        if (indexedShape == false) {
+            builder = new GeoShapeQueryBuilder(fieldName(), shape);
+        } else {
+            indexedShapeToReturn = shape;
+            indexedShapeId = randomAlphaOfLengthBetween(3, 20);
+            builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId);
+            if (randomBoolean()) {
+                indexedShapeIndex = randomAlphaOfLengthBetween(3, 20);
+                builder.indexedShapeIndex(indexedShapeIndex);
+            }
+            if (randomBoolean()) {
+                indexedShapePath = randomAlphaOfLengthBetween(3, 20);
+                builder.indexedShapePath(indexedShapePath);
+            }
+            if (randomBoolean()) {
+                indexedShapeRouting = randomAlphaOfLengthBetween(3, 20);
+                builder.indexedShapeRouting(indexedShapeRouting);
+            }
+        }
+        if (randomBoolean()) {
+            QueryShardContext context = createShardContext();
+            if (context.indexVersionCreated().onOrAfter(Version.V_7_5_0)) { // CONTAINS is only supported from version 7.5
+                if (shapeType == RandomShapeGenerator.ShapeType.LINESTRING || shapeType == RandomShapeGenerator.ShapeType.MULTILINESTRING) {
+                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS));
+                } else {
+                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS,
+                        ShapeRelation.WITHIN, ShapeRelation.CONTAINS));
+                }
+            } else {
+                if (shapeType == RandomShapeGenerator.ShapeType.LINESTRING || shapeType == RandomShapeGenerator.ShapeType.MULTILINESTRING) {
+                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS));
+                } else {
+                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN));
+                }
+            }
+        }
+
+        if (randomBoolean()) {
+            builder.ignoreUnmapped(randomBoolean());
+        }
+        return builder;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index 07abc89ee430f..c69006023818f 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -48,13 +47,13 @@
 
 import java.io.IOException;
 
-import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.matchesPattern;
 
-public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQueryBuilder> {
+public abstract class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQueryBuilder> {
 
     protected static String indexedShapeId;
     protected static String indexedShapePath;
@@ -62,9 +61,7 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQue
     protected static String indexedShapeRouting;
     protected static ShapeBuilder<?, ?, ?> indexedShapeToReturn;
 
-    protected String fieldName() {
-        return GEO_SHAPE_FIELD_NAME;
-    }
+    protected abstract String fieldName();
 
     @Override
     protected Settings createTestIndexSettings() {
@@ -81,54 +78,7 @@ protected GeoShapeQueryBuilder doCreateTestQueryBuilder() {
         return doCreateTestQueryBuilder(randomBoolean());
     }
 
-    protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) {
-        RandomShapeGenerator.ShapeType shapeType =
-            randomFrom(ShapeType.POINT, ShapeType.MULTIPOINT, ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON);
-        ShapeBuilder<?, ?, ?> shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType);
-        GeoShapeQueryBuilder builder;
-        clearShapeFields();
-        if (indexedShape == false) {
-            builder = new GeoShapeQueryBuilder(fieldName(), shape);
-        } else {
-            indexedShapeToReturn = shape;
-            indexedShapeId = randomAlphaOfLengthBetween(3, 20);
-            builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId);
-            if (randomBoolean()) {
-                indexedShapeIndex = randomAlphaOfLengthBetween(3, 20);
-                builder.indexedShapeIndex(indexedShapeIndex);
-            }
-            if (randomBoolean()) {
-                indexedShapePath = randomAlphaOfLengthBetween(3, 20);
-                builder.indexedShapePath(indexedShapePath);
-            }
-            if (randomBoolean()) {
-                indexedShapeRouting = randomAlphaOfLengthBetween(3, 20);
-                builder.indexedShapeRouting(indexedShapeRouting);
-            }
-        }
-        if (randomBoolean()) {
-            QueryShardContext context = createShardContext();
-            if (context.indexVersionCreated().onOrAfter(Version.V_7_5_0)) { // CONTAINS is only supported from version 7.5
-                if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) {
-                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS));
-                } else {
-                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS,
-                        ShapeRelation.WITHIN, ShapeRelation.CONTAINS));
-                }
-            } else {
-                if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) {
-                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS));
-                } else {
-                    builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN));
-                }
-            }
-        }
-
-        if (randomBoolean()) {
-            builder.ignoreUnmapped(randomBoolean());
-        }
-        return builder;
-    }
+    abstract GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape);
 
     @Override
     protected GetResponse executeGet(GetRequest getRequest) {
@@ -276,7 +226,7 @@ public void testIgnoreUnmapped() throws IOException {
             new GeoShapeQueryBuilder("unmapped", shape.buildGeometry());
         failingQueryBuilder.ignoreUnmapped(false);
         QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
-        assertThat(e.getMessage(), containsString("failed to find geo_shape field [unmapped]"));
+        assertThat(e.getMessage(), matchesPattern("failed to find .*geo_shape.* field \\[unmapped\\]"));
     }
 
     public void testWrongFieldType() throws IOException {
@@ -286,7 +236,8 @@ public void testWrongFieldType() throws IOException {
             new GeoShapeQueryBuilder(STRING_FIELD_NAME, shape) :
             new GeoShapeQueryBuilder(STRING_FIELD_NAME, shape.buildGeometry());
         QueryShardException e = expectThrows(QueryShardException.class, () -> queryBuilder.toQuery(createShardContext()));
-        assertThat(e.getMessage(), containsString("Field [mapped_string] is not of type [geo_shape] but of type [text]"));
+        assertThat(e.getMessage(), matchesPattern("Field \\[mapped_string\\] is of unsupported type \\[text\\]." +
+            " \\[geo_shape\\] query supports the following types \\[.*geo_shape.*\\]"));
     }
 
     public void testSerializationFailsUnlessFetched() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java
index 51702c6d08c6c..1715194f62079 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java
@@ -19,23 +19,158 @@
 
 package org.elasticsearch.search.geo;
 
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
+import org.elasticsearch.common.geo.builders.LineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiLineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiPointBuilder;
+import org.elasticsearch.common.geo.builders.PointBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.geometry.Line;
+import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiLine;
+import org.elasticsearch.geometry.MultiPoint;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.index.query.GeoShapeQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+
+import static org.hamcrest.Matchers.containsString;
 
 public class GeoPointShapeQueryTests extends GeoQueryTests {
 
     @Override
     protected XContentBuilder createDefaultMapping() throws Exception {
         XContentBuilder xcb = XContentFactory.jsonBuilder().startObject()
-            .startObject("properties").startObject("location")
+            .startObject("properties").startObject(defaultGeoFieldName)
             .field("type", "geo_point")
             .endObject().endObject().endObject();
 
         return xcb;
     }
 
-    public void testIndexPointsFilterRectangle() throws Exception {
-        //super.testIndexPointsFilterRectangle(Strings.toString(createDefaultMapping()));
+    public void testProcessRelationSupport() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        Rectangle rectangle = new Rectangle(-35, -25, -25, -35);
+
+        for (ShapeRelation shapeRelation : ShapeRelation.values()) {
+            if (!shapeRelation.equals(ShapeRelation.INTERSECTS)) {
+                SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () ->
+                    client().prepareSearch("test")
+                        .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, rectangle)
+                            .relation(shapeRelation))
+                        .get());
+                assertThat(e.getCause().getMessage(),
+                    containsString(shapeRelation
+                        + " query relation not supported for Field [" + defaultGeoFieldName + "]"));
+            }
+        }
+    }
+
+    public void testQueryLine() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        Line line = new Line(new double[]{-25, -25}, new double[]{-35, -35});
+
+        try {
+            client().prepareSearch("test")
+                .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, line)).get();
+        } catch (
+            SearchPhaseExecutionException e) {
+            assertThat(e.getCause().getMessage(),
+                containsString("does not support " + GeoShapeType.LINESTRING + " queries"));
+        }
+    }
+
+    public void testQueryLinearRing() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        LinearRing linearRing = new LinearRing(new double[]{-25,-35,-25}, new double[]{-25,-35,-25});
+
+        try {
+            // LinearRing extends Line implements Geometry: expose the build process
+            GeoShapeQueryBuilder queryBuilder = new GeoShapeQueryBuilder(defaultGeoFieldName, linearRing);
+            SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE);
+            searchRequestBuilder.setQuery(queryBuilder);
+            searchRequestBuilder.setIndices("test");
+            searchRequestBuilder.get();
+        } catch (
+            SearchPhaseExecutionException e) {
+            assertThat(e.getCause().getMessage(),
+                containsString("Field [" + defaultGeoFieldName + "] does not support LINEARRING queries"));
+        }
+    }
+
+    public void testQueryMultiLine() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        CoordinatesBuilder coords1 = new CoordinatesBuilder()
+            .coordinate(-35,-35)
+            .coordinate(-25,-25);
+        CoordinatesBuilder coords2 = new CoordinatesBuilder()
+            .coordinate(-15,-15)
+            .coordinate(-5,-5);
+        LineStringBuilder lsb1 = new LineStringBuilder(coords1);
+        LineStringBuilder lsb2 = new LineStringBuilder(coords2);
+        MultiLineStringBuilder mlb = new MultiLineStringBuilder().linestring(lsb1).linestring(lsb2);
+        MultiLine multiline = (MultiLine) mlb.buildGeometry();
+
+        try {
+            client().prepareSearch("test")
+                .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, multiline)).get();
+        } catch (Exception e) {
+            assertThat(e.getCause().getMessage(),
+                containsString("does not support " + GeoShapeType.MULTILINESTRING + " queries"));
+        }
+    }
+
+    public void testQueryMultiPoint() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        MultiPointBuilder mpb = new MultiPointBuilder().coordinate(-35,-25).coordinate(-15,-5);
+        MultiPoint multiPoint = mpb.buildGeometry();
+
+        try {
+            client().prepareSearch("test")
+                .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, multiPoint)).get();
+        } catch (Exception e) {
+            assertThat(e.getCause().getMessage(),
+                containsString("does not support " + GeoShapeType.MULTIPOINT + " queries"));
+        }
+    }
+
+    public void testQueryPoint() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        PointBuilder pb = new PointBuilder().coordinate(-35, -25);
+        Point point = pb.buildGeometry();
+
+        try {
+            client().prepareSearch("test")
+                .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, point)).get();
+        } catch (Exception e) {
+            assertThat(e.getCause().getMessage(),
+                containsString("does not support " + GeoShapeType.POINT + " queries"));
+        }
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoQueryTests.java
index 6b00f4a24ebb4..8fce489319e1d 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoQueryTests.java
@@ -22,59 +22,76 @@
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.CircleBuilder;
+import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
 import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
+import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
+import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
+import org.elasticsearch.common.geo.builders.PolygonBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.index.query.GeoShapeQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.locationtech.jts.geom.Coordinate;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.geoIntersectionQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 public abstract class GeoQueryTests extends ESSingleNodeTestCase {
 
     protected abstract XContentBuilder createDefaultMapping() throws Exception;
 
+    static String defaultGeoFieldName = "geo";
+    static String defaultIndexName = "test";
+
     public void testNullShape() throws Exception {
         String mapping = Strings.toString(createDefaultMapping());
-        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
         ensureGreen();
 
-        client().prepareIndex("test").setId("aNullshape").setSource("{\"geo\": null}", XContentType.JSON)
+        client().prepareIndex(defaultIndexName)
+            .setId("aNullshape")
+            .setSource("{\"geo\": null}", XContentType.JSON)
             .setRefreshPolicy(IMMEDIATE).get();
-        GetResponse result = client().prepareGet("test", "aNullshape").get();
+        GetResponse result = client().prepareGet(defaultIndexName, "aNullshape").get();
         assertThat(result.getField("location"), nullValue());
     };
 
-    public void testIndexPointsFilterRectangle(String mapping) throws Exception {
-        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+    public void testIndexPointsFilterRectangle() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
         ensureGreen();
 
-        client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject()
-            .field("name", "Document 1")
-            .startObject("geo")
-            .field("type", "point")
-            .startArray("coordinates").value(-30).value(-30).endArray()
-            .endObject()
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+              .field("name", "Document 1")
+              .field(defaultGeoFieldName, "POINT(-30 -30)")
             .endObject()).setRefreshPolicy(IMMEDIATE).get();
 
-        client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject()
-            .field("name", "Document 2")
-            .startObject("geo")
-            .field("type", "point")
-            .startArray("coordinates").value(-45).value(-50).endArray()
-            .endObject()
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+              .field("name", "Document 2")
+              .field(defaultGeoFieldName, "POINT(-45 -50)")
             .endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         EnvelopeBuilder shape = new EnvelopeBuilder(new Coordinate(-45, 45), new Coordinate(45, -45));
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-            .setQuery(geoIntersectionQuery("geo", shape))
+        GeometryCollectionBuilder builder = new GeometryCollectionBuilder().shape(shape);
+        Geometry geometry = builder.buildGeometry().get(0);
+        SearchResponse searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, geometry)
+                .relation(ShapeRelation.INTERSECTS))
             .get();
 
         assertSearchResponse(searchResponse);
@@ -82,8 +99,9 @@ public void testIndexPointsFilterRectangle(String mapping) throws Exception {
         assertThat(searchResponse.getHits().getHits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
 
-        searchResponse = client().prepareSearch("test")
-            .setQuery(geoShapeQuery("geo", shape))
+        // default query, without specifying relation (expect intersects)
+        searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, geometry))
             .get();
 
         assertSearchResponse(searchResponse);
@@ -92,4 +110,335 @@ public void testIndexPointsFilterRectangle(String mapping) throws Exception {
         assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
     }
 
+    public void testIndexPointsCircle() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 1")
+            .field(defaultGeoFieldName, "POINT(-30 -30)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 2")
+            .field(defaultGeoFieldName, "POINT(-45 -50)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        CircleBuilder shape = new CircleBuilder().center(new Coordinate(-30, -30)).radius("100m");
+        GeometryCollectionBuilder builder = new GeometryCollectionBuilder().shape(shape);
+        Geometry geometry = builder.buildGeometry().get(0);
+
+        try {
+            client().prepareSearch(defaultIndexName)
+                .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, geometry)
+                    .relation(ShapeRelation.INTERSECTS))
+                .get();
+        } catch (
+            Exception e) {
+            assertThat(e.getCause().getMessage(),
+                containsString("failed to create query: "
+                    + GeoShapeType.CIRCLE + " geometry is not supported"));
+        }
+    }
+
+    public void testIndexPointsPolygon() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-30 -30)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-45 -50)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        CoordinatesBuilder cb = new CoordinatesBuilder();
+        cb.coordinate(new Coordinate(-35, -35))
+            .coordinate(new Coordinate(-35, -25))
+            .coordinate(new Coordinate(-25, -25))
+            .coordinate(new Coordinate(-25, -35))
+            .coordinate(new Coordinate(-35, -35));
+        PolygonBuilder shape = new PolygonBuilder(cb);
+        GeometryCollectionBuilder builder = new GeometryCollectionBuilder().shape(shape);
+        Geometry geometry = builder.buildGeometry();
+        SearchResponse searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, geometry)
+                .relation(ShapeRelation.INTERSECTS))
+            .get();
+
+        assertSearchResponse(searchResponse);
+        SearchHits searchHits = searchResponse.getHits();
+        assertThat(searchHits.getTotalHits().value, equalTo(1L));
+        assertThat(searchHits.getAt(0).getId(), equalTo("1"));
+    }
+
+    public void testIndexPointsMultiPolygon() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 1")
+            .field(defaultGeoFieldName, "POINT(-30 -30)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 2")
+            .field(defaultGeoFieldName, "POINT(-40 -40)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("3").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 3")
+            .field(defaultGeoFieldName, "POINT(-50 -50)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        CoordinatesBuilder encloseDocument1Cb = new CoordinatesBuilder();
+        encloseDocument1Cb.coordinate(new Coordinate(-35, -35))
+            .coordinate(new Coordinate(-35, -25))
+            .coordinate(new Coordinate(-25, -25))
+            .coordinate(new Coordinate(-25, -35))
+            .coordinate(new Coordinate(-35, -35));
+        PolygonBuilder encloseDocument1Shape = new PolygonBuilder(encloseDocument1Cb);
+
+        CoordinatesBuilder encloseDocument2Cb = new CoordinatesBuilder();
+        encloseDocument2Cb.coordinate(new Coordinate(-55, -55))
+            .coordinate(new Coordinate(-55, -45))
+            .coordinate(new Coordinate(-45, -45))
+            .coordinate(new Coordinate(-45, -55))
+            .coordinate(new Coordinate(-55, -55));
+        PolygonBuilder encloseDocument2Shape = new PolygonBuilder(encloseDocument2Cb);
+
+        MultiPolygonBuilder mp = new MultiPolygonBuilder();
+        mp.polygon(encloseDocument1Shape).polygon(encloseDocument2Shape);
+
+        GeometryCollectionBuilder builder = new GeometryCollectionBuilder().shape(mp);
+        Geometry geometry = builder.buildGeometry();
+        SearchResponse searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, geometry)
+                .relation(ShapeRelation.INTERSECTS))
+            .get();
+
+        assertSearchResponse(searchResponse);
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
+        assertThat(searchResponse.getHits().getHits().length, equalTo(2));
+        assertThat(searchResponse.getHits().getAt(0).getId(), not(equalTo("2")));
+        assertThat(searchResponse.getHits().getAt(1).getId(), not(equalTo("2")));
+    }
+
+    public void testIndexPointsRectangle() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 1")
+            .field(defaultGeoFieldName, "POINT(-30 -30)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field("name", "Document 2")
+            .field(defaultGeoFieldName, "POINT(-45 -50)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        Rectangle rectangle = new Rectangle(-50, -40, -45, -55);
+
+        SearchResponse searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, rectangle)
+                .relation(ShapeRelation.INTERSECTS))
+            .get();
+
+        assertSearchResponse(searchResponse);
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+        assertThat(searchResponse.getHits().getHits().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("2"));
+    }
+
+    public void testIndexPointsIndexedRectangle() throws Exception {
+        String mapping = Strings.toString(createDefaultMapping());
+        client().admin().indices().prepareCreate(defaultIndexName).setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("point1").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-30 -30)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("point2").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-45 -50)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        String indexedShapeIndex = "indexed_query_shapes";
+        String indexedShapePath = "shape";
+        String queryShapesMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("properties").startObject(indexedShapePath)
+            .field("type", "geo_shape")
+            .endObject()
+            .endObject()
+            .endObject());
+        client().admin().indices().prepareCreate(indexedShapeIndex).setMapping(queryShapesMapping).get();
+        ensureGreen();
+
+        client().prepareIndex(indexedShapeIndex).setId("shape1").setSource(jsonBuilder()
+            .startObject()
+            .field(indexedShapePath, "BBOX(-50, -40, -45, -55)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(indexedShapeIndex).setId("shape2").setSource(jsonBuilder()
+            .startObject()
+            .field(indexedShapePath, "BBOX(-60, -50, -50, -60)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        SearchResponse searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, "shape1")
+                .relation(ShapeRelation.INTERSECTS)
+                .indexedShapeIndex(indexedShapeIndex)
+                .indexedShapePath(indexedShapePath))
+            .get();
+
+        assertSearchResponse(searchResponse);
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+        assertThat(searchResponse.getHits().getHits().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("point2"));
+
+        searchResponse = client().prepareSearch(defaultIndexName)
+            .setQuery(QueryBuilders.geoShapeQuery(defaultGeoFieldName, "shape2")
+                .relation(ShapeRelation.INTERSECTS)
+                .indexedShapeIndex(indexedShapeIndex)
+                .indexedShapePath(indexedShapePath))
+            .get();
+        assertSearchResponse(searchResponse);
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
+    }
+
+    public void testRectangleSpanningDateline() throws Exception {
+        XContentBuilder mapping = createDefaultMapping();
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-169 0)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-179 0)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("3").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(171 0)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        Rectangle rectangle = new Rectangle(
+            169, -178, 1, -1);
+
+        GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("geo", rectangle);
+        SearchResponse response = client().prepareSearch("test").setQuery(geoShapeQueryBuilder).get();
+        SearchHits searchHits = response.getHits();
+        assertEquals(2, searchHits.getTotalHits().value);
+        assertNotEquals("1", searchHits.getAt(0).getId());
+        assertNotEquals("1", searchHits.getAt(1).getId());
+    }
+
+    public void testPolygonSpanningDateline() throws Exception {
+        XContentBuilder mapping = createDefaultMapping();
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-169 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-179 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("3").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(179 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("4").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(171 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        PolygonBuilder polygon = new PolygonBuilder(new CoordinatesBuilder()
+                    .coordinate(-177, 10)
+                    .coordinate(177, 10)
+                    .coordinate(177, 5)
+                    .coordinate(-177, 5)
+                    .coordinate(-177, 10));
+
+        GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("geo", polygon.buildGeometry());
+        geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS);
+        SearchResponse response = client().prepareSearch("test").setQuery(geoShapeQueryBuilder).get();
+        SearchHits searchHits = response.getHits();
+        assertEquals(2, searchHits.getTotalHits().value);
+        assertNotEquals("1", searchHits.getAt(0).getId());
+        assertNotEquals("4", searchHits.getAt(0).getId());
+        assertNotEquals("1", searchHits.getAt(1).getId());
+        assertNotEquals("4", searchHits.getAt(1).getId());
+    }
+
+    public void testMultiPolygonSpanningDateline() throws Exception {
+        XContentBuilder mapping = createDefaultMapping();
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        client().prepareIndex(defaultIndexName).setId("1").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-169 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("2").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(-179 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        client().prepareIndex(defaultIndexName).setId("3").setSource(jsonBuilder()
+            .startObject()
+            .field(defaultGeoFieldName, "POINT(171 7)")
+            .endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        MultiPolygonBuilder multiPolygon = new MultiPolygonBuilder()
+            .polygon(new PolygonBuilder(new CoordinatesBuilder()
+                .coordinate(-167, 10)
+                .coordinate(-171, 10)
+                .coordinate(171, 5)
+                .coordinate(-167, 5)
+                .coordinate(-167, 10)))
+            .polygon(new PolygonBuilder(new CoordinatesBuilder()
+                .coordinate(-177, 10)
+                .coordinate(177, 10)
+                .coordinate(177, 5)
+                .coordinate(-177, 5)
+                .coordinate(-177, 10)));
+
+        GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery(
+            "geo",
+            multiPolygon.buildGeometry());
+        geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS);
+        SearchResponse response = client().prepareSearch("test").setQuery(geoShapeQueryBuilder).get();
+        SearchHits searchHits = response.getHits();
+        assertEquals(2, searchHits.getTotalHits().value);
+        assertNotEquals("3", searchHits.getAt(0).getId());
+        assertNotEquals("3", searchHits.getAt(1).getId());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index df2a044159f9f..afb87348a541f 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -109,10 +109,6 @@ protected XContentBuilder createRandomMapping() throws Exception {
         return xcb;
     }
 
-    public void testIndexPointsFilterRectangle() throws Exception {
-        super.testIndexPointsFilterRectangle(Strings.toString(createRandomMapping()));
-    }
-
     public void testShapeFetchingPath() throws Exception {
         createIndex("shapes");
         String mapping = Strings.toString(createDefaultMapping());
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
index 408da2ed69943..a92e827b8f66e 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Supplier;
@@ -35,6 +36,9 @@
 public class ShapeQueryBuilder extends AbstractGeometryQueryBuilder<ShapeQueryBuilder> {
     public static final String NAME = "shape";
 
+    protected static final List<String> validContentTypes =
+        Collections.unmodifiableList(Arrays.asList(ShapeFieldMapper.CONTENT_TYPE));
+
     /**
      * Creates a new GeoShapeQueryBuilder whose Query will be against the given
      * field name using the given Shape
@@ -100,23 +104,20 @@ protected ShapeQueryBuilder newShapeQueryBuilder(String fieldName, Supplier<Geom
         return new ShapeQueryBuilder(fieldName, shapeSupplier, indexedShapeId);
     }
 
-    @Override
-    public String queryFieldType() {
-        return ShapeFieldMapper.CONTENT_TYPE;
-    }
-
     @Override
     @SuppressWarnings({ "rawtypes" })
-    protected List validContentTypes() {
-        return Arrays.asList(ShapeFieldMapper.CONTENT_TYPE);
+    protected List<String> validContentTypes(){
+        return validContentTypes;
     }
 
     @Override
     @SuppressWarnings({ "rawtypes" })
     public Query buildShapeQuery(QueryShardContext context, MappedFieldType fieldType) {
-        if (fieldType.typeName().equals(ShapeFieldMapper.CONTENT_TYPE) == false) {
+        List<String> validContentTypes = validContentTypes();
+        if (validContentTypes.contains(fieldType.typeName()) == false) {
             throw new QueryShardException(context,
-                "Field [" + fieldName + "] is not of type [" + queryFieldType() + "] but of type [" + fieldType.typeName() + "]");
+                "Field [" + fieldName + "] is not of type [" + String.join(" or ", validContentTypes())
+                    + "] but of type [" + fieldType.typeName() + "]");
         }
 
         final AbstractGeometryFieldMapper.AbstractGeometryFieldType ft = (AbstractGeometryFieldMapper.AbstractGeometryFieldType) fieldType;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
index 006577bde0266..da9bc32355bac 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Polygon;
 import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
+import org.elasticsearch.index.mapper.AbstractSearchableGeometryFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
@@ -36,7 +36,7 @@
 
 import static org.elasticsearch.xpack.spatial.index.mapper.ShapeIndexer.toLucenePolygon;
 
-public class ShapeQueryProcessor implements AbstractGeometryFieldMapper.QueryProcessor {
+public class ShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
 
     @Override
     public Query process(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) {

From 022ad0a56b85d05616cf2cafc844b7050d513254 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Wed, 18 Mar 2020 18:49:55 +0200
Subject: [PATCH 116/285] Revert "Mute TimeSeriesLifecycleActionsIT (#53740)"

This reverts commit a29a83e29cbb25f16e61223bc418491097744821.
---
 .../xpack/ilm/TimeSeriesLifecycleActionsIT.java    | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 979fc447db172..14c99d471e983 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -10,7 +10,6 @@
 import org.apache.http.entity.StringEntity;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
@@ -76,7 +75,6 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53738")
 public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
     private static final Logger logger = LogManager.getLogger(TimeSeriesLifecycleActionsIT.class);
     private static final String FAILED_STEP_RETRY_COUNT_FIELD = "failed_step_retry_count";
@@ -96,7 +94,7 @@ public void refreshIndex() {
     public static void updatePolicy(String indexName, String policy) throws IOException {
         Request changePolicyRequest = new Request("PUT", "/" + indexName + "/_settings");
         final StringEntity changePolicyEntity = new StringEntity("{ \"index.lifecycle.name\": \"" + policy + "\" }",
-            ContentType.APPLICATION_JSON);
+                ContentType.APPLICATION_JSON);
         changePolicyRequest.setEntity(changePolicyEntity);
         assertOK(client().performRequest(changePolicyRequest));
     }
@@ -651,9 +649,9 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception
         createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
 
-        ensureGreen(index);
+       ensureGreen(index);
 
-        // unallocate all index shards
+       // unallocate all index shards
         Request setAllocationToMissingAttribute = new Request("PUT", "/" + index + "/_settings");
         setAllocationToMissingAttribute.setJsonEntity("{\n" +
             "  \"settings\": {\n" +
@@ -1328,8 +1326,8 @@ public void testWaitForActiveShardsStep() throws Exception {
         String originalIndex = index + "-000001";
         String secondIndex = index + "-000002";
         createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias),
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias),
             true);
 
         // create policy
@@ -1809,7 +1807,7 @@ private void createSlmPolicy(String smlPolicy, String repo) throws IOException {
                 .field("repository", repo)
                 .field("name", "snap" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT))
                 .startObject("config")
-                .field("include_global_state", false)
+                    .field("include_global_state", false)
                 .endObject()
                 .endObject()));
 

From c46e0f596fd8524c81953220fb1d25827a2f0e92 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Wed, 18 Mar 2020 19:43:26 +0200
Subject: [PATCH 117/285] Mute failing tests (#53749)

---
 .../elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 14c99d471e983..feb2854a0986a 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -207,6 +207,7 @@ public void testMoveToRolloverStep() throws Exception {
         assertBusy(() -> assertFalse(indexExists(shrunkenOriginalIndex)));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53738")
     public void testRetryFailedDeleteAction() throws Exception {
         createIndexWithSettings(index, Settings.builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
@@ -282,6 +283,7 @@ public void testRetryFailedShrinkAction() throws Exception {
         expectThrows(ResponseException.class, this::indexDocument);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53738")
     public void testRolloverAction() throws Exception {
         String originalIndex = index + "-000001";
         String secondIndex = index + "-000002";
@@ -300,6 +302,7 @@ public void testRolloverAction() throws Exception {
         assertBusy(() -> assertEquals("true", getOnlyIndexSettings(originalIndex).get(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE)));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53738")
     public void testRolloverActionWithIndexingComplete() throws Exception {
         String originalIndex = index + "-000001";
         String secondIndex = index + "-000002";
@@ -752,6 +755,7 @@ public void testFreezeDuringSnapshot() throws Exception {
         assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot")));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53738")
     public void testSetPriority() throws Exception {
         createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.INDEX_PRIORITY_SETTING.getKey(), 100));

From 070ea7eff1d67a054bc23a6557ea081ebdff57d4 Mon Sep 17 00:00:00 2001
From: Stuart Tettemer <stu@elastic.co>
Date: Wed, 18 Mar 2020 12:31:30 -0600
Subject: [PATCH 118/285] Scripting: Per-context script cache, default off
 (#52855)

* Adds per context settings:
  `script.context.${CONTEXT}.cache_max_size` ~
  `script.cache.max_size`

  `script.context.${CONTEXT}.cache_expire` ~
  `script.cache.expire`

  `script.context.${CONTEXT}.max_compilations_rate` ~
  `script.max_compilations_rate`

* Context cache is used if:
  `script.max_compilations_rate=use-context`.  This
  value is dynamically updatable, so users can
  switch back to the general cache if desired.

* Settings for context caches take the first value
  that applies:
  1) Context specific settings if set, eg
     `script.context.ingest.cache_max_size`
  2) Correlated general setting is set to the non-default
     value, eg `script.cache.max_size`
  3) Context default

The reason for 2's inclusion is to allow an easy
transition for users who've customized their general
cache settings.

Using the general cache settings for the context caches
results in higher effective settings, since they are
multiplied across the number of contexts.  So a general
cache max size of 200 will become 200 * # of contexts.
However, this behavior it will avoid users snapping to a
value that is too low for them.


Refs: #50152
---
 .../settings/AbstractScopedSettings.java      |  12 +
 .../common/settings/ClusterSettings.java      |   5 +-
 .../common/settings/Setting.java              |   6 +
 .../org/elasticsearch/script/ScriptCache.java |  66 ++---
 .../elasticsearch/script/ScriptService.java   | 233 ++++++++++++++++--
 .../org/elasticsearch/script/ScriptStats.java |  16 ++
 .../common/settings/SettingTests.java         |  52 ++++
 .../indices/IndicesServiceCloseTests.java     |   2 +-
 .../script/ScriptCacheTests.java              |  27 +-
 .../script/ScriptServiceTests.java            | 203 ++++++++++++++-
 .../test/InternalTestCluster.java             |   4 +-
 .../elasticsearch/xpack/CcrIntegTestCase.java |   2 +-
 12 files changed, 544 insertions(+), 84 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 99c651ef7df80..c7e88a3c07c60 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -224,6 +224,18 @@ public synchronized void addSettingsUpdateConsumer(Consumer<Settings> consumer,
         addSettingsUpdater(Setting.groupedSettingsUpdater(consumer, settings));
     }
 
+    /**
+     * Adds a settings consumer that is only executed if any setting in the supplied list of settings is changed. In that case all the
+     * settings are specified in the argument are returned.  The validator is run across all specified settings before the settings are
+     * applied.
+     *
+     * Also automatically adds empty consumers for all settings in order to activate logging
+     */
+    public synchronized void addSettingsUpdateConsumer(Consumer<Settings> consumer, List<? extends Setting<?>> settings,
+                                                       Consumer<Settings> validator) {
+        addSettingsUpdater(Setting.groupedSettingsUpdater(consumer, settings, validator));
+    }
+
     /**
      * Adds a settings consumer for affix settings. Affix settings have a namespace associated to it that needs to be available to the
      * consumer in order to be processed correctly.
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index f4f53f939982d..d121752017c58 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -364,10 +364,13 @@ public void apply(Settings value, Settings current, Settings previous) {
             NetworkService.TCP_RECEIVE_BUFFER_SIZE,
             IndexSettings.QUERY_STRING_ANALYZE_WILDCARD,
             IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD,
+            ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING,
+            ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
+            ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
             ScriptService.SCRIPT_CACHE_SIZE_SETTING,
             ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
+            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
             ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
-            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE,
             ScriptService.TYPES_ALLOWED_SETTING,
             ScriptService.CONTEXTS_ALLOWED_SETTING,
             IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
index 4e926f6c3ca11..0680dd7c2ccfd 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -668,7 +668,12 @@ public String toString() {
 
     static AbstractScopedSettings.SettingUpdater<Settings> groupedSettingsUpdater(Consumer<Settings> consumer,
                                                                                   final List<? extends Setting<?>> configuredSettings) {
+        return groupedSettingsUpdater(consumer, configuredSettings, (v) -> {});
+    }
 
+    static AbstractScopedSettings.SettingUpdater<Settings> groupedSettingsUpdater(Consumer<Settings> consumer,
+                                                                                  final List<? extends Setting<?>> configuredSettings,
+                                                                                  Consumer<Settings> validator) {
         return new AbstractScopedSettings.SettingUpdater<Settings>() {
 
             private Settings get(Settings settings) {
@@ -691,6 +696,7 @@ public boolean hasChanged(Settings current, Settings previous) {
 
             @Override
             public Settings getValue(Settings current, Settings previous) {
+                validator.accept(current);
                 return get(current);
             }
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java
index e5b351e223965..a37851cc81050 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java
@@ -40,53 +40,47 @@ public class ScriptCache {
 
     private static final Logger logger = LogManager.getLogger(ScriptService.class);
 
-    private Cache<CacheKey, Object> cache;
-    private final ScriptMetrics scriptMetrics = new ScriptMetrics();
+    private final Cache<CacheKey, Object> cache;
+    private final ScriptMetrics scriptMetrics;
 
     private final Object lock = new Object();
 
-    private Tuple<Integer, TimeValue> rate;
+    // Mutable fields
     private long lastInlineCompileTime;
     private double scriptsPerTimeWindow;
-    private double compilesAllowedPerNano;
 
-    // Cache settings
-    private int cacheSize;
-    private TimeValue cacheExpire;
+    // Cache settings or derived from settings
+    final int cacheSize;
+    final TimeValue cacheExpire;
+    final Tuple<Integer, TimeValue> rate;
+    private final double compilesAllowedPerNano;
 
-    public ScriptCache(
+    ScriptCache(
             int cacheMaxSize,
             TimeValue cacheExpire,
             Tuple<Integer, TimeValue> maxCompilationRate
     ) {
+        this.cacheSize = cacheMaxSize;
+        this.cacheExpire = cacheExpire;
+
         CacheBuilder<CacheKey, Object> cacheBuilder = CacheBuilder.builder();
-        if (cacheMaxSize >= 0) {
-            cacheBuilder.setMaximumWeight(cacheMaxSize);
+        if (this.cacheSize >= 0) {
+            cacheBuilder.setMaximumWeight(this.cacheSize);
         }
 
-        if (cacheExpire.getNanos() != 0) {
-            cacheBuilder.setExpireAfterAccess(cacheExpire);
+        if (this.cacheExpire.getNanos() != 0) {
+            cacheBuilder.setExpireAfterAccess(this.cacheExpire);
         }
 
-        logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire);
+        logger.debug("using script cache with max_size [{}], expire [{}]", this.cacheSize, this.cacheExpire);
         this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
 
-        this.lastInlineCompileTime = System.nanoTime();
-
-        this.cacheSize = cacheMaxSize;
-        this.cacheExpire = cacheExpire;
-        this.setMaxCompilationRate(maxCompilationRate);
-    }
+        this.rate = maxCompilationRate;
+        this.scriptsPerTimeWindow = this.rate.v1();
+        this.compilesAllowedPerNano = ((double) rate.v1()) / rate.v2().nanos();
 
-    private Cache<CacheKey,Object> buildCache() {
-        CacheBuilder<CacheKey, Object> cacheBuilder = CacheBuilder.builder();
-        if (cacheSize >= 0) {
-            cacheBuilder.setMaximumWeight(cacheSize);
-        }
-        if (cacheExpire.getNanos() != 0) {
-            cacheBuilder.setExpireAfterAccess(cacheExpire);
-        }
-        return cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
+        this.lastInlineCompileTime = System.nanoTime();
+        this.scriptMetrics = new ScriptMetrics();
     }
 
     <FactoryType> FactoryType compile(
@@ -185,22 +179,6 @@ void checkCompilationLimit() {
         }
     }
 
-    /**
-     * This configures the maximum script compilations per five minute window.
-     *
-     * @param newRate the new expected maximum number of compilations per five minute window
-     */
-    void setMaxCompilationRate(Tuple<Integer, TimeValue> newRate) {
-        synchronized (lock) {
-            this.rate = newRate;
-            // Reset the counter to allow new compilations
-            this.scriptsPerTimeWindow = rate.v1();
-            this.compilesAllowedPerNano = ((double) rate.v1()) / newRate.v2().nanos();
-
-            this.cache = buildCache();
-        }
-    }
-
     /**
      * A small listener for the script cache that calls each
      * {@code ScriptEngine}'s {@code scriptRemoved} method when the
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java
index 7cf23d26187ad..66ed3fee0d6ca 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -45,13 +45,16 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -61,6 +64,11 @@ public class ScriptService implements Closeable, ClusterStateApplier {
 
     static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic";
 
+    // Special setting value for SCRIPT_GENERAL_MAX_COMPILATIONS_RATE to indicate the script service should use context
+    // specific caches
+    static final Tuple<Integer, TimeValue> USE_CONTEXT_RATE_VALUE = new Tuple<>(-1, TimeValue.MINUS_ONE);
+    static final String USE_CONTEXT_RATE_KEY = "use-context";
+
     // a parsing function that requires a non negative int and a timevalue as arguments split by a slash
     // this allows you to easily define rates
     static final Function<String, Tuple<Integer, TimeValue>> MAX_COMPILATION_RATE_FUNCTION =
@@ -93,14 +101,39 @@ public class ScriptService implements Closeable, ClusterStateApplier {
                 }
             };
 
-    public static final Setting<Integer> SCRIPT_CACHE_SIZE_SETTING =
+    public static final Setting<Integer> SCRIPT_GENERAL_CACHE_SIZE_SETTING =
         Setting.intSetting("script.cache.max_size", 100, 0, Property.NodeScope);
-    public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
+    public static final Setting<TimeValue> SCRIPT_GENERAL_CACHE_EXPIRE_SETTING =
         Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final Setting<Integer> SCRIPT_MAX_SIZE_IN_BYTES =
         Setting.intSetting("script.max_size_in_bytes", 65535, 0, Property.Dynamic, Property.NodeScope);
-    public static final Setting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE =
-            new Setting<>("script.max_compilations_rate", "75/5m", MAX_COMPILATION_RATE_FUNCTION, Property.Dynamic, Property.NodeScope);
+    public static final Setting<Tuple<Integer, TimeValue>> SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING =
+        new Setting<>("script.max_compilations_rate", "75/5m",
+            (String value) -> value.equals(USE_CONTEXT_RATE_KEY) ? USE_CONTEXT_RATE_VALUE: MAX_COMPILATION_RATE_FUNCTION.apply(value),
+            Property.Dynamic, Property.NodeScope);
+
+    // Per-context settings
+    static final String CONTEXT_PREFIX = "script.context.";
+
+    // script.context.<context-name>.{cache_max_size, cache_expire, max_compilations_rate}
+
+    public static final Setting.AffixSetting<Integer> SCRIPT_CACHE_SIZE_SETTING =
+        Setting.affixKeySetting(CONTEXT_PREFIX,
+            "cache_max_size",
+            key -> Setting.intSetting(key, SCRIPT_GENERAL_CACHE_SIZE_SETTING, 0, Property.NodeScope, Property.Dynamic));
+
+    public static final Setting.AffixSetting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
+        Setting.affixKeySetting(CONTEXT_PREFIX,
+            "cache_expire",
+            key -> Setting.positiveTimeSetting(key, SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, TimeValue.timeValueMillis(0),
+                                               Property.NodeScope, Property.Dynamic));
+
+    public static final Setting.AffixSetting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE_SETTING =
+        Setting.affixKeySetting(CONTEXT_PREFIX,
+            "max_compilations_rate",
+            key -> new Setting<>(key, "75/5m", MAX_COMPILATION_RATE_FUNCTION, Property.NodeScope, Property.Dynamic));
+
+    private static final Tuple<Integer, TimeValue> SCRIPT_COMPILATION_RATE_ZERO = new Tuple<>(0, TimeValue.ZERO);
 
     public static final String ALLOW_NONE = "none";
 
@@ -109,12 +142,9 @@ public class ScriptService implements Closeable, ClusterStateApplier {
     public static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING =
         Setting.listSetting("script.allowed_contexts", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
 
-    private final Settings settings;
     private final Set<String> typesAllowed;
     private final Set<String> contextsAllowed;
 
-    final ScriptCache compiler;
-
     private final Map<String, ScriptEngine> engines;
     private final Map<String, ScriptContext<?>> contexts;
 
@@ -122,8 +152,9 @@ public class ScriptService implements Closeable, ClusterStateApplier {
 
     private int maxSizeInBytes;
 
+    private final AtomicReference<CacheHolder> cacheHolder;
+
     public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<String, ScriptContext<?>> contexts) {
-        this.settings = Objects.requireNonNull(settings);
         this.engines = Objects.requireNonNull(engines);
         this.contexts = Objects.requireNonNull(contexts);
 
@@ -201,13 +232,10 @@ public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<S
         }
 
         this.setMaxSizeInBytes(SCRIPT_MAX_SIZE_IN_BYTES.get(settings));
-        compiler = new ScriptCache(
-                    SCRIPT_CACHE_SIZE_SETTING.get(settings),
-                    SCRIPT_CACHE_EXPIRE_SETTING.get(settings),
-                    compilationLimitsEnabled() ?
-                        SCRIPT_MAX_COMPILATIONS_RATE.get(settings):
-                        new Tuple<>(0, TimeValue.ZERO)
-                    );
+
+        // Validation requires knowing which contexts exist.
+        this.validateCacheSettings(settings);
+        cacheHolder = new AtomicReference<>(new CacheHolder(settings, contexts.keySet(), compilationLimitsEnabled()));
     }
 
     /**
@@ -219,7 +247,66 @@ boolean compilationLimitsEnabled() {
 
     void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
         clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_SIZE_IN_BYTES, this::setMaxSizeInBytes);
-        clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_COMPILATIONS_RATE, compiler::setMaxCompilationRate);
+
+        // Handle all updatable per-context settings at once for each context.
+        for (String context: contexts.keySet()) {
+            clusterSettings.addSettingsUpdateConsumer(
+                (settings) -> cacheHolder.get().updateContextSettings(settings, context),
+                List.of(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context),
+                        SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context),
+                        SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context),
+                        SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, // general settings used for fallbacks
+                        SCRIPT_GENERAL_CACHE_SIZE_SETTING
+                )
+            );
+        }
+
+        // Handle all settings for context and general caches, this flips between general and context caches.
+        clusterSettings.addSettingsUpdateConsumer(
+            (settings) -> cacheHolder.set(cacheHolder.get().withUpdatedCacheSettings(settings)),
+            List.of(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
+                    SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
+                    SCRIPT_GENERAL_CACHE_SIZE_SETTING,
+                    SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
+                    SCRIPT_CACHE_EXPIRE_SETTING,
+                    SCRIPT_CACHE_SIZE_SETTING),
+            this::validateCacheSettings
+        );
+    }
+
+    /**
+     * Throw an IllegalArgumentException if any per-context setting does not match a context or if per-context settings are configured
+     * when using the general cache.
+     */
+    void validateCacheSettings(Settings settings) {
+        boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE);
+        List<Setting.AffixSetting<?>> affixes = List.of(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, SCRIPT_CACHE_EXPIRE_SETTING,
+                                                        SCRIPT_CACHE_SIZE_SETTING);
+        List<String> keys = new ArrayList<>();
+        for (Setting.AffixSetting<?> affix: affixes) {
+            keys.addAll(getConcreteSettingKeys(affix, settings));
+        }
+        if (useContext == false && keys.isEmpty() == false) {
+            throw new IllegalArgumentException("Context cache settings [" + String.join(", ", keys) + "] requires [" +
+                SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey() + "] to be [" + USE_CONTEXT_RATE_KEY + "]");
+        }
+    }
+
+    /**
+     * Get concrete settings keys from affix settings for given Settings.  Throws an IllegalArgumentException if the namespace of matching
+     * affix settings do not match any context name.
+     */
+    List<String> getConcreteSettingKeys(Setting.AffixSetting<?> setting, Settings settings) {
+        List<String> concreteKeys = new ArrayList<>();
+        for (String context: setting.getAsMap(settings).keySet()) {
+            String s = setting.getConcreteSettingForNamespace(context).getKey();
+            if (contexts.containsKey(context) == false) {
+                throw new IllegalArgumentException("Context [" + context + "] doesn't exist for setting [" + s + "]");
+            }
+            concreteKeys.add(s);
+        }
+        concreteKeys.sort(Comparator.naturalOrder());
+        return concreteKeys;
     }
 
     @Override
@@ -304,7 +391,9 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
             logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode);
         }
 
-        return compiler.compile(context, scriptEngine, id, idOrCode, type, options);
+        ScriptCache scriptCache = cacheHolder.get().get(context.name);
+        assert scriptCache != null : "script context [" + context.name + "] has no script cache";
+        return scriptCache.compile(context, scriptEngine, id, idOrCode, type, options);
     }
 
     public boolean isLangSupported(String lang) {
@@ -468,11 +557,119 @@ public ScriptLanguagesInfo getScriptLanguages() {
     }
 
     public ScriptStats stats() {
-        return compiler.stats();
+        return cacheHolder.get().stats();
     }
 
     @Override
     public void applyClusterState(ClusterChangedEvent event) {
         clusterState = event.state();
     }
+
+    /**
+     * Container for the ScriptCache(s).  This class operates in two modes:
+     * 1) general mode, if the general script cache is configured.  There are no context caches in this case.
+     * 2) context mode, if the context script cache is configured.  There is no general cache in this case.
+     */
+    static class CacheHolder {
+        final ScriptCache general;
+        final Map<String, AtomicReference<ScriptCache>> contextCache;
+
+        final Set<String> contexts;
+        final boolean compilationLimitsEnabled;
+
+        CacheHolder(Settings settings, Set<String> contexts, boolean compilationLimitsEnabled) {
+            this.compilationLimitsEnabled = compilationLimitsEnabled;
+            this.contexts = Set.copyOf(contexts);
+            if (SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE)) {
+                this.general = null;
+                Map<String, AtomicReference<ScriptCache>> contextCache = new HashMap<>(this.contexts.size());
+                for (String context : this.contexts) {
+                    contextCache.put(context, new AtomicReference<>(contextFromSettings(settings, context, this.compilationLimitsEnabled)));
+                }
+                this.contextCache = Collections.unmodifiableMap(contextCache);
+            } else {
+                this.contextCache = null;
+                this.general = new ScriptCache(
+                    SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings),
+                    SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings),
+                    compilationLimitsEnabled ?
+                        SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) :
+                        SCRIPT_COMPILATION_RATE_ZERO);
+            }
+        }
+
+        /**
+         * Create a ScriptCache for the given context.
+         */
+        private static ScriptCache contextFromSettings(Settings settings, String context, boolean compilationLimitsEnabled) {
+            return new ScriptCache(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(settings),
+                SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(settings),
+                compilationLimitsEnabled ?
+                    SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).get(settings) :
+                    SCRIPT_COMPILATION_RATE_ZERO);
+        }
+
+        /**
+         * Returns a CacheHolder with the given settings.  Flips between general and context caches if necessary.  Creates new general
+         * cache if in general cache mode and {@code script.max_compilations_rate} has changed to any value other than {@code use-context}.
+         */
+        CacheHolder withUpdatedCacheSettings(Settings settings) {
+            if (SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE)) {
+                if (general != null) {
+                    // Flipping to context specific
+                    logger.debug("Switching to context cache from general cache");
+                    return new CacheHolder(settings, contexts, compilationLimitsEnabled);
+                }
+            } else if (general == null) {
+                // Flipping to general
+                logger.debug("Switching from context cache to general cache");
+                return new CacheHolder(settings, contexts, compilationLimitsEnabled);
+            } else if (general.rate.equals(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings)) == false) {
+                // General compilation rate changed, that setting is the only dynamically updated general setting
+                logger.debug("General compilation rate changed from [" + general.rate + "] to [" +
+                    SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) + "], creating new general cache");
+                return new CacheHolder(settings, contexts, compilationLimitsEnabled);
+            }
+
+            // no-op change, this is possible when context settings change while in context mode
+            return this;
+        }
+
+        /**
+         * get the cache appropriate for the context.  If in general mode, return the general cache.  Otherwise return the ScriptCache for
+         * the given context. Returns null in context mode if the requested context does not exist.
+         */
+        ScriptCache get(String context) {
+            if (general != null) {
+                return general;
+            }
+            AtomicReference<ScriptCache> ref = contextCache.get(context);
+            if (ref == null) {
+                return null;
+            }
+            return ref.get();
+        }
+
+        ScriptStats stats() {
+            if (general != null) {
+                return general.stats();
+            }
+            return ScriptStats.sum(contextCache.values().stream().map(AtomicReference::get).map(ScriptCache::stats)::iterator);
+        }
+
+        /**
+         * Update settings for the context cache, if we're in the context cache mode otherwise no-op.
+         */
+        void updateContextSettings(Settings settings, String context) {
+            if (general != null) {
+                return;
+            }
+            AtomicReference<ScriptCache> ref = contextCache.get(context);
+            assert ref != null : "expected script cache to exist for context [" + context + "]";
+            ScriptCache cache = ref.get();
+            assert cache != null : "expected script cache to be non-null for context [" + context + "]";
+            ref.set(contextFromSettings(settings, context, compilationLimitsEnabled));
+            logger.debug("Replaced context [" + context + "] with new settings");
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
index 86a6a0d13569b..953b3a18ae92c 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
@@ -79,4 +79,20 @@ static final class Fields {
         static final String CACHE_EVICTIONS = "cache_evictions";
         static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
     }
+
+    public static ScriptStats sum(Iterable<ScriptStats> stats) {
+        long compilations = 0;
+        long cacheEvictions = 0;
+        long compilationLimitTriggered = 0;
+        for (ScriptStats stat: stats) {
+            compilations += stat.compilations;
+            cacheEvictions += stat.cacheEvictions;
+            compilationLimitTriggered += stat.compilationLimitTriggered;
+        }
+        return new ScriptStats(
+            compilations,
+            cacheEvictions,
+            compilationLimitTriggered
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 56d20d30d7f87..a129b314bd73b 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -1079,6 +1079,58 @@ public void testAffixNamespacesWithGroupSetting() {
         assertThat(affixSetting.getNamespaces(Settings.builder().put("prefix.infix.suffix.anything", "anything").build()), hasSize(1));
     }
 
+    public void testGroupSettingUpdaterValidator() {
+        final Setting.AffixSetting<Integer> affixSetting =
+            Setting.affixKeySetting("prefix.","suffix",
+                (key) -> Setting.intSetting(key, 5, Setting.Property.Dynamic, Setting.Property.NodeScope));
+        Setting<Integer> fixSetting = Setting.intSetting("abc", 1, Property.NodeScope);
+
+        Consumer<Settings> validator = s -> {
+            if (affixSetting.getNamespaces(s).contains("foo")) {
+                if (fixSetting.get(s) == 2) {
+                    throw new IllegalArgumentException("foo and 2 can't go together");
+                }
+            } else if (affixSetting.getNamespaces(s).contains("bar")) {
+                throw new IllegalArgumentException("no bar");
+            }
+        };
+
+        AbstractScopedSettings.SettingUpdater<Settings> updater = Setting.groupedSettingsUpdater(s -> {},
+            Arrays.asList(affixSetting, fixSetting), validator);
+
+        IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> {
+            updater.getValue(
+                Settings.builder()
+                    .put("prefix.foo.suffix", 5)
+                    .put("abc", 2)
+                    .build(),
+                Settings.EMPTY
+            );
+        });
+        assertEquals("foo and 2 can't go together", illegal.getMessage());
+
+        illegal = expectThrows(IllegalArgumentException.class, () -> {
+            updater.getValue(
+                Settings.builder()
+                    .put("prefix.bar.suffix", 6)
+                    .put("abc", 3)
+                    .build(),
+                Settings.EMPTY
+            );
+        });
+        assertEquals("no bar", illegal.getMessage());
+
+        Settings s = updater.getValue(
+            Settings.builder()
+                .put("prefix.foo.suffix", 5)
+                .put("prefix.bar.suffix", 5)
+                .put("abc", 3)
+                .build(),
+            Settings.EMPTY
+        );
+        assertNotNull(s);
+    }
+
     public void testExists() {
         final Setting<?> fooSetting = Setting.simpleString("foo", Property.NodeScope);
         assertFalse(fooSetting.exists(Settings.EMPTY));
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java
index 96bb4c13cb8aa..29b9f12484470 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java
@@ -69,7 +69,7 @@ private Node startNode() throws NodeValidationException {
             .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
-            .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m")
+            .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "1000/1m")
             .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("transport.type", getTestTransportType())
             .put(Node.NODE_DATA_SETTING.getKey(), true)
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
index 279456321aad7..dfa946099306f 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
@@ -28,31 +28,28 @@ public class ScriptCacheTests extends ESTestCase {
     // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes
     // simply by multiplying by five, so even setting it to one, requires five compilations to break
     public void testCompilationCircuitBreaking() throws Exception {
-        ScriptCache cache = new ScriptCache(
-            ScriptService.SCRIPT_CACHE_SIZE_SETTING.get(Settings.EMPTY),
-            ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.get(Settings.EMPTY),
-            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.get(Settings.EMPTY)
-        );
-        cache.setMaxCompilationRate(Tuple.tuple(1, TimeValue.timeValueMinutes(1)));
+        final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY);
+        final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY);
+        Tuple<Integer, TimeValue> rate = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(Settings.EMPTY);
+        ScriptCache cache = new ScriptCache(size, expire, Tuple.tuple(1, TimeValue.timeValueMinutes(1)));
         cache.checkCompilationLimit(); // should pass
-        expectThrows(CircuitBreakingException.class, () -> cache.checkCompilationLimit());
-        cache.setMaxCompilationRate(Tuple.tuple(2, TimeValue.timeValueMinutes(1)));
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        cache = new ScriptCache(size, expire, (Tuple.tuple(2, TimeValue.timeValueMinutes(1))));
         cache.checkCompilationLimit(); // should pass
         cache.checkCompilationLimit(); // should pass
-        expectThrows(CircuitBreakingException.class, () -> cache.checkCompilationLimit());
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
         int count = randomIntBetween(5, 50);
-        cache.setMaxCompilationRate(Tuple.tuple(count, TimeValue.timeValueMinutes(1)));
+        cache = new ScriptCache(size, expire, (Tuple.tuple(count, TimeValue.timeValueMinutes(1))));
         for (int i = 0; i < count; i++) {
             cache.checkCompilationLimit(); // should pass
         }
-        expectThrows(CircuitBreakingException.class, () -> cache.checkCompilationLimit());
-        cache.setMaxCompilationRate(Tuple.tuple(0, TimeValue.timeValueMinutes(1)));
-        expectThrows(CircuitBreakingException.class, () -> cache.checkCompilationLimit());
-        cache.setMaxCompilationRate(Tuple.tuple(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)));
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        cache = new ScriptCache(size, expire, (Tuple.tuple(0, TimeValue.timeValueMinutes(1))));
+        expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
+        cache = new ScriptCache(size, expire, (Tuple.tuple(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1))));
         int largeLimit = randomIntBetween(1000, 10000);
         for (int i = 0; i < largeLimit; i++) {
             cache.checkCompilationLimit();
         }
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 814578a29f2f7..77ce3fe127c7f 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -38,10 +38,14 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.script.ScriptService.MAX_COMPILATION_RATE_FUNCTION;
+import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING;
+import static org.elasticsearch.script.ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -222,7 +226,7 @@ public void testMultipleCompilationsCountedInCompilationStats() throws IOExcepti
 
     public void testCompilationStatsOnCacheHit() throws IOException {
         Settings.Builder builder = Settings.builder();
-        builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
         buildScriptService(builder.build());
         Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
         ScriptContext<?> context = randomFrom(contexts.values());
@@ -239,7 +243,7 @@ public void testIndexedScriptCountedInCompilationStats() throws IOException {
 
     public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
         Settings.Builder builder = Settings.builder();
-        builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
         buildScriptService(builder.build());
         scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values()));
         scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(contexts.values()));
@@ -309,6 +313,201 @@ public void testMaxSizeLimit() throws Exception {
                 iae.getMessage());
     }
 
+    public void testConflictContextSettings() throws IOException {
+        IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> {
+            buildScriptService(Settings.builder()
+                .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123).build());
+        });
+        assertEquals("Context cache settings [script.context.field.cache_max_size] requires " +
+                     "[script.max_compilations_rate] to be [use-context]",
+            illegal.getMessage()
+        );
+
+        illegal = expectThrows(IllegalArgumentException.class, () -> {
+            buildScriptService(Settings.builder()
+                .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m").build());
+        });
+
+        assertEquals("Context cache settings [script.context.ingest.cache_expire] requires " +
+                     "[script.max_compilations_rate] to be [use-context]",
+            illegal.getMessage()
+        );
+
+        illegal = expectThrows(IllegalArgumentException.class, () -> {
+            buildScriptService(Settings.builder()
+                .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m").build());
+        });
+
+        assertEquals("Context cache settings [script.context.score.max_compilations_rate] requires " +
+                     "[script.max_compilations_rate] to be [use-context]",
+            illegal.getMessage()
+        );
+
+        buildScriptService(
+            Settings.builder()
+                .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m")
+                .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123)
+                .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m")
+                .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY).build());
+    }
+
+    public void testFallbackContextSettings() {
+        int cacheSizeBackup = randomIntBetween(0, 1024);
+        int cacheSizeFoo = randomValueOtherThan(cacheSizeBackup, () -> randomIntBetween(0, 1024));
+
+        String cacheExpireBackup = randomTimeValue(1, 1000, "h");
+        TimeValue cacheExpireBackupParsed = TimeValue.parseTimeValue(cacheExpireBackup, "");
+        String cacheExpireFoo = randomValueOtherThan(cacheExpireBackup, () -> randomTimeValue(1, 1000, "h"));
+        TimeValue cacheExpireFooParsed = TimeValue.parseTimeValue(cacheExpireFoo, "");
+
+        Settings s = Settings.builder()
+            .put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), cacheSizeBackup)
+            .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheSizeFoo)
+            .put(ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), cacheExpireBackup)
+            .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheExpireFoo)
+            .build();
+
+        assertEquals(cacheSizeFoo, ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("foo").get(s).intValue());
+        assertEquals(cacheSizeBackup, ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("bar").get(s).intValue());
+
+        assertEquals(cacheExpireFooParsed, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").get(s));
+        assertEquals(cacheExpireBackupParsed, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("bar").get(s));
+    }
+
+    public void testUseContextSettingValue() {
+        Settings s = Settings.builder()
+            .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY)
+            .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(),
+                 ScriptService.USE_CONTEXT_RATE_KEY)
+            .build();
+
+        assertEquals(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(s), ScriptService.USE_CONTEXT_RATE_VALUE);
+
+        IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> {
+            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getAsMap(s);
+        });
+
+        assertEquals("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [use-context]", illegal.getMessage());
+    }
+
+    public void testCacheHolderGeneralConstructor() {
+        String compilationRate = "77/5m";
+        Tuple<Integer, TimeValue> rate = ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(compilationRate);
+        boolean compilationLimitsEnabled = true;
+        ScriptService.CacheHolder holder = new ScriptService.CacheHolder(
+            Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build(),
+            new HashSet<>(Collections.singleton("foo")),
+            compilationLimitsEnabled
+        );
+
+        assertNotNull(holder.general);
+        assertNull(holder.contextCache);
+        assertEquals(holder.general.rate, rate);
+
+        compilationLimitsEnabled = false;
+        holder = new ScriptService.CacheHolder(
+            Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build(),
+            new HashSet<>(Collections.singleton("foo")),
+            compilationLimitsEnabled
+        );
+
+        assertNotNull(holder.general);
+        assertNull(holder.contextCache);
+        assertEquals(holder.general.rate, new Tuple<>(0, TimeValue.ZERO));
+    }
+
+    public void testCacheHolderContextConstructor() {
+        String fooCompilationRate = "77/5m";
+        String barCompilationRate = "78/6m";
+        boolean compilationLimitsEnabled = true;
+
+        Settings s = Settings.builder()
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), fooCompilationRate)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), barCompilationRate)
+            .build();
+        Set<String> contexts = Set.of("foo", "bar", "baz");
+        ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, compilationLimitsEnabled);
+
+        assertNull(holder.general);
+        assertNotNull(holder.contextCache);
+        assertEquals(3, holder.contextCache.size());
+        assertEquals(contexts, holder.contextCache.keySet());
+
+        assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate);
+        assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate);
+        assertEquals(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getDefault(Settings.EMPTY),
+                     holder.contextCache.get("baz").get().rate);
+
+        Tuple<Integer, TimeValue> zero = new Tuple<>(0, TimeValue.ZERO);
+        compilationLimitsEnabled = false;
+        holder = new ScriptService.CacheHolder(s, contexts, compilationLimitsEnabled);
+
+        assertNotNull(holder.contextCache);
+        assertEquals(3, holder.contextCache.size());
+        assertEquals(zero, holder.contextCache.get("foo").get().rate);
+        assertEquals(zero, holder.contextCache.get("bar").get().rate);
+        assertEquals(zero, holder.contextCache.get("baz").get().rate);
+    }
+
+    public void testCacheHolderChangeSettings() {
+        String fooCompilationRate = "77/5m";
+        String barCompilationRate = "78/6m";
+        String compilationRate = "77/5m";
+        Tuple<Integer, TimeValue> generalRate = MAX_COMPILATION_RATE_FUNCTION.apply(compilationRate);
+
+        Settings s = Settings.builder()
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate)
+            .build();
+        Set<String> contexts = Set.of("foo", "bar", "baz");
+        ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, true);
+
+        assertNotNull(holder.general);
+        assertNull(holder.contextCache);
+        assertEquals(generalRate, holder.general.rate);
+
+        holder = holder.withUpdatedCacheSettings(Settings.builder()
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), fooCompilationRate)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), barCompilationRate)
+            .build()
+        );
+
+        assertNull(holder.general);
+        assertNotNull(holder.contextCache);
+        assertEquals(3, holder.contextCache.size());
+        assertEquals(contexts, holder.contextCache.keySet());
+
+        assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate);
+        assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate);
+        assertEquals(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getDefault(Settings.EMPTY),
+            holder.contextCache.get("baz").get().rate);
+
+        holder.updateContextSettings(Settings.builder()
+                .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), fooCompilationRate).build(),
+                "bar"
+        );
+        assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("bar").get().rate);
+
+        holder = holder.withUpdatedCacheSettings(s);
+        assertNotNull(holder.general);
+        assertNull(holder.contextCache);
+        assertEquals(generalRate, holder.general.rate);
+
+        holder = holder.withUpdatedCacheSettings(
+            Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), barCompilationRate).build()
+        );
+
+        assertNotNull(holder.general);
+        assertNull(holder.contextCache);
+        assertEquals(MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.general.rate);
+
+        ScriptService.CacheHolder update = holder.withUpdatedCacheSettings(
+            Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), barCompilationRate).build()
+        );
+        assertSame(holder, update);
+    }
+
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
             scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index fef1dcacba309..e913f1bfc8a02 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -495,10 +495,10 @@ private static Settings getRandomNodeSettings(long seed) {
         }
 
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 0, 2000));
+            builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 0, 2000));
         }
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(),
+            builder.put(ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(),
                     timeValueMillis(RandomNumbers.randomIntBetween(random, 750, 10000000)).getStringRep());
         }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index c349ea764101f..9a177e74ab2f0 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -215,7 +215,7 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b");
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b");
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b");
-        builder.put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "2048/1m");
+        builder.put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "2048/1m");
         // wait short time for other active shards before actually deleting, default 30s not needed in tests
         builder.put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
         builder.putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes

From 03caeaad79bd29bda66f3af70791b06fc67a5e4f Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Wed, 18 Mar 2020 14:33:40 -0400
Subject: [PATCH 119/285] [DOCS] Remove incorrect parms from put index template
 API docs (#53750)

Removes the `flat_settings` and `timeout` query parameters from the JSON
spec and asciidoc docs for the put index template API.

These parameters are not supported by the API.
---
 docs/reference/indices/templates.asciidoc                 | 4 +---
 .../resources/rest-api-spec/api/indices.put_template.json | 8 --------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc
index 995efe28eea74..eeea74b5a544a 100644
--- a/docs/reference/indices/templates.asciidoc
+++ b/docs/reference/indices/templates.asciidoc
@@ -93,8 +93,6 @@ Name of the index template to create.
 If `true`, this request cannot replace or update existing index templates.
 Defaults to `false`.
 
-include::{docdir}/rest-api/common-parms.asciidoc[tag=flat-settings]
-
 `order`::
 (Optional,integer)
 Order in which {es} applies this template
@@ -104,7 +102,7 @@ Templates with lower `order` values are merged first.
 Templates with higher `order` values are merged later,
 overriding templates with lower values.
 
-include::{docdir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
+include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
 
 [[put-index-template-api-request-body]]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
index 133776199b3ac..75a328af929ef 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
@@ -32,17 +32,9 @@
         "description":"Whether the index template should only be added if new or can also replace an existing one",
         "default":false
       },
-      "timeout":{
-        "type":"time",
-        "description":"Explicit operation timeout"
-      },
       "master_timeout":{
         "type":"time",
         "description":"Specify timeout for connection to master"
-      },
-      "flat_settings":{
-        "type":"boolean",
-        "description":"Return settings in flat format (default: false)"
       }
     },
     "body":{

From ddedb94b6c0456368d60e9ba9c1a04d55f82d751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Wed, 18 Mar 2020 19:59:30 +0100
Subject: [PATCH 120/285] Add unsupported parameters to HLRC search request
 (#53745)

Currently we don't send values for the `pre_filter_shard_size` and
`max_concurrent_shard_requests` SearchRequest parameters over http when using
the High Level Rest Client. This change adds these parameters to the
RequestConverters and tests.
---
 .../java/org/elasticsearch/client/RequestConverters.java  | 2 ++
 .../org/elasticsearch/client/RequestConvertersTests.java  | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index b9b3c4b31a414..a8532f6d16339 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -410,6 +410,8 @@ private static void addSearchRequestParams(Params params, SearchRequest searchRe
         params.withIndicesOptions(searchRequest.indicesOptions());
         params.putParam("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT));
         params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips()));
+        params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        params.putParam("max_concurrent_shard_requests", Integer.toString(searchRequest.getMaxConcurrentShardRequests()));
         if (searchRequest.requestCache() != null) {
             params.putParam("request_cache", Boolean.toString(searchRequest.requestCache()));
         }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 4e04eb9d07d64..3e4ecda385388 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -1857,6 +1857,14 @@ private static void setRandomSearchParams(SearchRequest searchRequest,
             searchRequest.setCcsMinimizeRoundtrips(randomBoolean());
         }
         expectedParams.put("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips()));
+        if (randomBoolean()) {
+            searchRequest.setMaxConcurrentShardRequests(randomIntBetween(1, Integer.MAX_VALUE));
+        }
+        expectedParams.put("max_concurrent_shard_requests", Integer.toString(searchRequest.getMaxConcurrentShardRequests()));
+        if (randomBoolean()) {
+            searchRequest.setPreFilterShardSize(randomIntBetween(2, Integer.MAX_VALUE));
+        }
+        expectedParams.put("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
     }
 
     static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<IndicesOptions> getter,

From f26ccb2ab10c4fde7b8f2d40cdc7f279d54fbb30 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 18 Mar 2020 20:29:25 +0100
Subject: [PATCH 121/285] Restore bwc tests (#53754)

This change restores the bwc tests now that #53659 is merged.
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index b83f19efbba42..2dbf94ae7dabb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53659" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 9467dbf1208970b7e8e60d3650b53dc0e6151514 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Wed, 18 Mar 2020 12:43:15 -0700
Subject: [PATCH 122/285] Decouple AuditTrailService from AuditTrail (#53450)

The AuditTrailService has historically been an AuditTrail itself, acting
as a composite of the configured audit trails. This commit removes that
interface from the service and instead builds a composite delegating
implementation internally. The service now has a single get() method to
get an AuditTrail implementation which may be called. If auditing is not
allowed by the license, an empty noop version is returned.
---
 .../security/audit/AuditTrailService.java     | 263 +++++++++++-------
 .../security/authc/AuthenticationService.java |  13 +-
 .../security/authz/AuthorizationService.java  |  22 +-
 .../SecuritySearchOperationListener.java      |   2 +-
 .../IndicesAliasesRequestInterceptor.java     |   4 +-
 .../interceptor/ResizeRequestInterceptor.java |   4 +-
 .../security/transport/filter/IPFilter.java   |   8 +-
 .../audit/AuditTrailServiceTests.java         |  26 +-
 .../authc/AuthenticationServiceTests.java     |  35 ++-
 .../support/SecondaryAuthenticatorTests.java  |   3 +-
 .../authz/AuthorizationServiceTests.java      |  21 +-
 .../SecuritySearchOperationListenerTests.java |  39 +--
 .../transport/filter/IPFilterTests.java       |  34 ++-
 13 files changed, 287 insertions(+), 187 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
index ddf6d49a632ae..fb84df489ff65 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
@@ -19,219 +19,280 @@
 import java.util.Collections;
 import java.util.List;
 
-public class AuditTrailService implements AuditTrail {
+public class AuditTrailService {
 
+    private static final AuditTrail NOOP_AUDIT_TRAIL = new NoopAuditTrail();
+    private final CompositeAuditTrail compositeAuditTrail;
     private final XPackLicenseState licenseState;
-    private final List<AuditTrail> auditTrails;
-
-    @Override
-    public String name() {
-        return "service";
-    }
 
     public AuditTrailService(List<AuditTrail> auditTrails, XPackLicenseState licenseState) {
-        this.auditTrails = Collections.unmodifiableList(auditTrails);
+        this.compositeAuditTrail = new CompositeAuditTrail(Collections.unmodifiableList(auditTrails));
         this.licenseState = licenseState;
     }
 
-    /** Returns the audit trail implementations that this service delegates to. */
+    public AuditTrail get() {
+        if (compositeAuditTrail.isEmpty() == false && licenseState.isAuditingAllowed()) {
+            return compositeAuditTrail;
+        } else {
+            return NOOP_AUDIT_TRAIL;
+        }
+    }
+
+    // TODO: this method only exists for access to LoggingAuditTrail in a Node for testing.
+    // DO NOT USE IT, IT WILL BE REMOVED IN THE FUTURE
     public List<AuditTrail> getAuditTrails() {
-        return auditTrails;
+        return compositeAuditTrail.auditTrails;
+    }
+
+    private static class NoopAuditTrail implements AuditTrail {
+
+        @Override
+        public String name() {
+            return "noop";
+        }
+
+        @Override
+        public void authenticationSuccess(String requestId, String realm, User user, RestRequest request) {}
+
+        @Override
+        public void authenticationSuccess(String requestId, String realm, User user, String action, TransportMessage message) {}
+
+        @Override
+        public void anonymousAccessDenied(String requestId, String action, TransportMessage message) {}
+
+        @Override
+        public void anonymousAccessDenied(String requestId, RestRequest request) {}
+
+        @Override
+        public void authenticationFailed(String requestId, RestRequest request) {}
+
+        @Override
+        public void authenticationFailed(String requestId, String action, TransportMessage message) {}
+
+        @Override
+        public void authenticationFailed(String requestId, AuthenticationToken token, String action, TransportMessage message) {}
+
+        @Override
+        public void authenticationFailed(String requestId, AuthenticationToken token, RestRequest request) {}
+
+        @Override
+        public void authenticationFailed(String requestId, String realm, AuthenticationToken token,
+                                         String action, TransportMessage message) {}
+
+        @Override
+        public void authenticationFailed(String requestId, String realm, AuthenticationToken token, RestRequest request) {}
+
+        @Override
+        public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                                  AuthorizationInfo authorizationInfo) {}
+
+        @Override
+        public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                                 AuthorizationInfo authorizationInfo) {}
+
+        @Override
+        public void tamperedRequest(String requestId, RestRequest request) {}
+
+        @Override
+        public void tamperedRequest(String requestId, String action, TransportMessage message) {}
+
+        @Override
+        public void tamperedRequest(String requestId, User user, String action, TransportMessage request) {}
+
+        @Override
+        public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {}
+
+        @Override
+        public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {}
+
+        @Override
+        public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                                 AuthorizationInfo authorizationInfo) {}
+
+        @Override
+        public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                                AuthorizationInfo authorizationInfo) {}
+
+        @Override
+        public void runAsDenied(String requestId, Authentication authentication, RestRequest request,
+                                AuthorizationInfo authorizationInfo) {}
+
+        @Override
+        public void explicitIndexAccessEvent(String requestId, AuditLevel eventType, Authentication authentication,
+                                             String action, String indices, String requestName, TransportAddress remoteAddress,
+                                             AuthorizationInfo authorizationInfo) {}
     }
 
-    @Override
-    public void authenticationSuccess(String requestId, String realm, User user, RestRequest request) {
-        if (licenseState.isAuditingAllowed()) {
+    private static class CompositeAuditTrail implements AuditTrail {
+
+        private final List<AuditTrail> auditTrails;
+
+        private CompositeAuditTrail(List<AuditTrail> auditTrails) {
+            this.auditTrails = auditTrails;
+        }
+
+        boolean isEmpty() {
+            return auditTrails.isEmpty();
+        }
+
+        @Override
+        public String name() {
+            return "service";
+        }
+
+        @Override
+        public void authenticationSuccess(String requestId, String realm, User user, RestRequest request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationSuccess(requestId, realm, user, request);
             }
         }
-    }
 
-    @Override
-    public void authenticationSuccess(String requestId, String realm, User user, String action, TransportMessage message) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationSuccess(String requestId, String realm, User user, String action, TransportMessage message) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationSuccess(requestId, realm, user, action, message);
             }
         }
-    }
 
-    @Override
-    public void anonymousAccessDenied(String requestId, String action, TransportMessage message) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void anonymousAccessDenied(String requestId, String action, TransportMessage message) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.anonymousAccessDenied(requestId, action, message);
             }
         }
-    }
 
-    @Override
-    public void anonymousAccessDenied(String requestId, RestRequest request) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void anonymousAccessDenied(String requestId, RestRequest request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.anonymousAccessDenied(requestId, request);
             }
         }
-    }
 
-    @Override
-    public void authenticationFailed(String requestId, RestRequest request) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationFailed(String requestId, RestRequest request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationFailed(requestId, request);
             }
         }
-    }
 
-    @Override
-    public void authenticationFailed(String requestId, String action, TransportMessage message) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationFailed(String requestId, String action, TransportMessage message) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationFailed(requestId, action, message);
             }
         }
-    }
 
-    @Override
-    public void authenticationFailed(String requestId, AuthenticationToken token, String action, TransportMessage message) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationFailed(String requestId, AuthenticationToken token, String action, TransportMessage message) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationFailed(requestId, token, action, message);
             }
         }
-    }
 
-    @Override
-    public void authenticationFailed(String requestId, String realm, AuthenticationToken token, String action, TransportMessage message) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationFailed(String requestId, String realm, AuthenticationToken token, String action,
+                                         TransportMessage message) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationFailed(requestId, realm, token, action, message);
             }
         }
-    }
 
-    @Override
-    public void authenticationFailed(String requestId, AuthenticationToken token, RestRequest request) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationFailed(String requestId, AuthenticationToken token, RestRequest request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationFailed(requestId, token, request);
             }
         }
-    }
 
-    @Override
-    public void authenticationFailed(String requestId, String realm, AuthenticationToken token, RestRequest request) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void authenticationFailed(String requestId, String realm, AuthenticationToken token, RestRequest request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.authenticationFailed(requestId, realm, token, request);
             }
         }
-    }
 
-    @Override
-    public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage msg,
-                              AuthorizationInfo authorizationInfo) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage msg,
+                                  AuthorizationInfo authorizationInfo) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.accessGranted(requestId, authentication, action, msg, authorizationInfo);
             }
         }
-    }
 
-    @Override
-    public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message,
-                             AuthorizationInfo authorizationInfo) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                                 AuthorizationInfo authorizationInfo) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.accessDenied(requestId, authentication, action, message, authorizationInfo);
             }
         }
-    }
 
-    @Override
-    public void tamperedRequest(String requestId, RestRequest request) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void tamperedRequest(String requestId, RestRequest request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.tamperedRequest(requestId, request);
             }
         }
-    }
 
-    @Override
-    public void tamperedRequest(String requestId, String action, TransportMessage message) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void tamperedRequest(String requestId, String action, TransportMessage message) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.tamperedRequest(requestId, action, message);
             }
         }
-    }
 
-    @Override
-    public void tamperedRequest(String requestId, User user, String action, TransportMessage request) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void tamperedRequest(String requestId, User user, String action, TransportMessage request) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.tamperedRequest(requestId, user, action, request);
             }
         }
-    }
 
-    @Override
-    public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.connectionGranted(inetAddress, profile, rule);
             }
         }
-    }
 
-    @Override
-    public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.connectionDenied(inetAddress, profile, rule);
             }
         }
-    }
 
-    @Override
-    public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message,
-                             AuthorizationInfo authorizationInfo) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                                 AuthorizationInfo authorizationInfo) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.runAsGranted(requestId, authentication, action, message, authorizationInfo);
             }
         }
-    }
 
-    @Override
-    public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message,
-                            AuthorizationInfo authorizationInfo) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                                AuthorizationInfo authorizationInfo) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.runAsDenied(requestId, authentication, action, message, authorizationInfo);
             }
         }
-    }
 
-    @Override
-    public void runAsDenied(String requestId, Authentication authentication, RestRequest request,
-                            AuthorizationInfo authorizationInfo) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void runAsDenied(String requestId, Authentication authentication, RestRequest request,
+                                AuthorizationInfo authorizationInfo) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.runAsDenied(requestId, authentication, request, authorizationInfo);
             }
         }
-    }
 
-    @Override
-    public void explicitIndexAccessEvent(String requestId, AuditLevel eventType, Authentication authentication, String action,
-                                         String indices, String requestName, TransportAddress remoteAddress,
-                                         AuthorizationInfo authorizationInfo) {
-        if (licenseState.isAuditingAllowed()) {
+        @Override
+        public void explicitIndexAccessEvent(String requestId, AuditLevel eventType, Authentication authentication, String action,
+                                             String indices, String requestName, TransportAddress remoteAddress,
+                                             AuthorizationInfo authorizationInfo) {
             for (AuditTrail auditTrail : auditTrails) {
                 auditTrail.explicitIndexAccessEvent(requestId, eventType, authentication, action, indices, requestName, remoteAddress,
-                        authorizationInfo);
+                    authorizationInfo);
             }
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
index 8a9be8b5a9cc2..fe46d078ee13e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
@@ -77,7 +77,7 @@ public class AuthenticationService {
     private static final Logger logger = LogManager.getLogger(AuthenticationService.class);
 
     private final Realms realms;
-    private final AuditTrail auditTrail;
+    private final AuditTrailService auditTrailService;
     private final AuthenticationFailureHandler failureHandler;
     private final ThreadContext threadContext;
     private final String nodeName;
@@ -90,12 +90,12 @@ public class AuthenticationService {
     private final boolean isAnonymousUserEnabled;
     private final AuthenticationContextSerializer authenticationSerializer;
 
-    public AuthenticationService(Settings settings, Realms realms, AuditTrailService auditTrail,
+    public AuthenticationService(Settings settings, Realms realms, AuditTrailService auditTrailService,
                                  AuthenticationFailureHandler failureHandler, ThreadPool threadPool,
                                  AnonymousUser anonymousUser, TokenService tokenService, ApiKeyService apiKeyService) {
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.realms = realms;
-        this.auditTrail = auditTrail;
+        this.auditTrailService = auditTrailService;
         this.failureHandler = failureHandler;
         this.threadContext = threadPool.getThreadContext();
         this.anonymousUser = anonymousUser;
@@ -274,16 +274,17 @@ class Authenticator {
         private AuthenticationResult authenticationResult = null;
 
         Authenticator(RestRequest request, boolean fallbackToAnonymous, ActionListener<Authentication> listener) {
-            this(new AuditableRestRequest(auditTrail, failureHandler, threadContext, request), null, fallbackToAnonymous, listener);
+            this(new AuditableRestRequest(auditTrailService.get(), failureHandler, threadContext, request),
+                 null, fallbackToAnonymous, listener);
         }
 
         Authenticator(String action, TransportMessage message, boolean fallbackToAnonymous, ActionListener<Authentication> listener) {
-            this(new AuditableTransportRequest(auditTrail, failureHandler, threadContext, action, message),
+            this(new AuditableTransportRequest(auditTrailService.get(), failureHandler, threadContext, action, message),
                 null, fallbackToAnonymous, listener);
         }
 
         Authenticator(String action, TransportMessage message, User fallbackUser, ActionListener<Authentication> listener) {
-            this(new AuditableTransportRequest(auditTrail, failureHandler, threadContext, action, message),
+            this(new AuditableTransportRequest(auditTrailService.get(), failureHandler, threadContext, action, message),
                 Objects.requireNonNull(fallbackUser, "Fallback user cannot be null"), false, listener);
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index 40f5e4f711ada..67d88efc0bc1d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -63,6 +63,7 @@
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
@@ -101,7 +102,7 @@ public class AuthorizationService {
 
     private final Settings settings;
     private final ClusterService clusterService;
-    private final AuditTrailService auditTrail;
+    private final AuditTrailService auditTrailService;
     private final IndicesAndAliasesResolver indicesAndAliasesResolver;
     private final AuthenticationFailureHandler authcFailureHandler;
     private final ThreadContext threadContext;
@@ -114,12 +115,12 @@ public class AuthorizationService {
     private final boolean anonymousAuthzExceptionEnabled;
 
     public AuthorizationService(Settings settings, CompositeRolesStore rolesStore, ClusterService clusterService,
-                                AuditTrailService auditTrail, AuthenticationFailureHandler authcFailureHandler,
+                                AuditTrailService auditTrailService, AuthenticationFailureHandler authcFailureHandler,
                                 ThreadPool threadPool, AnonymousUser anonymousUser, @Nullable AuthorizationEngine authorizationEngine,
                                 Set<RequestInterceptor> requestInterceptors, XPackLicenseState licenseState,
                                 IndexNameExpressionResolver resolver) {
         this.clusterService = clusterService;
-        this.auditTrail = auditTrail;
+        this.auditTrailService = auditTrailService;
         this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver);
         this.authcFailureHandler = authcFailureHandler;
         this.threadContext = threadPool.getThreadContext();
@@ -172,7 +173,7 @@ public void authorize(final Authentication authentication, final String action,
             if (isInternalUser(authentication.getUser()) != false) {
                 auditId = AuditUtil.getOrGenerateRequestId(threadContext);
             } else {
-                auditTrail.tamperedRequest(null, authentication.getUser(), action, originalRequest);
+                auditTrailService.get().tamperedRequest(null, authentication.getUser(), action, originalRequest);
                 final String message = "Attempt to authorize action [" + action + "] for [" + authentication.getUser().principal()
                     + "] without an existing request-id";
                 assert false : message;
@@ -204,6 +205,7 @@ private void maybeAuthorizeRunAs(final RequestInfo requestInfo, final String req
         final TransportRequest request = requestInfo.getRequest();
         final String action = requestInfo.getAction();
         final boolean isRunAs = authentication.getUser().isRunAs();
+        final AuditTrail auditTrail = auditTrailService.get();
         if (isRunAs) {
             ActionListener<AuthorizationResult> runAsListener = wrapPreservingContext(ActionListener.wrap(result -> {
                 if (result.isGranted()) {
@@ -236,6 +238,7 @@ private void authorizeAction(final RequestInfo requestInfo, final String request
         final TransportRequest request = requestInfo.getRequest();
         final String action = requestInfo.getAction();
         final AuthorizationEngine authzEngine = getAuthorizationEngine(authentication);
+        final AuditTrail auditTrail = auditTrailService.get();
         if (ClusterPrivilegeResolver.isClusterAction(action)) {
             final ActionListener<AuthorizationResult> clusterAuthzListener =
                 wrapPreservingContext(new AuthorizationResultListener<>(result -> {
@@ -373,6 +376,7 @@ private AuthorizationEngine getAuthorizationEngineForUser(final User user) {
 
     private void authorizeSystemUser(final Authentication authentication, final String action, final String requestId,
                                      final TransportRequest request, final ActionListener<Void> listener) {
+        final AuditTrail auditTrail = auditTrailService.get();
         if (SystemUser.isAuthorized(action)) {
             putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
             putTransientIfNonExisting(AUTHORIZATION_INFO_KEY, SYSTEM_AUTHZ_INFO);
@@ -394,6 +398,7 @@ private TransportRequest maybeUnwrapRequest(Authentication authentication, Trans
             request = TransportActionProxy.unwrapRequest(originalRequest);
             final boolean isOriginalRequestProxyRequest = TransportActionProxy.isProxyRequest(originalRequest);
             final boolean isProxyAction = TransportActionProxy.isProxyAction(action);
+            final AuditTrail auditTrail = auditTrailService.get();
             if (isProxyAction && isOriginalRequestProxyRequest == false) {
                 IllegalStateException cause = new IllegalStateException("originalRequest is not a proxy request: [" + originalRequest +
                     "] but action: [" + action + "] is a proxy action");
@@ -451,6 +456,7 @@ private void authorizeBulkItems(RequestInfo requestInfo, AuthorizationInfo authz
         final Map<String, String> resolvedIndexNames = new HashMap<>();
         // Maps action -> resolved indices set
         final Map<String, Set<String>> actionToIndicesMap = new HashMap<>();
+        final AuditTrail auditTrail = auditTrailService.get();
 
         authorizedIndicesSupplier.getAsync(ActionListener.wrap(authorizedIndices -> {
             resolvedIndicesAsyncSupplier.getAsync(ActionListener.wrap(overallResolvedIndices -> {
@@ -611,8 +617,8 @@ private AuthorizationResultListener(Consumer<T> responseConsumer, Consumer<Excep
         public void onResponse(T result) {
             if (result.isGranted()) {
                 if (result.isAuditable()) {
-                    auditTrail.accessGranted(requestId, requestInfo.getAuthentication(), requestInfo.getAction(), requestInfo.getRequest(),
-                        authzInfo);
+                    auditTrailService.get().accessGranted(requestId, requestInfo.getAuthentication(),
+                        requestInfo.getAction(), requestInfo.getRequest(), authzInfo);
                 }
                 try {
                     responseConsumer.accept(result);
@@ -631,8 +637,8 @@ public void onFailure(Exception e) {
 
         private void handleFailure(boolean audit, @Nullable Exception e) {
             if (audit) {
-                auditTrail.accessDenied(requestId, requestInfo.getAuthentication(), requestInfo.getAction(), requestInfo.getRequest(),
-                    authzInfo);
+                auditTrailService.get().accessDenied(requestId, requestInfo.getAuthentication(), requestInfo.getAction(),
+                    requestInfo.getRequest(), authzInfo);
             }
             failureConsumer.accept(denialException(requestInfo.getAuthentication(), requestInfo.getAction(), e));
         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index d898003a50147..4ac73e9c9df4b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -99,7 +99,7 @@ static void ensureAuthenticatedUserIsSame(Authentication original, Authenticatio
 
         final boolean sameUser = samePrincipal && sameRealmType;
         if (sameUser == false) {
-            auditTrailService.accessDenied(requestId, current, action, request, authorizationInfo);
+            auditTrailService.get().accessDenied(requestId, current, action, request, authorizationInfo);
             throw new SearchContextMissingException(id);
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
index cefa42eb0b9bd..1df0f3ae95076 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 
@@ -48,6 +49,7 @@ public void intercept(RequestInfo requestInfo, AuthorizationEngine authorization
         if (requestInfo.getRequest() instanceof IndicesAliasesRequest) {
             final IndicesAliasesRequest request = (IndicesAliasesRequest) requestInfo.getRequest();
             final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
+            final AuditTrail auditTrail = auditTrailService.get();
             if (frozenLicenseState.isAuthAllowed()) {
                 if (frozenLicenseState.isDocumentAndFieldLevelSecurityAllowed()) {
                     IndicesAccessControl indicesAccessControl =
@@ -89,7 +91,7 @@ public void intercept(RequestInfo requestInfo, AuthorizationEngine authorization
                         // do not audit success again
                         listener.onResponse(null);
                     } else {
-                        auditTrailService.accessDenied(AuditUtil.extractRequestId(threadContext), requestInfo.getAuthentication(),
+                        auditTrail.accessDenied(AuditUtil.extractRequestId(threadContext), requestInfo.getAuthentication(),
                             requestInfo.getAction(), request, authorizationInfo);
                         listener.onFailure(Exceptions.authorizationError("Adding an alias is not allowed when the alias " +
                             "has more permissions than any of the indices"));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
index ba4f7a61faf1e..a1190482e0b84 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import java.util.Collections;
@@ -44,6 +45,7 @@ public void intercept(RequestInfo requestInfo, AuthorizationEngine authorization
         if (requestInfo.getRequest() instanceof ResizeRequest) {
             final ResizeRequest request = (ResizeRequest) requestInfo.getRequest();
             final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
+            final AuditTrail auditTrail = auditTrailService.get();
             if (frozenLicenseState.isAuthAllowed()) {
                 if (frozenLicenseState.isDocumentAndFieldLevelSecurityAllowed()) {
                     IndicesAccessControl indicesAccessControl =
@@ -68,7 +70,7 @@ public void intercept(RequestInfo requestInfo, AuthorizationEngine authorization
                             listener.onResponse(null);
                         } else {
                             if (authzResult.isAuditable()) {
-                                auditTrailService.accessDenied(extractRequestId(threadContext), requestInfo.getAuthentication(),
+                                auditTrail.accessDenied(extractRequestId(threadContext), requestInfo.getAuthentication(),
                                     requestInfo.getAction(), request, authorizationInfo);
                             }
                             listener.onFailure(Exceptions.authorizationError("Resizing an index is not allowed when the target index " +
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java
index 1dfedd3f87d42..750b0b2a496ac 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportSettings;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import java.net.InetSocketAddress;
@@ -96,7 +97,7 @@ public IpFilterRuleType ruleType() {
 
     private static final Logger logger = LogManager.getLogger(IPFilter.class);
 
-    private final AuditTrailService auditTrail;
+    private final AuditTrailService auditTrailService;
     private final XPackLicenseState licenseState;
     private final boolean alwaysAllowBoundAddresses;
 
@@ -114,9 +115,9 @@ public IpFilterRuleType ruleType() {
     private final Map<String, List<String>> profileAllowRules = Collections.synchronizedMap(new HashMap<>());
     private final Map<String, List<String>> profileDenyRules = Collections.synchronizedMap(new HashMap<>());
 
-    public IPFilter(final Settings settings, AuditTrailService auditTrail, ClusterSettings clusterSettings,
+    public IPFilter(final Settings settings, AuditTrailService auditTrailService, ClusterSettings clusterSettings,
                     XPackLicenseState licenseState) {
-        this.auditTrail = auditTrail;
+        this.auditTrailService = auditTrailService;
         this.licenseState = licenseState;
         this.alwaysAllowBoundAddresses = ALLOW_BOUND_ADDRESSES_SETTING.get(settings);
         httpDenyFilter = HTTP_FILTER_DENY_SETTING.get(settings);
@@ -205,6 +206,7 @@ public boolean accept(String profile, InetSocketAddress peerAddress) {
             return true;
         }
 
+        AuditTrail auditTrail = auditTrailService.get();
         for (SecurityIpFilterRule rule : rules.get(profile)) {
             if (rule.matches(peerAddress)) {
                 boolean isAllowed = rule.ruleType() == IpFilterRuleType.ACCEPT;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
index fb194ecefc671..f8951f7a43aca 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
@@ -58,7 +58,7 @@ public void init() throws Exception {
 
     public void testAuthenticationFailed() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationFailed(requestId, token, "_action", message);
+        service.get().authenticationFailed(requestId, token, "_action", message);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -71,7 +71,7 @@ public void testAuthenticationFailed() throws Exception {
 
     public void testAuthenticationFailedNoToken() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationFailed(requestId, "_action", message);
+        service.get().authenticationFailed(requestId, "_action", message);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -84,7 +84,7 @@ public void testAuthenticationFailedNoToken() throws Exception {
 
     public void testAuthenticationFailedRestNoToken() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationFailed(requestId, restRequest);
+        service.get().authenticationFailed(requestId, restRequest);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -97,7 +97,7 @@ public void testAuthenticationFailedRestNoToken() throws Exception {
 
     public void testAuthenticationFailedRest() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationFailed(requestId, token, restRequest);
+        service.get().authenticationFailed(requestId, token, restRequest);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -110,7 +110,7 @@ public void testAuthenticationFailedRest() throws Exception {
 
     public void testAuthenticationFailedRealm() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationFailed(requestId, "_realm", token, "_action", message);
+        service.get().authenticationFailed(requestId, "_realm", token, "_action", message);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -123,7 +123,7 @@ public void testAuthenticationFailedRealm() throws Exception {
 
     public void testAuthenticationFailedRestRealm() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationFailed(requestId, "_realm", token, restRequest);
+        service.get().authenticationFailed(requestId, "_realm", token, restRequest);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -136,7 +136,7 @@ public void testAuthenticationFailedRestRealm() throws Exception {
 
     public void testAnonymousAccess() throws Exception {
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.anonymousAccessDenied(requestId, "_action", message);
+        service.get().anonymousAccessDenied(requestId, "_action", message);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -153,7 +153,7 @@ public void testAccessGranted() throws Exception {
         AuthorizationInfo authzInfo =
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) });
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.accessGranted(requestId, authentication, "_action", message, authzInfo);
+        service.get().accessGranted(requestId, authentication, "_action", message, authzInfo);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -170,7 +170,7 @@ public void testAccessDenied() throws Exception {
         AuthorizationInfo authzInfo =
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) });
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.accessDenied(requestId, authentication, "_action", message, authzInfo);
+        service.get().accessDenied(requestId, authentication, "_action", message, authzInfo);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -184,7 +184,7 @@ public void testAccessDenied() throws Exception {
     public void testConnectionGranted() throws Exception {
         InetAddress inetAddress = InetAddress.getLoopbackAddress();
         SecurityIpFilterRule rule = randomBoolean() ? SecurityIpFilterRule.ACCEPT_ALL : IPFilter.DEFAULT_PROFILE_ACCEPT_ALL;
-        service.connectionGranted(inetAddress, "client", rule);
+        service.get().connectionGranted(inetAddress, "client", rule);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -198,7 +198,7 @@ public void testConnectionGranted() throws Exception {
     public void testConnectionDenied() throws Exception {
         InetAddress inetAddress = InetAddress.getLoopbackAddress();
         SecurityIpFilterRule rule = new SecurityIpFilterRule(false, "_all");
-        service.connectionDenied(inetAddress, "client", rule);
+        service.get().connectionDenied(inetAddress, "client", rule);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -213,7 +213,7 @@ public void testAuthenticationSuccessRest() throws Exception {
         User user = new User("_username", "r1");
         String realm = "_realm";
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationSuccess(requestId, realm, user, restRequest);
+        service.get().authenticationSuccess(requestId, realm, user, restRequest);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
@@ -228,7 +228,7 @@ public void testAuthenticationSuccessTransport() throws Exception {
         User user = new User("_username", "r1");
         String realm = "_realm";
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.authenticationSuccess(requestId, realm, user, "_action", message);
+        service.get().authenticationSuccess(requestId, realm, user, "_action", message);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index 953442d3a658d..e8bb282d6de4a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -75,6 +75,7 @@
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authc.AuthenticationService.Authenticator;
@@ -147,7 +148,8 @@ public class AuthenticationServiceTests extends ESTestCase {
     private Realms realms;
     private Realm firstRealm;
     private Realm secondRealm;
-    private AuditTrailService auditTrail;
+    private AuditTrail auditTrail;
+    private AuditTrailService auditTrailService;
     private AuthenticationToken token;
     private ThreadPool threadPool;
     private ThreadContext threadContext;
@@ -190,6 +192,7 @@ public void init() throws Exception {
         when(licenseState.isAuthAllowed()).thenReturn(true);
         when(licenseState.isApiKeyServiceAllowed()).thenReturn(true);
         when(licenseState.isTokenServiceAllowed()).thenReturn(true);
+        when(licenseState.isAuditingAllowed()).thenReturn(true);
         ReservedRealm reservedRealm = mock(ReservedRealm.class);
         when(reservedRealm.type()).thenReturn("reserved");
         when(reservedRealm.name()).thenReturn("reserved_realm");
@@ -197,7 +200,8 @@ public void init() throws Exception {
                 licenseState, threadContext, reservedRealm, Arrays.asList(firstRealm, secondRealm),
                 Collections.singletonList(firstRealm)));
 
-        auditTrail = mock(AuditTrailService.class);
+        auditTrail = mock(AuditTrail.class);
+        auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
         client = mock(Client.class);
         threadPool = new ThreadPool(settings,
                 new FixedExecutorBuilder(settings, TokenService.THREAD_POOL_NAME, 1, 1000, "xpack.security.authc.token.thread_pool",
@@ -237,7 +241,8 @@ public void init() throws Exception {
         apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, licenseState, securityIndex, clusterService, threadPool);
         tokenService = new TokenService(settings, Clock.systemUTC(), client, licenseState, securityContext, securityIndex, securityIndex,
             clusterService);
-        service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
+        service = new AuthenticationService(settings, realms, auditTrailService,
+            new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
             threadPool, new AnonymousUser(settings), tokenService, apiKeyService);
     }
 
@@ -421,7 +426,7 @@ public void testAuthenticateSmartRealmOrderingDisabled() {
         final Settings settings = Settings.builder()
             .put(AuthenticationService.SUCCESS_AUTH_CACHE_ENABLED.getKey(), false)
             .build();
-        service = new AuthenticationService(settings, realms, auditTrail,
+        service = new AuthenticationService(settings, realms, auditTrailService,
             new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(Settings.EMPTY),
             tokenService, apiKeyService);
         User user = new User("_username", "r1");
@@ -686,7 +691,7 @@ public void testAuthenticateTransportContextAndHeader() throws Exception {
         ThreadPool threadPool1 = new TestThreadPool("testAutheticateTransportContextAndHeader1");
         try {
             ThreadContext threadContext1 = threadPool1.getThreadContext();
-            service = new AuthenticationService(Settings.EMPTY, realms, auditTrail,
+            service = new AuthenticationService(Settings.EMPTY, realms, auditTrailService,
                 new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool1, new AnonymousUser(Settings.EMPTY),
                 tokenService, apiKeyService);
 
@@ -710,7 +715,7 @@ public void testAuthenticateTransportContextAndHeader() throws Exception {
             ThreadContext threadContext2 = threadPool2.getThreadContext();
             final String header;
             try (ThreadContext.StoredContext ignore = threadContext2.stashContext()) {
-                service = new AuthenticationService(Settings.EMPTY, realms, auditTrail,
+                service = new AuthenticationService(Settings.EMPTY, realms, auditTrailService,
                     new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool2, new AnonymousUser(Settings.EMPTY),
                     tokenService, apiKeyService);
                 threadContext2.putHeader(AuthenticationField.AUTHENTICATION_KEY, authHeaderRef.get());
@@ -724,7 +729,7 @@ public void testAuthenticateTransportContextAndHeader() throws Exception {
             }
 
             threadPool2.getThreadContext().putHeader(AuthenticationField.AUTHENTICATION_KEY, header);
-            service = new AuthenticationService(Settings.EMPTY, realms, auditTrail,
+            service = new AuthenticationService(Settings.EMPTY, realms, auditTrailService,
                 new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool2, new AnonymousUser(Settings.EMPTY),
                 tokenService, apiKeyService);
             service.authenticate("_action", new InternalMessage(), SystemUser.INSTANCE, ActionListener.wrap(result -> {
@@ -763,7 +768,7 @@ public void testWrongTokenDoesNotFallbackToAnonymous() {
         }
         Settings anonymousEnabledSettings = builder.build();
         final AnonymousUser anonymousUser = new AnonymousUser(anonymousEnabledSettings);
-        service = new AuthenticationService(anonymousEnabledSettings, realms, auditTrail,
+        service = new AuthenticationService(anonymousEnabledSettings, realms, auditTrailService,
             new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, anonymousUser, tokenService, apiKeyService);
 
         try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
@@ -786,7 +791,7 @@ public void testWrongApiKeyDoesNotFallbackToAnonymous() {
         }
         Settings anonymousEnabledSettings = builder.build();
         final AnonymousUser anonymousUser = new AnonymousUser(anonymousEnabledSettings);
-        service = new AuthenticationService(anonymousEnabledSettings, realms, auditTrail,
+        service = new AuthenticationService(anonymousEnabledSettings, realms, auditTrailService,
             new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, anonymousUser, tokenService, apiKeyService);
         doAnswer(invocationOnMock -> {
             final GetRequest request = (GetRequest) invocationOnMock.getArguments()[0];
@@ -816,7 +821,8 @@ public void testAnonymousUserRest() throws Exception {
         }
         Settings settings = builder.build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
+        service = new AuthenticationService(settings, realms, auditTrailService,
+            new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
             threadPool, anonymousUser, tokenService, apiKeyService);
         RestRequest request = new FakeRestRequest();
 
@@ -841,7 +847,8 @@ public void testAuthenticateRestRequestDisallowAnonymous() throws Exception {
         Settings settings = builder.build();
 
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
+        service = new AuthenticationService(settings, realms, auditTrailService,
+            new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
             threadPool, anonymousUser, tokenService, apiKeyService);
         RestRequest request = new FakeRestRequest();
 
@@ -863,7 +870,8 @@ public void testAnonymousUserTransportNoDefaultUser() throws Exception {
                 .putList(AnonymousUser.ROLES_SETTING.getKey(), "r1", "r2", "r3")
                 .build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
+        service = new AuthenticationService(settings, realms, auditTrailService,
+            new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
             threadPool, anonymousUser, tokenService, apiKeyService);
         InternalMessage message = new InternalMessage();
 
@@ -879,7 +887,8 @@ public void testAnonymousUserTransportWithDefaultUser() throws Exception {
                 .putList(AnonymousUser.ROLES_SETTING.getKey(), "r1", "r2", "r3")
                 .build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
+        service = new AuthenticationService(settings, realms, auditTrailService,
+            new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
             threadPool, anonymousUser, tokenService, apiKeyService);
 
         InternalMessage message = new InternalMessage();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
index 2daf11ab4d23d..d13f03f531bac 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
@@ -57,6 +57,7 @@
 import java.nio.charset.StandardCharsets;
 import java.time.Clock;
 import java.util.Base64;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -97,7 +98,7 @@ public void setupMocks() throws Exception {
         when(realms.asList()).thenReturn(List.of(realm));
         when(realms.getUnlicensedRealms()).thenReturn(List.of());
 
-        final AuditTrailService auditTrail = mock(AuditTrailService.class);
+        final AuditTrailService auditTrail = new AuditTrailService(Collections.emptyList(), null);
         final AuthenticationFailureHandler failureHandler = new DefaultAuthenticationFailureHandler(Map.of());
         final AnonymousUser anonymous = new AnonymousUser(settings);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
index e1bb5dfac02cc..7bed19e4aab87 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -127,6 +127,7 @@
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
@@ -178,7 +179,8 @@
 import static org.mockito.Mockito.when;
 
 public class AuthorizationServiceTests extends ESTestCase {
-    private AuditTrailService auditTrail;
+    private AuditTrail auditTrail;
+    private AuditTrailService auditTrailService;
     private ClusterService clusterService;
     private AuthorizationService authorizationService;
     private ThreadContext threadContext;
@@ -197,7 +199,10 @@ public void setup() {
         final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
-        auditTrail = mock(AuditTrailService.class);
+        auditTrail = mock(AuditTrail.class);
+        XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isAuditingAllowed()).thenReturn(true);
+        auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
         threadContext = new ThreadContext(settings);
         threadPool = mock(ThreadPool.class);
         when(threadPool.getThreadContext()).thenReturn(threadContext);
@@ -238,8 +243,8 @@ public void setup() {
         }).when(rolesStore).getRoles(any(User.class), any(Authentication.class), any(ActionListener.class));
         roleMap.put(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR);
         authorizationService = new AuthorizationService(settings, rolesStore, clusterService,
-            auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(settings), null,
-            Collections.emptySet(), new XPackLicenseState(settings), new IndexNameExpressionResolver());
+            auditTrailService, new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(settings),
+            null, Collections.emptySet(), licenseState, new IndexNameExpressionResolver());
     }
 
     private void authorize(Authentication authentication, String action, TransportRequest request) {
@@ -676,7 +681,7 @@ public void testDenialForAnonymousUser() throws IOException {
         ClusterState state = mockEmptyMetaData();
         Settings settings = Settings.builder().put(AnonymousUser.ROLES_SETTING.getKey(), "a_all").build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrail,
+        authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrailService,
             new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, anonymousUser, null, Collections.emptySet(),
             new XPackLicenseState(settings), new IndexNameExpressionResolver());
 
@@ -704,7 +709,7 @@ public void testDenialForAnonymousUserAuthorizationExceptionDisabled() throws IO
             .put(AuthorizationService.ANONYMOUS_AUTHORIZATION_EXCEPTION_SETTING.getKey(), false)
             .build();
         final Authentication authentication = createAuthentication(new AnonymousUser(settings));
-        authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrail,
+        authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrailService,
             new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(settings), null,
             Collections.emptySet(), new XPackLicenseState(settings), new IndexNameExpressionResolver());
 
@@ -1448,8 +1453,8 @@ public void getUserPrivileges(Authentication authentication, AuthorizationInfo a
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isAuthorizationEngineAllowed()).thenReturn(true);
         authorizationService = new AuthorizationService(Settings.EMPTY, rolesStore, clusterService,
-            auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool, new AnonymousUser(Settings.EMPTY),
-            engine, Collections.emptySet(), licenseState, new IndexNameExpressionResolver());
+            auditTrailService, new DefaultAuthenticationFailureHandler(Collections.emptyMap()), threadPool,
+            new AnonymousUser(Settings.EMPTY), engine, Collections.emptySet(), licenseState, new IndexNameExpressionResolver());
         Authentication authentication;
         try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
             authentication = createAuthentication(new User("test user", "a_all"));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index c766ab211c173..833d787275ba9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -25,13 +25,13 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 
 import java.util.Collections;
 
-import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
 import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 import static org.elasticsearch.xpack.security.authz.AuthorizationService.AUTHORIZATION_INFO_KEY;
 import static org.elasticsearch.xpack.security.authz.AuthorizationService.ORIGINATING_ACTION_KEY;
@@ -41,6 +41,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
@@ -94,9 +95,11 @@ public void testValidateSearchContext() throws Exception {
         testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isAuthAllowed()).thenReturn(true);
+        when(licenseState.isAuditingAllowed()).thenReturn(true);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
-        AuditTrailService auditTrailService = mock(AuditTrailService.class);
+        AuditTrail auditTrail = mock(AuditTrail.class);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
         try (StoredContext ignore = threadContext.newStoredContext(false)) {
@@ -104,7 +107,7 @@ public void testValidateSearchContext() throws Exception {
             authentication.writeToContext(threadContext);
             listener.validateSearchContext(testSearchContext, Empty.INSTANCE);
             verify(licenseState).isAuthAllowed();
-            verifyZeroInteractions(auditTrailService);
+            verifyZeroInteractions(auditTrail);
         }
 
         try (StoredContext ignore = threadContext.newStoredContext(false)) {
@@ -114,7 +117,7 @@ public void testValidateSearchContext() throws Exception {
             authentication.writeToContext(threadContext);
             listener.validateSearchContext(testSearchContext, Empty.INSTANCE);
             verify(licenseState, times(2)).isAuthAllowed();
-            verifyZeroInteractions(auditTrailService);
+            verifyZeroInteractions(auditTrail);
         }
 
         try (StoredContext ignore = threadContext.newStoredContext(false)) {
@@ -131,7 +134,7 @@ public void testValidateSearchContext() throws Exception {
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
             assertEquals(testSearchContext.id(), expected.contextId());
             verify(licenseState, times(3)).isAuthAllowed();
-            verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
+            verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                 authzInfoRoles(authentication.getUser().roles()));
         }
 
@@ -148,7 +151,7 @@ public void testValidateSearchContext() throws Exception {
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             listener.validateSearchContext(testSearchContext, request);
             verify(licenseState, times(4)).isAuthAllowed();
-            verifyNoMoreInteractions(auditTrailService);
+            verifyNoMoreInteractions(auditTrail);
         }
 
         // the user that authenticated for the run as request
@@ -167,7 +170,7 @@ public void testValidateSearchContext() throws Exception {
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
             assertEquals(testSearchContext.id(), expected.contextId());
             verify(licenseState, times(5)).isAuthAllowed();
-            verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
+            verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                 authzInfoRoles(authentication.getUser().roles()));
         }
     }
@@ -179,10 +182,14 @@ public void testEnsuredAuthenticatedUserIsSame() {
         SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
         final String action = randomAlphaOfLength(4);
         TransportRequest request = Empty.INSTANCE;
-        AuditTrailService auditTrail = mock(AuditTrailService.class);
+        XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isAuthAllowed()).thenReturn(true);
+        when(licenseState.isAuditingAllowed()).thenReturn(true);
+        AuditTrail auditTrail = mock(AuditTrail.class);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
 
         final String auditId = randomAlphaOfLengthBetween(8, 20);
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, contextId, action, request, auditId,
+        ensureAuthenticatedUserIsSame(original, current, auditTrailService, contextId, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -190,7 +197,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         User user = new User(new User("test", "role"), new User("authenticated", "runas"));
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, contextId, action, request, auditId,
+        ensureAuthenticatedUserIsSame(original, current, auditTrailService, contextId, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -198,7 +205,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
         Authentication runAs = current;
-        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, contextId, action, request, auditId,
+        ensureAuthenticatedUserIsSame(runAs, current, auditTrailService, contextId, action, request, auditId,
             () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
@@ -206,7 +213,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication differentRealmType =
                 new Authentication(new User("test", "role"), new RealmRef("realm", randomAlphaOfLength(5), "node"), null);
         SearchContextMissingException e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrailService, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(differentRealmType), eq(action), eq(request),
@@ -216,7 +223,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication differentUser =
                 new Authentication(new User("test2", "role"), new RealmRef("realm", "realm", "node"), null);
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrailService, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(differentUser), eq(action), eq(request),
@@ -226,7 +233,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")),
                 new RealmRef("realm", "file", "node1"), new RealmRef("realm", "file", "node1"));
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrailService, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(diffRunAs), eq(action), eq(request), authzInfoRoles(original.getUser().roles()));
@@ -235,7 +242,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication runAsDiffType = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node"));
         e = expectThrows(SearchContextMissingException.class,
-                () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrail, contextId, action, request, auditId,
+                () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrailService, contextId, action, request, auditId,
                     () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(contextId, e.contextId());
         verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java
index e3777fc854599..714f1651a440c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.test.junit.annotations.Network;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
+import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
@@ -49,7 +50,8 @@
 public class IPFilterTests extends ESTestCase {
     private IPFilter ipFilter;
     private XPackLicenseState licenseState;
-    private AuditTrailService auditTrail;
+    private AuditTrail auditTrail;
+    private AuditTrailService auditTrailService;
     private Transport transport;
     private HttpServerTransport httpTransport;
     private ClusterSettings clusterSettings;
@@ -58,7 +60,9 @@ public class IPFilterTests extends ESTestCase {
     public void init() {
         licenseState = mock(XPackLicenseState.class);
         when(licenseState.isIpFilteringAllowed()).thenReturn(true);
-        auditTrail = mock(AuditTrailService.class);
+        when(licenseState.isAuditingAllowed()).thenReturn(true);
+        auditTrail = mock(AuditTrail.class);
+        auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
         clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(
                 IPFilter.HTTP_FILTER_ALLOW_SETTING,
                 IPFilter.HTTP_FILTER_DENY_SETTING,
@@ -90,7 +94,7 @@ public void testThatIpV4AddressesCanBeProcessed() throws Exception {
                 .put("xpack.security.transport.filter.allow", "127.0.0.1")
                 .put("xpack.security.transport.filter.deny", "10.0.0.0/8")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         assertAddressIsAllowed("127.0.0.1");
         assertAddressIsDenied("10.2.3.4");
@@ -103,7 +107,7 @@ public void testThatIpV6AddressesCanBeProcessed() throws Exception {
                 .put("xpack.security.transport.filter.allow", "2001:0db8:1234::/48")
                 .putList("xpack.security.transport.filter.deny", "1234:db8:85a3:0:0:8a2e:370:7334", "4321:db8:1234::/48")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
 
         assertAddressIsAllowed("2001:0db8:1234:0000:0000:8a2e:0370:7334");
@@ -118,7 +122,7 @@ public void testThatHostnamesCanBeProcessed() throws Exception {
                 .put("xpack.security.transport.filter.allow", "127.0.0.1")
                 .put("xpack.security.transport.filter.deny", "*.google.com")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
 
         assertAddressIsAllowed("127.0.0.1");
@@ -129,7 +133,7 @@ public void testThatAnAllowAllAuthenticatorWorks() throws Exception {
         Settings settings = Settings.builder()
                 .put("xpack.security.transport.filter.allow", "_all")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         assertAddressIsAllowed("127.0.0.1");
         assertAddressIsAllowed("173.194.70.100");
@@ -142,7 +146,7 @@ public void testThatProfilesAreSupported() throws Exception {
                 .put("transport.profiles.client.xpack.security.filter.allow", "192.168.0.1")
                 .put("transport.profiles.client.xpack.security.filter.deny", "_all")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         assertAddressIsAllowed("127.0.0.1");
         // when "localhost" is used, ES considers all local addresses see PatternRule#isLocalhost()
@@ -158,7 +162,7 @@ public void testThatProfilesAreUpdateable() throws Exception {
                 .put("transport.profiles.client.xpack.security.filter.allow", "192.168.0.1")
                 .put("transport.profiles.client.xpack.security.filter.deny", "_all")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         Settings newSettings = Settings.builder().putList("transport.profiles.client.xpack.security.filter.allow", "192.168.0.1",
                 "192.168.0.2")
@@ -178,7 +182,7 @@ public void testThatAllowWinsOverDeny() throws Exception {
                 .put("xpack.security.transport.filter.allow", "10.0.0.1")
                 .put("xpack.security.transport.filter.deny", "10.0.0.0/8")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         assertAddressIsAllowed("10.0.0.1");
         assertAddressIsDenied("10.0.0.2");
@@ -186,7 +190,7 @@ public void testThatAllowWinsOverDeny() throws Exception {
 
     public void testDefaultAllow() throws Exception {
         Settings settings = Settings.builder().build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         assertAddressIsAllowed("10.0.0.1");
         assertAddressIsAllowed("10.0.0.2");
@@ -199,7 +203,7 @@ public void testThatHttpWorks() throws Exception {
                 .put("xpack.security.http.filter.allow", "10.0.0.0/8")
                 .put("xpack.security.http.filter.deny", "192.168.0.1")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundHttpTransportAddress(httpTransport.boundAddress());
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         assertAddressIsAllowedForProfile(IPFilter.HTTP_PROFILE_NAME, "10.2.3.4");
@@ -211,7 +215,7 @@ public void testThatHttpFallsbackToDefault() throws Exception {
                 .put("xpack.security.transport.filter.allow", "127.0.0.1")
                 .put("xpack.security.transport.filter.deny", "10.0.0.0/8")
                 .build();
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundHttpTransportAddress(httpTransport.boundAddress());
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
 
@@ -232,7 +236,7 @@ public void testThatBoundAddressIsNeverRejected() throws Exception {
         } else {
             settings = Settings.builder().put("xpack.security.transport.filter.deny", "_all").build();
         }
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         ipFilter.setBoundHttpTransportAddress(httpTransport.boundAddress());
 
@@ -247,7 +251,7 @@ public void testThatAllAddressesAreAllowedWhenLicenseDisablesSecurity() {
                 .put("xpack.security.transport.filter.deny", "_all")
                 .build();
         when(licenseState.isIpFilteringAllowed()).thenReturn(false);
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
 
         // don't use the assert helper because we don't want the audit trail to be invoked here
@@ -258,7 +262,7 @@ public void testThatAllAddressesAreAllowedWhenLicenseDisablesSecurity() {
 
         // for sanity enable license and check that it is denied
         when(licenseState.isIpFilteringAllowed()).thenReturn(true);
-        ipFilter = new IPFilter(settings, auditTrail, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
 
         assertAddressIsDeniedForProfile("default", "8.8.8.8");

From 36d8d848170b6e652778e3ddacb8dd8f6b7f74cb Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 18 Mar 2020 16:16:58 -0400
Subject: [PATCH 123/285] Upgrade the bundled JDK to JDK 14 (#53748)

This commit upgrades the bundled JDK to JDK 14.
---
 buildSrc/version.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 9ef5da5062255..ee2ce1b8d67b3 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -2,7 +2,7 @@ elasticsearch     = 8.0.0
 lucene            = 8.5.0-snapshot-7f057455901
 
 bundled_jdk_vendor = adoptopenjdk
-bundled_jdk = 13.0.2+8
+bundled_jdk = 14+36
 
 # optional dependencies
 spatial4j         = 0.7

From 73d2addaa255948ec08fc0bd5a0fef173731a2f7 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Wed, 18 Mar 2020 13:19:13 -0700
Subject: [PATCH 124/285] supported field type tests for max agg (#53701)

Adds test hooks for testing supported ValuesSource types for the max
aggregation
---
 .../aggregations/metrics/MaxAggregatorTests.java   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
index 598d19795db20..e37731388e43a 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
@@ -71,7 +71,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
@@ -89,6 +91,7 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -158,6 +161,17 @@ protected ScriptService getMockScriptService() {
         return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
     }
 
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return singletonList(CoreValuesSourceType.NUMERIC);
+    }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new MaxAggregationBuilder("_name")
+            .field(fieldName);
+    }
+
     public void testNoDocs() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
             // Intentionally not writing any docs

From 6a04394aaf994fadcf68473f4ef8b0261bb53d59 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Wed, 18 Mar 2020 23:00:12 +0200
Subject: [PATCH 125/285] [ML] Register ML named writeables from ML plugin
 (#53757)

... instead of XPackClientPlugin.
---
 .../xpack/core/XPackClientPlugin.java         | 259 ++++++------------
 .../xpack/ml/MachineLearning.java             |  42 +++
 2 files changed, 131 insertions(+), 170 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 26abe1af66fa0..623afe212e369 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -34,9 +34,6 @@
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
-import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
-import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
@@ -67,7 +64,6 @@
 import org.elasticsearch.xpack.core.ilm.action.RemoveIndexLifecyclePolicyAction;
 import org.elasticsearch.xpack.core.ilm.action.RetryAction;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
-import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
@@ -134,30 +130,6 @@
 import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
-import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
-import org.elasticsearch.xpack.core.ml.inference.preprocessing.CustomWordEmbedding;
-import org.elasticsearch.xpack.core.ml.inference.preprocessing.FrequencyEncoding;
-import org.elasticsearch.xpack.core.ml.inference.preprocessing.OneHotEncoding;
-import org.elasticsearch.xpack.core.ml.inference.preprocessing.PreProcessor;
-import org.elasticsearch.xpack.core.ml.inference.preprocessing.TargetMeanEncoding;
-import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults;
-import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
-import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ensemble.Ensemble;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ensemble.LogisticRegression;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ensemble.OutputAggregator;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ensemble.WeightedMode;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ensemble.WeightedSum;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.langident.LangIdentNeuralNetwork;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
 import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
@@ -171,6 +143,9 @@
 import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus;
+import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
 import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
@@ -243,9 +218,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
-import java.util.stream.Stream;
-
-import static java.util.stream.Collectors.toList;
 
 // TODO: merge this into XPackPlugin
 public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPlugin {
@@ -432,146 +404,93 @@ public List<ActionType<? extends ActionResponse>> getClientActions() {
 
     @Override
     public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
-        return Stream.concat(
-            Arrays.asList(
-                // graph
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
-                // logstash
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new),
-                // ML - Custom metadata
-                new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),
-                // ML - Persistent action requests
-                new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATAFEED_TASK_NAME,
-                        StartDatafeedAction.DatafeedParams::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME,
-                        OpenJobAction.JobParams::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
-                    StartDataFrameAnalyticsAction.TaskParams::new),
-                // ML - Task states
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DatafeedState.NAME, DatafeedState::fromStream),
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameAnalyticsTaskState.NAME,
-                    DataFrameAnalyticsTaskState::new),
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MACHINE_LEARNING,
-                        MachineLearningFeatureSetUsage::new),
-                // ML - Data frame analytics
-                new NamedWriteableRegistry.Entry(DataFrameAnalysis.class, OutlierDetection.NAME.getPreferredName(), OutlierDetection::new),
-                new NamedWriteableRegistry.Entry(DataFrameAnalysis.class, Regression.NAME.getPreferredName(), Regression::new),
-                new NamedWriteableRegistry.Entry(DataFrameAnalysis.class, Classification.NAME.getPreferredName(), Classification::new),
-                // ML - Inference preprocessing
-                new NamedWriteableRegistry.Entry(PreProcessor.class, FrequencyEncoding.NAME.getPreferredName(), FrequencyEncoding::new),
-                new NamedWriteableRegistry.Entry(PreProcessor.class, OneHotEncoding.NAME.getPreferredName(), OneHotEncoding::new),
-                new NamedWriteableRegistry.Entry(PreProcessor.class, TargetMeanEncoding.NAME.getPreferredName(), TargetMeanEncoding::new),
-                new NamedWriteableRegistry.Entry(PreProcessor.class, CustomWordEmbedding.NAME.getPreferredName(), CustomWordEmbedding::new),
-                // ML - Inference models
-                new NamedWriteableRegistry.Entry(TrainedModel.class, Tree.NAME.getPreferredName(), Tree::new),
-                new NamedWriteableRegistry.Entry(TrainedModel.class, Ensemble.NAME.getPreferredName(), Ensemble::new),
-                new NamedWriteableRegistry.Entry(TrainedModel.class,
-                    LangIdentNeuralNetwork.NAME.getPreferredName(),
-                    LangIdentNeuralNetwork::new),
-                // ML - Inference aggregators
-                new NamedWriteableRegistry.Entry(OutputAggregator.class, WeightedSum.NAME.getPreferredName(), WeightedSum::new),
-                new NamedWriteableRegistry.Entry(OutputAggregator.class, WeightedMode.NAME.getPreferredName(), WeightedMode::new),
-                new NamedWriteableRegistry.Entry(OutputAggregator.class,
-                    LogisticRegression.NAME.getPreferredName(),
-                    LogisticRegression::new),
-                // ML - Inference Results
-                new NamedWriteableRegistry.Entry(InferenceResults.class,
-                    ClassificationInferenceResults.NAME,
-                    ClassificationInferenceResults::new),
-                new NamedWriteableRegistry.Entry(InferenceResults.class,
-                    RegressionInferenceResults.NAME,
-                    RegressionInferenceResults::new),
-                // ML - Inference Configuration
-                new NamedWriteableRegistry.Entry(InferenceConfig.class, ClassificationConfig.NAME.getPreferredName(),
-                        ClassificationConfig::new),
-                new NamedWriteableRegistry.Entry(InferenceConfig.class, RegressionConfig.NAME.getPreferredName(),
-                        RegressionConfig::new),
+        return Arrays.asList(
+            // graph
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
+            // logstash
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new),
 
-                // monitoring
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MONITORING, MonitoringFeatureSetUsage::new),
-                // security
-                new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetaData.TYPE, TokenMetaData::readDiffFrom),
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SECURITY, SecurityFeatureSetUsage::new),
-                // security : conditional privileges
-                new NamedWriteableRegistry.Entry(ConfigurableClusterPrivilege.class,
-                    ConfigurableClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
-                    ConfigurableClusterPrivileges.ManageApplicationPrivileges::createFrom),
-                // security : role-mappings
-                new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AllExpression.NAME, AllExpression::new),
-                new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
-                new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),
-                new NamedWriteableRegistry.Entry(RoleMapperExpression.class, ExceptExpression.NAME, ExceptExpression::new),
-                // eql
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.EQL, EqlFeatureSetUsage::new),
-                // sql
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SQL, SqlFeatureSetUsage::new),
-                // watcher
-                new NamedWriteableRegistry.Entry(MetaData.Custom.class, WatcherMetaData.TYPE, WatcherMetaData::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, WatcherMetaData.TYPE, WatcherMetaData::readDiffFrom),
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.WATCHER, WatcherFeatureSetUsage::new),
-                // licensing
-                new NamedWriteableRegistry.Entry(MetaData.Custom.class, LicensesMetaData.TYPE, LicensesMetaData::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, LicensesMetaData.TYPE, LicensesMetaData::readDiffFrom),
-                // rollup
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ROLLUP, RollupFeatureSetUsage::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskParams.class, RollupJob.NAME, RollupJob::new),
-                new NamedWriteableRegistry.Entry(Task.Status.class, RollupJobStatus.NAME, RollupJobStatus::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, RollupJobStatus.NAME, RollupJobStatus::new),
-                // ccr
-                new NamedWriteableRegistry.Entry(AutoFollowMetadata.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
-                new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
-                    in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRInfoTransportAction.Usage::new),
-                // ILM
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
-                    IndexLifecycleFeatureSetUsage::new),
-                // SLM
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SNAPSHOT_LIFECYCLE,
-                    SLMFeatureSetUsage::new),
-                // ILM - Custom Metadata
-                new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE,
-                    IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new),
-                new NamedWriteableRegistry.Entry(MetaData.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new),
-                new NamedWriteableRegistry.Entry(NamedDiff.class, SnapshotLifecycleMetadata.TYPE,
-                    SnapshotLifecycleMetadata.SnapshotLifecycleMetadataDiff::new),
-                // ILM - LifecycleTypes
-                new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
-                    (in) -> TimeseriesLifecycleType.INSTANCE),
-                // ILM - Lifecycle Actions
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, FreezeAction.NAME, FreezeAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new),
-                new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new),
-                // Transforms
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.TRANSFORM, TransformFeatureSetUsage::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new),
-                new NamedWriteableRegistry.Entry(Task.Status.class, TransformField.TASK_NAME, TransformState::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, TransformField.TASK_NAME, TransformState::new),
-                new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new),
-                // Vectors
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VECTORS, VectorsFeatureSetUsage::new),
-                // Voting Only Node
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VOTING_ONLY, VotingOnlyNodeFeatureSetUsage::new),
-                // Frozen indices
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FROZEN_INDICES, FrozenIndicesFeatureSetUsage::new),
-                // Spatial
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new),
-                // data science
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new)
-            ).stream(),
-            MlEvaluationNamedXContentProvider.getNamedWriteables().stream()
-        ).collect(toList());
+            // monitoring
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MONITORING, MonitoringFeatureSetUsage::new),
+            // security
+            new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetaData.TYPE, TokenMetaData::readDiffFrom),
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SECURITY, SecurityFeatureSetUsage::new),
+            // security : conditional privileges
+            new NamedWriteableRegistry.Entry(ConfigurableClusterPrivilege.class,
+                ConfigurableClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
+                ConfigurableClusterPrivileges.ManageApplicationPrivileges::createFrom),
+            // security : role-mappings
+            new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AllExpression.NAME, AllExpression::new),
+            new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
+            new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),
+            new NamedWriteableRegistry.Entry(RoleMapperExpression.class, ExceptExpression.NAME, ExceptExpression::new),
+            // eql
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.EQL, EqlFeatureSetUsage::new),
+            // sql
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SQL, SqlFeatureSetUsage::new),
+            // watcher
+            new NamedWriteableRegistry.Entry(MetaData.Custom.class, WatcherMetaData.TYPE, WatcherMetaData::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, WatcherMetaData.TYPE, WatcherMetaData::readDiffFrom),
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.WATCHER, WatcherFeatureSetUsage::new),
+            // licensing
+            new NamedWriteableRegistry.Entry(MetaData.Custom.class, LicensesMetaData.TYPE, LicensesMetaData::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, LicensesMetaData.TYPE, LicensesMetaData::readDiffFrom),
+            // rollup
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ROLLUP, RollupFeatureSetUsage::new),
+            new NamedWriteableRegistry.Entry(PersistentTaskParams.class, RollupJob.NAME, RollupJob::new),
+            new NamedWriteableRegistry.Entry(Task.Status.class, RollupJobStatus.NAME, RollupJobStatus::new),
+            new NamedWriteableRegistry.Entry(PersistentTaskState.class, RollupJobStatus.NAME, RollupJobStatus::new),
+            // ccr
+            new NamedWriteableRegistry.Entry(AutoFollowMetadata.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
+            new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
+                in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRInfoTransportAction.Usage::new),
+            // ILM
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
+                IndexLifecycleFeatureSetUsage::new),
+            // SLM
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SNAPSHOT_LIFECYCLE,
+                SLMFeatureSetUsage::new),
+            // ILM - Custom Metadata
+            new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE,
+                IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new),
+            new NamedWriteableRegistry.Entry(MetaData.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, SnapshotLifecycleMetadata.TYPE,
+                SnapshotLifecycleMetadata.SnapshotLifecycleMetadataDiff::new),
+            // ILM - LifecycleTypes
+            new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
+                (in) -> TimeseriesLifecycleType.INSTANCE),
+            // ILM - Lifecycle Actions
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, FreezeAction.NAME, FreezeAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new),
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new),
+            // Transforms
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.TRANSFORM, TransformFeatureSetUsage::new),
+            new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new),
+            new NamedWriteableRegistry.Entry(Task.Status.class, TransformField.TASK_NAME, TransformState::new),
+            new NamedWriteableRegistry.Entry(PersistentTaskState.class, TransformField.TASK_NAME, TransformState::new),
+            new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new),
+            // Vectors
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VECTORS, VectorsFeatureSetUsage::new),
+            // Voting Only Node
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VOTING_ONLY, VotingOnlyNodeFeatureSetUsage::new),
+            // Frozen indices
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FROZEN_INDICES, FrozenIndicesFeatureSetUsage::new),
+            // Spatial
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new),
+            // data science
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new)
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index d235982be4344..792e0ae25218c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -15,8 +15,10 @@
 import org.elasticsearch.client.OriginSettingClient;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -42,6 +44,8 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.os.OsStats;
+import org.elasticsearch.persistent.PersistentTaskParams;
+import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
@@ -56,12 +60,17 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
+import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction;
@@ -126,10 +135,13 @@
 import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
+import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
@@ -942,6 +954,36 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
         return namedXContent;
     }
 
+    @Override
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+
+        // Custom metadata
+        namedWriteables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new));
+
+        // Persistent tasks params
+        namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATAFEED_TASK_NAME,
+            StartDatafeedAction.DatafeedParams::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME,
+            OpenJobAction.JobParams::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
+            StartDataFrameAnalyticsAction.TaskParams::new));
+
+        // Persistent task states
+        namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DatafeedState.NAME, DatafeedState::fromStream));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameAnalyticsTaskState.NAME,
+            DataFrameAnalyticsTaskState::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MACHINE_LEARNING,
+            MachineLearningFeatureSetUsage::new));
+
+        namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
+        namedWriteables.addAll(MlEvaluationNamedXContentProvider.getNamedWriteables());
+        namedWriteables.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables());
+        return namedWriteables;
+    }
+
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         return List.of(

From 3dae2e97d81bf75052455bb092858271b075d971 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Wed, 18 Mar 2020 18:23:01 -0400
Subject: [PATCH 126/285] [Transform] renamed _cat/transform to _cat/transforms
 (#53743)

renaming _cat/transform to  _cat/transforms for uniformity with the other _cat apis.
---
 docs/reference/cat.asciidoc                   |  2 +-
 docs/reference/cat/transform.asciidoc         | 28 -------------------
 docs/reference/cat/transforms.asciidoc        | 28 +++++++++++++++++++
 docs/reference/redirects.asciidoc             |  5 ++++
 .../api/transform.cat_transform.json          |  6 ++--
 .../rest/action/RestCatTransformAction.java   |  8 +++---
 6 files changed, 41 insertions(+), 36 deletions(-)
 delete mode 100644 docs/reference/cat/transform.asciidoc
 create mode 100644 docs/reference/cat/transforms.asciidoc

diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc
index 62d5165984aa5..2b303e84ab955 100644
--- a/docs/reference/cat.asciidoc
+++ b/docs/reference/cat.asciidoc
@@ -269,4 +269,4 @@ include::cat/thread_pool.asciidoc[]
 
 include::cat/trainedmodel.asciidoc[]
 
-include::cat/transform.asciidoc[]
+include::cat/transforms.asciidoc[]
diff --git a/docs/reference/cat/transform.asciidoc b/docs/reference/cat/transform.asciidoc
deleted file mode 100644
index 835106c08b496..0000000000000
--- a/docs/reference/cat/transform.asciidoc
+++ /dev/null
@@ -1,28 +0,0 @@
-[[cat-transform]]
-=== cat transform API
-++++
-<titleabbrev>cat transform</titleabbrev>
-++++
-
-Returns configuration and usage information about {transforms}.
-
-
-[[cat-transform-api-request]]
-==== {api-request-title}
-
-`GET /_cat/transform`
-
-
-//[[cat-transform-api-desc]]
-//==== {api-description-title}
-
-
-//[[cat-transform-api-query-params]]
-//==== {api-query-parms-title}
-
-
-//[[cat-transform-api-response-codes]]
-//==== {api-response-codes-title}
-
-//[[cat-transform-api-examples]]
-//==== {api-examples-title}
diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc
new file mode 100644
index 0000000000000..78013e394ef24
--- /dev/null
+++ b/docs/reference/cat/transforms.asciidoc
@@ -0,0 +1,28 @@
+[[cat-transforms]]
+=== cat {transforms} API
+++++
+<titleabbrev>cat transforms</titleabbrev>
+++++
+
+Returns configuration and usage information about {transforms}.
+
+
+[[cat-transforms-api-request]]
+==== {api-request-title}
+
+`GET /_cat/transforms`
+
+
+//[[cat-transforms-api-desc]]
+//==== {api-description-title}
+
+
+//[[cat-transforms-api-query-params]]
+//==== {api-query-parms-title}
+
+
+//[[cat-transforms-api-response-codes]]
+//==== {api-response-codes-title}
+
+//[[cat-transforms-api-examples]]
+//==== {api-examples-title}
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 065450560d0a2..655e1156f1938 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -385,3 +385,8 @@ coming::[7.x]
 === Component template APIs
 
 coming::[7.x]
+
+[role="exclude",id="cat-transform"]
+=== cat transform API
+
+See <<cat-transforms>>.
\ No newline at end of file
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
index 26a7be322aa3c..9cdbc620eb93f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
@@ -1,19 +1,19 @@
 {
   "transform.cat_transform":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-transform.html"
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-transforms.html"
     },
     "stability":"stable",
     "url":{
       "paths":[
         {
-          "path":"/_cat/transform",
+          "path":"/_cat/transforms",
           "methods":[
             "GET"
           ]
         },
         {
-          "path":"/_cat/transform/{transform_id}",
+          "path":"/_cat/transforms/{transform_id}",
           "methods":[
             "GET"
           ],
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
index 7a5f669bbcb4d..a848769991ee6 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
@@ -41,8 +41,8 @@ public class RestCatTransformAction extends AbstractCatAction {
     @Override
     public List<Route> routes() {
         return List.of(
-            new Route(GET, "_cat/transform"),
-            new Route(GET, "_cat/transform/{" + TransformField.TRANSFORM_ID + "}"));
+            new Route(GET, "_cat/transforms"),
+            new Route(GET, "_cat/transforms/{" + TransformField.TRANSFORM_ID + "}"));
     }
 
     @Override
@@ -85,8 +85,8 @@ public RestResponse buildResponse(GetTransformStatsAction.Response statsResponse
 
     @Override
     protected void documentation(StringBuilder sb) {
-        sb.append("/_cat/transform\n");
-        sb.append("/_cat/transform/{" + TransformField.TRANSFORM_ID + "}\n");
+        sb.append("/_cat/transforms\n");
+        sb.append("/_cat/transforms/{" + TransformField.TRANSFORM_ID + "}\n");
     }
 
     @Override

From 35ad63036e37c8177b491b7b24178852d47587a1 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Wed, 18 Mar 2020 15:23:41 -0700
Subject: [PATCH 127/285] [DOCS] Add transform nodes (#53698)

---
 docs/reference/cat/nodes.asciidoc             |  2 +-
 docs/reference/modules/node.asciidoc          | 90 +++++++++++++++----
 .../transform/ecommerce-tutorial.asciidoc     | 20 ++---
 docs/reference/transform/index.asciidoc       |  2 +
 docs/reference/transform/limitations.asciidoc | 11 ---
 docs/reference/transform/setup.asciidoc       | 40 +++++++++
 6 files changed, 122 insertions(+), 43 deletions(-)
 create mode 100644 docs/reference/transform/setup.asciidoc

diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index d6d67a35ef342..94b004a6f9f45 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -44,7 +44,7 @@ Valid columns are:
 `node.role`, `r`, `role`, `nodeRole`::
 (Default) Roles of the node. Returned values include `d` (data node), `i`
 (ingest node), `m` (master-eligible node), `l` (machine learning node), `v`
-(voting-only node), and `-` (coordinating node only).
+(voting-only node), `t` ({transform} node), and `-` (coordinating node only).
 +
 For example, `dim` indicates a master-eligible data and ingest node. See
 <<modules-node>>.
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 987ab35204e79..47b64ac4bd560 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -50,6 +50,13 @@ there must be at least one {ml} node in your cluster. For more information about
 IMPORTANT: If you use the {oss-dist}, do not set `node.ml`. Otherwise, the node
 fails to start.
 
+<<transform-node,{transform-cap} node>>::
+
+A node that has `xpack.transform.enabled` and `node.transform` set to `true`. If
+you want to use {transforms}, there must be at least one {transform} node in
+your cluster. For more information, see <<transform-settings>> and
+<<transforms>>.
+
 [NOTE]
 [[coordinating-node]]
 .Coordinating node
@@ -96,7 +103,7 @@ restarts.
 ==== Dedicated master-eligible node
 
 It is important for the health of the cluster that the elected master node has 
-the resources it needs to fulfil its responsibilities. If the elected master 
+the resources it needs to fulfill its responsibilities. If the elected master 
 node is overloaded with other tasks then the cluster may not operate well. In 
 particular, indexing and searching your data can be very resource-intensive, so 
 in large or high-throughput clusters it is a good idea to avoid using the 
@@ -118,7 +125,9 @@ node.data: false <3>
 node.ingest: false <4>
 node.ml: false <5>
 xpack.ml.enabled: true <6>
-cluster.remote.connect: false <7>
+node.transform: false <7>
+xpack.transform.enabled: true <8>
+cluster.remote.connect: false <9>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> The `node.voting_only` role is disabled by default.
@@ -126,7 +135,9 @@ cluster.remote.connect: false <7>
 <4> Disable the `node.ingest` role (enabled by default).
 <5> Disable the `node.ml` role (enabled by default).
 <6> The `xpack.ml.enabled` setting is enabled by default.
-<7> Disable remote cluster connections (enabled by default).
+<7> Disable the `node.transform` role.
+<8> The `xpack.transform.enabled` setting is enabled by default.
+<9> Disable remote cluster connections (enabled by default).
 
 To create a dedicated master-eligible node in the {oss-dist}, set:
 
@@ -197,7 +208,9 @@ node.data: false <3>
 node.ingest: false <4>
 node.ml: false <5>
 xpack.ml.enabled: true <6>
-cluster.remote.connect: false <7>
+node.transform: false <7>
+xpack.transform.enabled: true <8>
+cluster.remote.connect: false <9>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> Enable the `node.voting_only` role (disabled by default).
@@ -205,11 +218,13 @@ cluster.remote.connect: false <7>
 <4> Disable the `node.ingest` role (enabled by default).
 <5> Disable the `node.ml` role (enabled by default).
 <6> The `xpack.ml.enabled` setting is enabled by default.
-<7> Disable remote cluster connections (enabled by default).
+<7> Disable the `node.transform` role.
+<8> The `xpack.transform.enabled` setting is enabled by default.
+<9> Disable remote cluster connections (enabled by default).
 
 [float]
 [[data-node]]
-=== Data Node
+=== Data node
 
 Data nodes hold the shards that contain the documents you have indexed. Data
 nodes handle data related operations like CRUD, search, and aggregations.
@@ -227,14 +242,16 @@ node.voting_only: false <2>
 node.data: true <3>
 node.ingest: false <4>
 node.ml: false <5>
-cluster.remote.connect: false <6>
+node.transform: false <6>
+cluster.remote.connect: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
 <3> The `node.data` role is enabled by default.
 <4> Disable the `node.ingest` role (enabled by default).
 <5> Disable the `node.ml` role (enabled by default).
-<6> Disable remote cluster connections (enabled by default).
+<6> Disable the `node.transform` role.
+<7> Disable remote cluster connections (enabled by default).
 
 To create a dedicated data node in the {oss-dist}, set:
 [source,yaml]
@@ -251,7 +268,7 @@ cluster.remote.connect: false <4>
 
 [float]
 [[node-ingest-node]]
-=== Ingest Node
+=== Ingest node
 
 Ingest nodes can execute pre-processing pipelines, composed of one or more
 ingest processors. Depending on the type of operations performed by the ingest
@@ -267,14 +284,16 @@ node.voting_only: false <2>
 node.data: false <3>
 node.ingest: true <4>
 node.ml: false <5>
-cluster.remote.connect: false <6>
+node.transform: false <6>
+cluster.remote.connect: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
 <3> Disable the `node.data` role (enabled by default).
 <4> The `node.ingest` role is enabled by default.
 <5> Disable the `node.ml` role (enabled by default).
-<6> Disable remote cluster connections (enabled by default).
+<6> Disable the `node.transform` role.
+<7> Disable remote cluster connections (enabled by default).
 
 To create a dedicated ingest node in the {oss-dist}, set:
 
@@ -320,14 +339,16 @@ node.voting_only: false <2>
 node.data: false <3>
 node.ingest: false <4>
 node.ml: false <5>
-cluster.remote.connect: false <6>
+node.transform: false <6>
+cluster.remote.connect: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
 <3> Disable the `node.data` role (enabled by default).
 <4> Disable the `node.ingest` role (enabled by default).
 <5> Disable the `node.ml` role (enabled by default).
-<6> Disable remote cluster connections (enabled by default).
+<6> Disable the `node.transform` role.
+<7> Disable remote cluster connections (enabled by default).
 
 To create a dedicated coordinating node in the {oss-dist}, set:
 
@@ -348,7 +369,7 @@ cluster.remote.connect: false <4>
 === [xpack]#Machine learning node#
 
 The {ml-features} provide {ml} nodes, which run jobs and handle {ml} API
-requests. If `xpack.ml.enabled` is set to true and `node.ml` is set to `false`,
+requests. If `xpack.ml.enabled` is set to `true` and `node.ml` is set to `false`,
 the node can service API requests but it cannot run jobs.
 
 If you want to use {ml-features} in your cluster, you must enable {ml}
@@ -367,7 +388,9 @@ node.data: false <3>
 node.ingest: false <4>
 node.ml: true <5>
 xpack.ml.enabled: true <6>
-cluster.remote.connect: false <7>
+node.transform: false <7>
+xpack.transform.enabled: true <8>
+cluster.remote.connect: false <9>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -375,7 +398,42 @@ cluster.remote.connect: false <7>
 <4> Disable the `node.ingest` role (enabled by default).
 <5> The `node.ml` role is enabled by default.
 <6> The `xpack.ml.enabled` setting is enabled by default.
-<7> Disable remote cluster connections (enabled by default).
+<7> Disable the `node.transform` role.
+<8> The `xpack.transform.enabled` setting is enabled by default.
+<9> Disable remote cluster connections (enabled by default).
+
+[discrete]
+[[transform-node]]
+=== [xpack]#{transform-cap} node#
+
+{transform-cap} nodes run {transforms} and handle {transform} API requests.
+
+If you want to use {transforms} in your cluster, you must have
+`xpack.transform.enabled` set to `true` on all master-eligible nodes and all
+data nodes. You must also have `node.transform` set to `true` on at least one
+node. This is the default behavior. If you have the {oss-dist}, do not use these
+settings. For more information, see <<transform-settings>>.
+
+To create a dedicated {transform} node in the {default-dist}, set:
+
+[source,yaml]
+-------------------
+node.master: false <1>
+node.voting_only: false <2>
+node.data: false <3>
+node.ingest: false <4>
+node.ml: false <5>
+node.transform: true <6>
+xpack.transform.enabled: true <7>
+cluster.remote.connect: false <8>
+-------------------
+<1> Disable the `node.master` role.
+<2> Disable the `node.voting_only`.
+<3> Disable the `node.data` role.
+<4> Disable the `node.ingest` role.
+<5> Enable the `node.ml` role.
+<6> Enable the `xpack.ml.enabled` setting.
+<7> Disable remote cluster connections.
 
 [float]
 [[change-node-role]]
diff --git a/docs/reference/transform/ecommerce-tutorial.asciidoc b/docs/reference/transform/ecommerce-tutorial.asciidoc
index 99391438aae0c..0e22002a5e7a5 100644
--- a/docs/reference/transform/ecommerce-tutorial.asciidoc
+++ b/docs/reference/transform/ecommerce-tutorial.asciidoc
@@ -8,21 +8,11 @@ from an {es} index, transform it, and store it in another index. Let's use the
 {kibana-ref}/add-sample-data.html[{kib} sample data] to demonstrate how you can
 pivot and summarize your data with {transforms}.
 
-
-. If the {es} {security-features} are enabled, obtain a user ID with sufficient
-privileges to complete these steps. 
-+
---
-You need `manage_transform` cluster privileges to preview and create 
-{transforms}. Members of the built-in `transform_admin` role have these 
-privileges.
-
-You also need `read` and `view_index_metadata` index privileges on the source
-index and `read`, `create_index`, and `index` privileges on the destination
-index. 
-
-For more information, see <<security-privileges>> and <<built-in-roles>>.
---
+. Verify that your environment is set up properly to use {transforms}. If the
+{es} {security-features} are enabled, to complete this tutorial you need a user
+that has authority to preview and create {transforms}. You must also have
+specific index privileges for the source and destination indices. See
+<<transform-setup>>.
 
 . Choose your _source index_.
 +
diff --git a/docs/reference/transform/index.asciidoc b/docs/reference/transform/index.asciidoc
index 595cbdef56d98..a71d99caac3f3 100644
--- a/docs/reference/transform/index.asciidoc
+++ b/docs/reference/transform/index.asciidoc
@@ -11,6 +11,7 @@ indices that summarize the behavior of users or sessions or other entities in
 your data.
 
 * <<transform-overview>>
+* <<transform-setup>>
 * <<transform-usage>>
 * <<transform-api-quickref>>
 * <<ecommerce-transforms>>
@@ -19,6 +20,7 @@ your data.
 * <<transform-limitations>>
 
 include::overview.asciidoc[]
+include::setup.asciidoc[]
 include::usage.asciidoc[]
 include::checkpoints.asciidoc[]
 include::api-quickref.asciidoc[]
diff --git a/docs/reference/transform/limitations.asciidoc b/docs/reference/transform/limitations.asciidoc
index c4c035e88fbcf..5a66dd8190768 100644
--- a/docs/reference/transform/limitations.asciidoc
+++ b/docs/reference/transform/limitations.asciidoc
@@ -46,17 +46,6 @@ A single cluster will support up to 1,000 {transforms}. When using the
 is returned. Use the `size` and `from` parameters to enumerate through the full 
 list.
 
-
-[float]
-[[transform-node-assignment-limitations]]
-==== {transforms-cap} node assignment not configurable
-
-{transforms-cap} persistent tasks are assigned to the data node running
-fewest persistent tasks at the time of assignment. This cannot be customized.
-It means that if {transforms} are being used then `xpack.transform.enabled`
-must be set to `true` (which is the default) on every data node in the cluster.
-
-
 [float]
 [[transform-aggresponse-limitations]]
 ==== Aggregation responses may be incompatible with destination index mappings
diff --git a/docs/reference/transform/setup.asciidoc b/docs/reference/transform/setup.asciidoc
new file mode 100644
index 0000000000000..862b86eeb74c8
--- /dev/null
+++ b/docs/reference/transform/setup.asciidoc
@@ -0,0 +1,40 @@
+[role="xpack"]
+[[transform-setup]]
+=== Set up {transforms}
+++++
+<titleabbrev>Setup</titleabbrev>
+++++
+
+To use the {transforms}, you must have the
+{subscriptions}[appropriate license] and at least one
+<<transform-setup-nodes,{transform} node>> in your {es} cluster. If {stack}
+{security-features} are enabled, you must also ensure your users have the
+<<transform-privileges,necessary privileges>>.
+
+[discrete]
+[[transform-setup-nodes]]
+==== {transform-cap} nodes
+
+To use {transforms}, there must be at least one node in your cluster with
+`xpack.transform.enabled` and `node.transform` set to `true`. By default, all
+nodes are {transform} nodes unless you explicitly change these settings or set 
+`node.data` to `false`.
+
+If you want to control which nodes run {transforms}, set `node.transform` to
+`false` on some nodes.
+
+For more information, see <<transform-settings>> and <<modules-node>>.
+
+[discrete]
+[[transform-privileges]]
+==== Security privileges
+
+The {es} {security-features} provide <<built-in-roles,built-in roles>>
+and <<security-privileges,privileges>> that make it easier to control
+which users can manage or view {transforms}. 
+
+For example, you need `manage_transform` cluster privileges to preview and
+create {transforms}. Members of the built-in `transform_admin` role have these 
+privileges. You also need `read` and `view_index_metadata` index privileges on
+the source index and `read`, `create_index`, and `index` privileges on the
+destination index.

From ca7a135e0892ce4ccd25635140f9c094ebe1b547 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 18 Mar 2020 20:57:50 -0400
Subject: [PATCH 128/285] Improve performance of shards limits decider (#53577)

On clusters with a large number of shards, the shards limits allocation
decider can exhibit poor performance leading to timeouts applying
cluster state updates. This occurs because for every shard, we do a loop
to count the number of shards on the node, and the number of shards for
the index of the shard. This is roughly quadratic in the number of
shards. This loop is not necessary, since we already have a O(1) method
to count the number of non-relocating shards on a node, and with this
commit we add some infrastructure to RoutingNode to make counting the
number of shards per index O(1).
---
 .../cluster/routing/RoutingNode.java          | 32 +++++++++++++++++++
 .../decider/ShardsLimitAllocationDecider.java | 24 +++++---------
 .../cluster/routing/RoutingNodeTests.java     | 14 ++++++++
 3 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index 1494513dee94e..37a4fa2721c31 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -21,15 +21,20 @@
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -48,6 +53,8 @@ public class RoutingNode implements Iterable<ShardRouting> {
 
     private final LinkedHashSet<ShardRouting> relocatingShards;
 
+    private final HashMap<Index, LinkedHashSet<ShardRouting>> shardsByIndex;
+
     public RoutingNode(String nodeId, DiscoveryNode node, ShardRouting... shards) {
         this(nodeId, node, buildShardRoutingMap(shards));
     }
@@ -58,12 +65,14 @@ public RoutingNode(String nodeId, DiscoveryNode node, ShardRouting... shards) {
         this.shards = shards;
         this.relocatingShards = new LinkedHashSet<>();
         this.initializingShards = new LinkedHashSet<>();
+        this.shardsByIndex = new LinkedHashMap<>();
         for (ShardRouting shardRouting : shards.values()) {
             if (shardRouting.initializing()) {
                 initializingShards.add(shardRouting);
             } else if (shardRouting.relocating()) {
                 relocatingShards.add(shardRouting);
             }
+            shardsByIndex.computeIfAbsent(shardRouting.index(), k -> new LinkedHashSet<>()).add(shardRouting);
         }
         assert invariant();
     }
@@ -128,6 +137,7 @@ void add(ShardRouting shard) {
         } else if (shard.relocating()) {
             relocatingShards.add(shard);
         }
+        shardsByIndex.computeIfAbsent(shard.index(), k -> new LinkedHashSet<>()).add(shard);
         assert invariant();
     }
 
@@ -148,11 +158,16 @@ void update(ShardRouting oldShard, ShardRouting newShard) {
             boolean exist = relocatingShards.remove(oldShard);
             assert exist : "expected shard " + oldShard + " to exist in relocatingShards";
         }
+        shardsByIndex.get(oldShard.index()).remove(oldShard);
+        if (shardsByIndex.get(oldShard.index()).isEmpty()) {
+            shardsByIndex.remove(oldShard.index());
+        }
         if (newShard.initializing()) {
             initializingShards.add(newShard);
         } else if (newShard.relocating()) {
             relocatingShards.add(newShard);
         }
+        shardsByIndex.computeIfAbsent(newShard.index(), k -> new LinkedHashSet<>()).add(newShard);
         assert invariant();
     }
 
@@ -167,6 +182,10 @@ void remove(ShardRouting shard) {
             boolean exist = relocatingShards.remove(shard);
             assert exist : "expected shard " + shard + " to exist in relocatingShards";
         }
+        shardsByIndex.get(shard.index()).remove(shard);
+        if (shardsByIndex.get(shard.index()).isEmpty()) {
+            shardsByIndex.remove(shard.index());
+        }
         assert invariant();
     }
 
@@ -269,6 +288,15 @@ public int numberOfOwningShards() {
         return shards.size() - relocatingShards.size();
     }
 
+    public int numberOfOwningShardsForIndex(final Index index) {
+        final LinkedHashSet<ShardRouting> shardRoutings = shardsByIndex.get(index);
+        if (shardRoutings == null) {
+            return 0;
+        } else {
+            return Math.toIntExact(shardRoutings.stream().filter(Predicate.not(ShardRouting::relocating)).count());
+        }
+    }
+
     public String prettyPrint() {
         StringBuilder sb = new StringBuilder();
         sb.append("-----node_id[").append(nodeId).append("][").append(node == null ? "X" : "V").append("]\n");
@@ -316,6 +344,10 @@ private boolean invariant() {
         assert relocatingShards.size() == shardRoutingsRelocating.size();
         assert relocatingShards.containsAll(shardRoutingsRelocating);
 
+        final Map<Index, Set<ShardRouting>> shardRoutingsByIndex =
+            shards.values().stream().collect(Collectors.groupingBy(ShardRouting::index, Collectors.toSet()));
+        assert shardRoutingsByIndex.equals(shardsByIndex);
+
         return true;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index 1c0a0c0ef0a6b..e63db7d4570aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -109,28 +109,20 @@ private Decision doDecide(ShardRouting shardRouting, RoutingNode node, RoutingAl
                     indexShardLimit, clusterShardLimit);
         }
 
-        int indexShardCount = 0;
-        int nodeShardCount = 0;
-        for (ShardRouting nodeShard : node) {
-            // don't count relocating shards...
-            if (nodeShard.relocating()) {
-                continue;
-            }
-            nodeShardCount++;
-            if (nodeShard.index().equals(shardRouting.index())) {
-                indexShardCount++;
-            }
-        }
+        final int nodeShardCount = node.numberOfOwningShards();
 
         if (clusterShardLimit > 0 && decider.test(nodeShardCount, clusterShardLimit)) {
             return allocation.decision(Decision.NO, NAME,
                 "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
                 nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
         }
-        if (indexShardLimit > 0 && decider.test(indexShardCount, indexShardLimit)) {
-            return allocation.decision(Decision.NO, NAME,
-                "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
-                indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
+        if (indexShardLimit > 0) {
+            final int indexShardCount = node.numberOfOwningShardsForIndex(shardRouting.index());
+            if (decider.test(indexShardCount, indexShardLimit)) {
+                return allocation.decision(Decision.NO, NAME,
+                    "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
+                    indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
+            }
         }
         return allocation.decision(Decision.YES, NAME,
             "the shard count [%d] for this node is under the index limit [%d] and cluster level node limit [%d]",
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java
index c88708a33f457..5cde6459e9339 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
@@ -115,4 +116,17 @@ public void testNumberOfOwningShards() {
         assertThat(routingNode.numberOfOwningShards(), equalTo(2));
     }
 
+    public void testNumberOfOwningShardsForIndex() {
+        final ShardRouting test1Shard0 =
+            TestShardRouting.newShardRouting("test1", 0, "node-1", false, ShardRoutingState.STARTED);
+        final ShardRouting test2Shard0 =
+            TestShardRouting.newShardRouting("test2", 0, "node-1", "node-2", false, ShardRoutingState.RELOCATING);
+        routingNode.add(test1Shard0);
+        routingNode.add(test2Shard0);
+        assertThat(routingNode.numberOfOwningShardsForIndex(new Index("test", IndexMetaData.INDEX_UUID_NA_VALUE)), equalTo(2));
+        assertThat(routingNode.numberOfOwningShardsForIndex(new Index("test1", IndexMetaData.INDEX_UUID_NA_VALUE)), equalTo(1));
+        assertThat(routingNode.numberOfOwningShardsForIndex(new Index("test2", IndexMetaData.INDEX_UUID_NA_VALUE)), equalTo(0));
+        assertThat(routingNode.numberOfOwningShardsForIndex(new Index("test3", IndexMetaData.INDEX_UUID_NA_VALUE)), equalTo(0));
+    }
+
 }

From a2b428f5b433104281c83f12366bad180c847113 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 19 Mar 2020 08:10:34 +0100
Subject: [PATCH 129/285] Disable distributed sort optimization on scroll
 requests (#53759)

This commit disables the sort optimization added in #51852 for scroll requests.
Scroll queries keep a state per shard so we cannot modify the request on
the first round (submit).
This bug was introduced in non-released versions which is why this pr
is marked as a non-issue.
---
 .../SearchQueryThenFetchAsyncAction.java      |  5 ++-
 .../SearchQueryThenFetchAsyncActionTests.java | 41 +++++++++++++++----
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index e42d8405da5b0..e8e864ddd1b47 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -88,7 +88,10 @@ protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget
     @Override
     protected void onShardResult(SearchPhaseResult result, SearchShardIterator shardIt) {
         QuerySearchResult queryResult = result.queryResult();
-        if (queryResult.isNull() == false && queryResult.topDocs().topDocs instanceof TopFieldDocs) {
+        if (queryResult.isNull() == false
+                // disable sort optims for scroll requests because they keep track of the last bottom doc locally (per shard)
+                && getRequest().scroll() == null
+                && queryResult.topDocs().topDocs instanceof TopFieldDocs) {
             TopFieldDocs topDocs = (TopFieldDocs) queryResult.topDocs().topDocs;
             if (bottomSortCollector == null) {
                 synchronized (this) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index 0d4e0ede61167..5a1d2fe351ea5 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.shard.ShardId;
@@ -59,6 +60,14 @@
 
 public class SearchQueryThenFetchAsyncActionTests extends ESTestCase {
     public void testBottomFieldSort() throws InterruptedException {
+        testCase(false);
+    }
+
+    public void testScrollDisableBottomFieldSort() throws InterruptedException {
+        testCase(true);
+    }
+
+    private void testCase(boolean withScroll) throws InterruptedException {
         final TransportSearchAction.SearchTimeProvider timeProvider =
             new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), System::nanoTime);
 
@@ -89,10 +98,10 @@ public void sendExecuteQuery(Transport.Connection connection, ShardSearchRequest
                     new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null, OriginalIndices.NONE));
                 SortField sortField = new SortField("timestamp", SortField.Type.LONG);
                 queryResult.topDocs(new TopDocsAndMaxScore(new TopFieldDocs(
-                    new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
-                    new FieldDoc[] {
-                        new FieldDoc(randomInt(1000), Float.NaN, new Object[] { request.shardId().id() })
-                    }, new SortField[] { sortField }), Float.NaN),
+                        new TotalHits(1, withScroll ? TotalHits.Relation.EQUAL_TO : TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO),
+                        new FieldDoc[] {
+                            new FieldDoc(randomInt(1000), Float.NaN, new Object[] { request.shardId().id() })
+                        }, new SortField[] { sortField }), Float.NaN),
                     new DocValueFormat[] { DocValueFormat.RAW });
                 queryResult.from(0);
                 queryResult.size(1);
@@ -109,8 +118,12 @@ public void sendExecuteQuery(Transport.Connection connection, ShardSearchRequest
         searchRequest.setBatchedReduceSize(2);
         searchRequest.source(new SearchSourceBuilder()
             .size(1)
-            .trackTotalHitsUpTo(2)
             .sort(SortBuilders.fieldSort("timestamp")));
+        if (withScroll) {
+            searchRequest.scroll(TimeValue.timeValueMillis(100));
+        } else {
+            searchRequest.source().trackTotalHitsUpTo(2);
+        }
         searchRequest.allowPartialSearchResults(false);
         SearchPhaseController controller = new SearchPhaseController((b) -> new InternalAggregation.ReduceContextBuilder() {
             @Override
@@ -143,12 +156,22 @@ public void run() {
         action.start();
         latch.await();
         assertThat(successfulOps.get(), equalTo(numShards));
-        assertTrue(canReturnNullResponse.get());
-        assertThat(numWithTopDocs.get(), greaterThanOrEqualTo(1));
+        if (withScroll) {
+            assertFalse(canReturnNullResponse.get());
+            assertThat(numWithTopDocs.get(), equalTo(0));
+        } else {
+            assertTrue(canReturnNullResponse.get());
+            assertThat(numWithTopDocs.get(), greaterThanOrEqualTo(1));
+        }
         SearchPhaseController.ReducedQueryPhase phase = action.results.reduce();
         assertThat(phase.numReducePhases, greaterThanOrEqualTo(1));
-        assertThat(phase.totalHits.value, equalTo(2L));
-        assertThat(phase.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+        if (withScroll) {
+            assertThat(phase.totalHits.value, equalTo((long) numShards));
+            assertThat(phase.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
+        } else {
+            assertThat(phase.totalHits.value, equalTo(2L));
+            assertThat(phase.totalHits.relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO));
+        }
         assertThat(phase.sortedTopDocs.scoreDocs.length, equalTo(1));
         assertThat(phase.sortedTopDocs.scoreDocs[0], instanceOf(FieldDoc.class));
         assertThat(((FieldDoc) phase.sortedTopDocs.scoreDocs[0]).fields.length, equalTo(1));

From 1dac8dffc657389d245918c055d5b0e1be1bbb1c Mon Sep 17 00:00:00 2001
From: Przemyslaw Gomulka <przemyslaw.gomulka@elastic.co>
Date: Thu, 19 Mar 2020 08:48:24 +0100
Subject: [PATCH 130/285] Fix NPE when logging null values in JSON (#53715)

Slow log's routing value when null was causing ESLogMessage.asJson
method to throw Null Pointer Exception. This should be fixed in
ESLogMessage as well as prevent passing that key at all.
This only happens in 8 because of previous refactoring #46702
---
 .../common/logging/ESLogMessage.java              |  2 +-
 .../org/elasticsearch/index/IndexingSlowLog.java  |  6 ++++--
 .../elasticsearch/index/IndexingSlowLogTests.java | 15 +++++++++++++++
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java b/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java
index 0a0816b4aa0b9..6aaf5f1bd862b 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java
@@ -79,7 +79,7 @@ protected void asJson(StringBuilder sb) {
             StringBuilders.escapeJson(sb, start);
             sb.append(Chars.DQUOTE).append(':').append(Chars.DQUOTE);
             start = sb.length();
-            sb.append(getIndexedReadOnlyStringMap().getValueAt(i).toString());
+            sb.append((Object) getIndexedReadOnlyStringMap().getValueAt(i));
             StringBuilders.escapeJson(sb, start);
             sb.append(Chars.DQUOTE);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
index 9dc8dd0a9167d..61b5c8f1acdb4 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
@@ -24,8 +24,8 @@
 import org.apache.logging.log4j.util.StringBuilders;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.logging.ESLogMessage;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
@@ -180,7 +180,9 @@ private static Map<String, Object> prepareMap(Index index, ParsedDocument doc, l
             map.put("took", TimeValue.timeValueNanos(tookInNanos));
             map.put("took_millis", ""+TimeUnit.NANOSECONDS.toMillis(tookInNanos));
             map.put("id", doc.id());
-            map.put("routing", doc.routing());
+            if (doc.routing() != null) {
+                map.put("routing", doc.routing());
+            }
 
             if (maxSourceCharsToLog == 0 || doc.source() == null || doc.source().length() == 0) {
                 return map;
diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index 78d25e996ab2a..7047b4799f12c 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -46,6 +46,7 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
 public class IndexingSlowLogTests extends ESTestCase {
@@ -72,6 +73,20 @@ public void testSlowLogMessageHasJsonFields() throws IOException {
         assertThat(p.get("source"), containsString("{\\\"foo\\\":\\\"bar\\\"}"));
     }
 
+    public void testEmptyRoutingField() throws IOException {
+        BytesReference source = BytesReference.bytes(JsonXContent.contentBuilder()
+                                                                 .startObject().field("foo", "bar").endObject());
+        ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1),
+            SeqNoFieldMapper.SequenceIDFields.emptySeqID(), "id",
+            null, null, source, XContentType.JSON, null);
+        Index index = new Index("foo", "123");
+
+        ESLogMessage p = IndexingSlowLogMessage.of(index, pd, 10, true, 0);
+        assertThat(p.get("routing"), nullValue());
+
+        assertThat(p.asString(), containsString("routing[]"));
+    }
+
     public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         BytesReference source = BytesReference.bytes(JsonXContent.contentBuilder()
             .startObject().field("foo", "bar").endObject());

From 55535b6be77e04099d8f39940081a7733b4ae4d1 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Thu, 19 Mar 2020 11:10:22 +0200
Subject: [PATCH 131/285] [ML] Register ML featureset in XPackClientPlugin
 (#53772)

Featureset needs to stay in XPackClientPlugin for now.
---
 .../java/org/elasticsearch/xpack/core/XPackClientPlugin.java | 4 +++-
 .../java/org/elasticsearch/xpack/ml/MachineLearning.java     | 5 -----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 623afe212e369..237f922816321 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -64,6 +64,7 @@
 import org.elasticsearch.xpack.core.ilm.action.RemoveIndexLifecyclePolicyAction;
 import org.elasticsearch.xpack.core.ilm.action.RetryAction;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
+import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
@@ -409,7 +410,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
             // logstash
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new),
-
+            // ML
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MACHINE_LEARNING, MachineLearningFeatureSetUsage::new),
             // monitoring
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MONITORING, MonitoringFeatureSetUsage::new),
             // security
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 792e0ae25218c..384063aa2e47a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -60,13 +60,10 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
-import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
@@ -975,8 +972,6 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DatafeedState.NAME, DatafeedState::fromStream));
         namedWriteables.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameAnalyticsTaskState.NAME,
             DataFrameAnalyticsTaskState::new));
-        namedWriteables.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MACHINE_LEARNING,
-            MachineLearningFeatureSetUsage::new));
 
         namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
         namedWriteables.addAll(MlEvaluationNamedXContentProvider.getNamedWriteables());

From 5c8cd16ab310bd8cdeed3117d37a0046d36732ac Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Thu, 19 Mar 2020 10:21:43 +0000
Subject: [PATCH 132/285] TermsLookup uses ObjectParser for x-content parsing
 (#53733)

This commit refactors the fromXContent method in TermsLookup to use an
ObjectParser and adds an explicit parsing test.

Related to #53731
---
 .../elasticsearch/indices/TermsLookup.java    | 54 +++++++------------
 .../indices/TermsLookupTests.java             | 13 +++++
 2 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/indices/TermsLookup.java b/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
index db3eef31db27a..a273a938e0dc5 100644
--- a/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
+++ b/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
@@ -20,10 +20,11 @@
 package org.elasticsearch.indices;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -33,10 +34,13 @@
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * Encapsulates the parameters needed to fetch terms.
  */
 public class TermsLookup implements Writeable, ToXContentFragment {
+
     private final String index;
     private final String id;
     private final String path;
@@ -102,40 +106,22 @@ public TermsLookup routing(String routing) {
         return this;
     }
 
+    private static final ConstructingObjectParser<TermsLookup, Void> PARSER = new ConstructingObjectParser<>("terms_lookup",
+        args -> {
+            String index = (String) args[0];
+            String id = (String) args[1];
+            String path = (String) args[2];
+            return new TermsLookup(index, id, path);
+        });
+    static {
+        PARSER.declareString(constructorArg(), new ParseField("index"));
+        PARSER.declareString(constructorArg(), new ParseField("id"));
+        PARSER.declareString(constructorArg(), new ParseField("path"));
+        PARSER.declareString(TermsLookup::routing, new ParseField("routing"));
+    }
+
     public static TermsLookup parseTermsLookup(XContentParser parser) throws IOException {
-        String index = null;
-        String id = null;
-        String path = null;
-        String routing = null;
-        XContentParser.Token token;
-        String currentFieldName = "";
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                switch (currentFieldName) {
-                case "index":
-                    index = parser.text();
-                    break;
-                case "id":
-                    id = parser.text();
-                    break;
-                case "routing":
-                    routing = parser.textOrNull();
-                    break;
-                case "path":
-                    path = parser.text();
-                    break;
-                default:
-                    throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME +
-                        "] query does not support [" + currentFieldName + "] within lookup element");
-                }
-            } else {
-                throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] unknown token ["
-                    + token + "] after [" + currentFieldName + "]");
-            }
-        }
-        return new TermsLookup(index, id, path).routing(routing);
+        return PARSER.parse(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/indices/TermsLookupTests.java b/server/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
index 14cc8316a520e..8ad711794184f 100644
--- a/server/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
@@ -21,6 +21,8 @@
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -78,6 +80,17 @@ public void testSerialization() throws IOException {
         }
     }
 
+    public void testXContentParsing() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+            "{ \"index\" : \"index\", \"id\" : \"id\", \"path\" : \"path\", \"routing\" : \"routing\" }");
+
+        TermsLookup tl = TermsLookup.parseTermsLookup(parser);
+        assertEquals("index", tl.index());
+        assertEquals("id", tl.id());
+        assertEquals("path", tl.path());
+        assertEquals("routing", tl.routing());
+    }
+
     public static TermsLookup randomTermsLookup() {
         return new TermsLookup(
             randomAlphaOfLength(10),

From 7636930ceb0c20ea34f2b9db42094a0a2af75f03 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Thu, 19 Mar 2020 11:26:04 +0000
Subject: [PATCH 133/285] Report parser name and location in XContent
 deprecation warnings (#53752)

It's simple to deprecate a field used in an ObjectParser just by adding deprecation
markers to the relevant ParseField objects. However, the warnings themselves don't
currently have any context; they simply say that a deprecated field has been used,
but not where in the input it appears. This commit adds the parent object parser
name and XContentLocation to these deprecation messages.
---
 .../org/elasticsearch/common/ParseField.java  | 24 ++++++++--
 .../common/xcontent/DeprecationHandler.java   | 48 +++++++++++++------
 .../common/xcontent/ObjectParser.java         |  2 +-
 .../common/xcontent/XContentLocation.java     |  5 +-
 .../elasticsearch/common/ParseFieldTests.java |  6 +--
 .../common/xcontent/ObjectParserTests.java    |  2 +-
 .../common/ScriptProcessorFactoryTests.java   |  2 +-
 .../xcontent/LoggingDeprecationHandler.java   | 20 +++++---
 .../index/query/BoolQueryBuilderTests.java    |  2 +-
 .../script/StoredScriptTests.java             |  2 +-
 ...LoggingDeprecationAccumulationHandler.java | 29 ++++++-----
 .../utils/XContentObjectTransformerTests.java |  4 +-
 .../inference/ingest/InferenceProcessor.java  |  2 +-
 .../xpack/security/authc/ApiKeyService.java   | 23 +++++----
 .../test/ml/data_frame_analytics_crud.yml     |  2 +-
 15 files changed, 115 insertions(+), 58 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
index f4f12ed4f4c81..3872ef4852274 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
@@ -19,11 +19,13 @@
 package org.elasticsearch.common;
 
 import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentLocation;
 
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Supplier;
 
 /**
  * Holds a field that can be found in a request while parsing and its different
@@ -115,6 +117,22 @@ public ParseField withAllDeprecated() {
      *         names for this {@link ParseField}.
      */
     public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
+        return match(null, () -> XContentLocation.UNKNOWN, fieldName, deprecationHandler);
+    }
+
+    /**
+     * Does {@code fieldName} match this field?
+     * @param parserName
+     *            the name of the parent object holding this field
+     * @param location
+     *            the XContentLocation of the field
+     * @param fieldName
+     *            the field name to match against this {@link ParseField}
+     * @param deprecationHandler called if {@code fieldName} is deprecated
+     * @return true if <code>fieldName</code> matches any of the acceptable
+     *         names for this {@link ParseField}.
+     */
+    public boolean match(String parserName, Supplier<XContentLocation> location, String fieldName, DeprecationHandler deprecationHandler) {
         Objects.requireNonNull(fieldName, "fieldName cannot be null");
         // if this parse field has not been completely deprecated then try to
         // match the preferred name
@@ -127,11 +145,11 @@ public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
         for (String depName : deprecatedNames) {
             if (fieldName.equals(depName)) {
                 if (fullyDeprecated) {
-                    deprecationHandler.usedDeprecatedField(fieldName);
+                    deprecationHandler.usedDeprecatedField(parserName, location, fieldName);
                 } else if (allReplacedWith == null) {
-                    deprecationHandler.usedDeprecatedName(fieldName, name);
+                    deprecationHandler.usedDeprecatedName(parserName, location, fieldName, name);
                 } else {
-                    deprecationHandler.usedDeprecatedField(fieldName, allReplacedWith);
+                    deprecationHandler.usedDeprecatedField(parserName, location, fieldName, allReplacedWith);
                 }
                 return true;
             }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
index 2cfda82eb4c48..65f52943f0b86 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common.xcontent;
 
+import java.util.function.Supplier;
+
 /**
  * Callback for notifying the creator of the {@link XContentParser} that
  * parsing hit a deprecated field.
@@ -32,20 +34,35 @@ public interface DeprecationHandler {
      */
     DeprecationHandler THROW_UNSUPPORTED_OPERATION = new DeprecationHandler() {
         @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {
-            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                + usedName + "] which is a deprecated name for [" + replacedWith + "]");
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+            if (parserName != null) {
+                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
+                    + usedName + "] at [" + location.get() + "] which is a deprecated name for [" + replacedWith + "]");
+            } else {
+                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                    + usedName + "] which is a deprecated name for [" + replacedWith + "]");
+            }
         }
         @Override
-        public void usedDeprecatedName(String usedName, String modernName) {
-            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                + usedName + "] which has been replaced with [" + modernName + "]");
+        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+            if (parserName != null) {
+                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
+                    + usedName + "] at [" + location.get() + "] which has been replaced with [" + modernName + "]");
+            } else {
+                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                    + usedName + "] which has been replaced with [" + modernName + "]");
+            }
         }
 
         @Override
-        public void usedDeprecatedField(String usedName) {
-            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                + usedName + "] which has been deprecated entirely");
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+            if (parserName != null) {
+                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
+                    + usedName + "] at [" + location.get() + "] which has been deprecated entirely");
+            } else {
+                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                    + usedName + "] which has been deprecated entirely");
+            }
         }
     };
 
@@ -54,17 +71,17 @@ public void usedDeprecatedField(String usedName) {
      */
     DeprecationHandler IGNORE_DEPRECATIONS = new DeprecationHandler() {
         @Override
-        public void usedDeprecatedName(String usedName, String modernName) {
+        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
 
         }
 
         @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
 
         }
 
         @Override
-        public void usedDeprecatedField(String usedName) {
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
 
         }
     };
@@ -74,7 +91,7 @@ public void usedDeprecatedField(String usedName) {
      * @param usedName the provided field name
      * @param modernName the modern name for the field
      */
-    void usedDeprecatedName(String usedName, String modernName);
+    void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName);
 
     /**
      * Called when the provided field name matches the current field but the entire
@@ -82,12 +99,13 @@ public void usedDeprecatedField(String usedName) {
      * @param usedName the provided field name
      * @param replacedWith the name of the field that replaced this field
      */
-    void usedDeprecatedField(String usedName, String replacedWith);
+    void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith);
 
     /**
      * Called when the provided field name matches the current field but the entire
      * field has been marked as deprecated with no replacement
      * @param usedName the provided field name
      */
-    void usedDeprecatedField(String usedName);
+    void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName);
+
 }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index b11f6afd4bb26..73bbf812f598c 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -571,7 +571,7 @@ private class FieldParser {
         }
 
         void assertSupports(String parserName, XContentParser parser, String currentFieldName) {
-            if (parseField.match(currentFieldName, parser.getDeprecationHandler()) == false) {
+            if (parseField.match(parserName, parser::getTokenLocation, currentFieldName, parser.getDeprecationHandler()) == false) {
                 throw new XContentParseException(parser.getTokenLocation(),
                         "[" + parserName  + "] parsefield doesn't accept: " + currentFieldName);
             }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
index 43ab7503cd1dd..1d5bfd6c2c429 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
@@ -25,7 +25,10 @@
  * position of a parsing error to end users and consequently have line and
  * column numbers starting from 1.
  */
-public class XContentLocation {
+public final class XContentLocation {
+
+    public static final XContentLocation UNKNOWN = new XContentLocation(-1, -1);
+
     public final int lineNumber;
     public final int columnNumber;
 
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
index 381c7c3e959be..e7420356c8fac 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
@@ -66,11 +66,11 @@ public void testDeprecatedWithNoReplacement() {
         ParseField field = new ParseField(name).withDeprecation(alternatives).withAllDeprecated();
         assertFalse(field.match("not a field name", LoggingDeprecationHandler.INSTANCE));
         assertTrue(field.match("dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [dep] used, which has been removed entirely");
+        assertWarnings("Deprecated field [dep] used, this field is unused and will be removed entirely");
         assertTrue(field.match("old_dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [old_dep] used, which has been removed entirely");
+        assertWarnings("Deprecated field [old_dep] used, this field is unused and will be removed entirely");
         assertTrue(field.match("new_dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [new_dep] used, which has been removed entirely");
+        assertWarnings("Deprecated field [new_dep] used, this field is unused and will be removed entirely");
 
 
     }
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index c99d1b10d6a4d..9a813c8a24616 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -223,7 +223,7 @@ class TestStruct {
         objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
         objectParser.parse(parser, s, null);
         assertEquals("foo", s.test);
-        assertWarnings("Deprecated field [old_test] used, expected [test] instead");
+        assertWarnings("[foo][1:15] Deprecated field [old_test] used, expected [test] instead");
     }
 
     public void testFailOnValueType() throws IOException {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index 9559c150df68a..c7018d40b18ca 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -98,7 +98,7 @@ public void testInlineBackcompat() throws Exception {
         configMap.put("inline", "code");
 
         factory.create(null, randomAlphaOfLength(10), configMap);
-        assertWarnings("Deprecated field [inline] used, expected [source] instead");
+        assertWarnings("[script][1:11] Deprecated field [inline] used, expected [source] instead");
     }
 
     public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception {
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
index 9a885b55d146c..5feca4bfaab5a 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.logging.DeprecationLogger;
 
+import java.util.function.Supplier;
+
 /**
  * Logs deprecations to the {@link DeprecationLogger}.
  * <p>
@@ -49,17 +51,23 @@ private LoggingDeprecationHandler() {
     }
 
     @Override
-    public void usedDeprecatedName(String usedName, String modernName) {
-        deprecationLogger.deprecated("Deprecated field [{}] used, expected [{}] instead", usedName, modernName);
+    public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecationLogger.deprecated("{}Deprecated field [{}] used, expected [{}] instead",
+            prefix, usedName, modernName);
     }
 
     @Override
-    public void usedDeprecatedField(String usedName, String replacedWith) {
-        deprecationLogger.deprecated("Deprecated field [{}] used, replaced by [{}]", usedName, replacedWith);
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecationLogger.deprecated("{}Deprecated field [{}] used, replaced by [{}]",
+            prefix, usedName, replacedWith);
     }
 
     @Override
-    public void usedDeprecatedField(String usedName) {
-        deprecationLogger.deprecated("Deprecated field [{}] used, which has been removed entirely", usedName);
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecationLogger.deprecated("{}Deprecated field [{}] used, this field is unused and will be removed entirely",
+            prefix, usedName);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 93d46ed592570..03916bac58a84 100644
--- a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -302,7 +302,7 @@ public void testDeprecation() throws IOException {
         QueryBuilder q = parseQuery(query);
         QueryBuilder expected = new BoolQueryBuilder().mustNot(new MatchAllQueryBuilder());
         assertEquals(expected, q);
-        assertWarnings("Deprecated field [mustNot] used, expected [must_not] instead");
+        assertWarnings("[bool][1:24] Deprecated field [mustNot] used, expected [must_not] instead");
     }
 
     public void testRewrite() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java b/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java
index 627d67dc833e4..47dc7aaae2a3b 100644
--- a/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java
@@ -104,7 +104,7 @@ public void testSourceParsing() throws Exception {
 
             assertThat(parsed, equalTo(source));
         }
-        assertWarnings("Deprecated field [code] used, expected [source] instead");
+        assertWarnings("[stored script source][1:33] Deprecated field [code] used, expected [source] instead");
 
         // complex script with script object and empty options
         try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
index 5e2a4d7a25302..323e3a84cdf2e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
@@ -8,10 +8,12 @@
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentLocation;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
 
 /**
  * Very similar to {@link org.elasticsearch.common.xcontent.LoggingDeprecationHandler} main differences are:
@@ -26,24 +28,27 @@ public class LoggingDeprecationAccumulationHandler implements DeprecationHandler
     private final List<String> deprecations = new ArrayList<>();
 
     @Override
-    public void usedDeprecatedName(String usedName, String modernName) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(usedName, modernName);
-        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, expected [{}] instead",
-            new Object[] {usedName, modernName}));
+    public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(parserName, location, usedName, modernName);
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, expected [{}] instead",
+            new Object[]{prefix, usedName, modernName}));
     }
 
     @Override
-    public void usedDeprecatedField(String usedName, String replacedWith) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName, replacedWith);
-        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, replaced by [{}]",
-            new Object[] {usedName, replacedWith}));
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(parserName, location, usedName, replacedWith);
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, replaced by [{}]",
+            new Object[]{prefix, usedName, replacedWith}));
     }
 
     @Override
-    public void usedDeprecatedField(String usedName) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName);
-        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, which has been deprecated entirely",
-            new Object[]{ usedName }));
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(parserName, location, usedName);
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, unused and will be removed entirely",
+            new Object[]{prefix, usedName}));
     }
 
     /**
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
index d6ebc9a2e150b..886602395238f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
@@ -127,8 +127,8 @@ public void testToMap() throws IOException {
     public void testDeprecationWarnings() throws IOException {
         XContentObjectTransformer<QueryBuilder> queryBuilderTransformer = new XContentObjectTransformer<>(NamedXContentRegistry.EMPTY,
             (p)-> {
-            p.getDeprecationHandler().usedDeprecatedField("oldField", "newField");
-            p.getDeprecationHandler().usedDeprecatedName("oldName", "modernName");
+            p.getDeprecationHandler().usedDeprecatedField(null, null, "oldField", "newField");
+            p.getDeprecationHandler().usedDeprecatedName(null, null, "oldName", "modernName");
             return new BoolQueryBuilder();
             });
         List<String> deprecations = new ArrayList<>();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index 2a4861c5247f7..b52b8b5b64e3f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -242,7 +242,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
                 fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
                 //TODO Remove in 8.x
                 if (fieldMap != null) {
-                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
+                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, FIELD_MAPPINGS, FIELD_MAP);
                 }
             }
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index f36ce90660a23..c363ad1253509 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -618,21 +619,25 @@ private ApiKeyLoggingDeprecationHandler(DeprecationLogger logger, String apiKeyI
         }
 
         @Override
-        public void usedDeprecatedName(String usedName, String modernName) {
-            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], expected [{}] instead",
-                usedName, apiKeyId, modernName);
+        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+            deprecationLogger.deprecated("{}Deprecated field [{}] used in api key [{}], expected [{}] instead",
+                prefix, usedName, apiKeyId, modernName);
         }
 
         @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {
-            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], replaced by [{}]",
-                usedName, apiKeyId, replacedWith);
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+            deprecationLogger.deprecated("{}Deprecated field [{}] used in api key [{}], replaced by [{}]",
+                prefix, usedName, apiKeyId, replacedWith);
         }
 
         @Override
-        public void usedDeprecatedField(String usedName) {
-            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], which has been removed entirely",
-                usedName);
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+            deprecationLogger.deprecated(
+                "{}Deprecated field [{}] used in api key [{}], which is unused and will be removed entirely",
+                prefix, usedName);
         }
     }
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index a5a99b30391ea..c5df3d9f6cc51 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -1869,7 +1869,7 @@ setup:
 
   - do:
       allowed_warnings:
-        - 'Deprecated field [maximum_number_trees] used, expected [max_trees] instead'
+        - '[classification][1:139] Deprecated field [maximum_number_trees] used, expected [max_trees] instead'
       ml.put_data_frame_analytics:
         id: "classification-with-maximum-number-trees"
         body: >

From 8d5478f56c94a42a396397892c6eb12d63f004f3 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Thu, 19 Mar 2020 07:42:26 -0400
Subject: [PATCH 134/285] [DOCS] Add token graph concept docs (#53339)

Adds conceptual docs for token graphs.
These docs cover:

* How a token graph is constructed from a token stream
* How synonyms and multi-position tokens impact token graphs
* How token graphs are used during search
* Why some token filters produce invalid token graphs

Also makes the following supporting changes:
* Adds anchors to the 'Anatomy of an Analyzer' docs for cross-linking
* Adds several SVGs for token graph diagrams
---
 docs/reference/analysis/anatomy.asciidoc      |   3 +
 docs/reference/analysis/concepts.asciidoc     |   4 +-
 docs/reference/analysis/token-graphs.asciidoc | 104 ++++++++++++++++++
 .../synonym-graph-tokenfilter.asciidoc        |   4 +-
 .../word-delimiter-graph-tokenfilter.asciidoc |   4 +-
 .../images/analysis/token-graph-dns-ex.svg    |  65 +++++++++++
 .../analysis/token-graph-dns-invalid-ex.svg   |  72 ++++++++++++
 .../analysis/token-graph-dns-synonym-ex.svg   |  72 ++++++++++++
 .../images/analysis/token-graph-qbf-ex.svg    |  45 ++++++++
 .../analysis/token-graph-qbf-synonym-ex.svg   |  52 +++++++++
 10 files changed, 420 insertions(+), 5 deletions(-)
 create mode 100644 docs/reference/analysis/token-graphs.asciidoc
 create mode 100644 docs/reference/images/analysis/token-graph-dns-ex.svg
 create mode 100644 docs/reference/images/analysis/token-graph-dns-invalid-ex.svg
 create mode 100644 docs/reference/images/analysis/token-graph-dns-synonym-ex.svg
 create mode 100644 docs/reference/images/analysis/token-graph-qbf-ex.svg
 create mode 100644 docs/reference/images/analysis/token-graph-qbf-synonym-ex.svg

diff --git a/docs/reference/analysis/anatomy.asciidoc b/docs/reference/analysis/anatomy.asciidoc
index 1db14e787a54a..22e7ffda667d4 100644
--- a/docs/reference/analysis/anatomy.asciidoc
+++ b/docs/reference/analysis/anatomy.asciidoc
@@ -10,6 +10,7 @@ blocks into analyzers suitable for different languages and types of text.
 Elasticsearch also exposes the individual building blocks so that they can be
 combined to define new <<analysis-custom-analyzer,`custom`>> analyzers.
 
+[[analyzer-anatomy-character-filters]]
 ==== Character filters
 
 A _character filter_ receives the original text as a stream of characters and
@@ -21,6 +22,7 @@ elements like `<b>` from the stream.
 An analyzer may have *zero or more* <<analysis-charfilters,character filters>>,
 which are applied in order.
 
+[[analyzer-anatomy-tokenizer]]
 ==== Tokenizer
 
 A _tokenizer_  receives a stream of characters, breaks it up into individual
@@ -35,6 +37,7 @@ the term represents.
 
 An analyzer must have *exactly one* <<analysis-tokenizers,tokenizer>>.
 
+[[analyzer-anatomy-token-filters]]
 ==== Token filters
 
 A _token filter_ receives the token stream and may add, remove, or change
diff --git a/docs/reference/analysis/concepts.asciidoc b/docs/reference/analysis/concepts.asciidoc
index 2468286e3a719..2e431efcd5fec 100644
--- a/docs/reference/analysis/concepts.asciidoc
+++ b/docs/reference/analysis/concepts.asciidoc
@@ -8,6 +8,8 @@ This section explains the fundamental concepts of text analysis in {es}.
 
 * <<analyzer-anatomy>>
 * <<analysis-index-search-time>>
+* <<token-graphs>>
 
 include::anatomy.asciidoc[]
-include::index-search-time.asciidoc[]
\ No newline at end of file
+include::index-search-time.asciidoc[]
+include::token-graphs.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/analysis/token-graphs.asciidoc b/docs/reference/analysis/token-graphs.asciidoc
new file mode 100644
index 0000000000000..ab1dc52f5131b
--- /dev/null
+++ b/docs/reference/analysis/token-graphs.asciidoc
@@ -0,0 +1,104 @@
+[[token-graphs]]
+=== Token graphs
+
+When a <<analyzer-anatomy-tokenizer,tokenizer>> converts a text into a stream of
+tokens, it also records the following:
+
+* The `position` of each token in the stream
+* The `positionLength`, the number of positions that a token spans
+
+Using these, you can create a
+https://en.wikipedia.org/wiki/Directed_acyclic_graph[directed acyclic graph],
+called a _token graph_, for a stream. In a token graph, each position represents
+a node. Each token represents an edge or arc, pointing to the next position.
+
+image::images/analysis/token-graph-qbf-ex.svg[align="center"]
+
+[[token-graphs-synonyms]]
+==== Synonyms
+
+Some <<analyzer-anatomy-token-filters,token filters>> can add new tokens, like
+synonyms, to an existing token stream. These synonyms often span the same
+positions as existing tokens.
+
+In the following graph, `quick` and its synonym `fast` both have a position of
+`0`. They span the same positions.
+
+image::images/analysis/token-graph-qbf-synonym-ex.svg[align="center"]
+
+[[token-graphs-multi-position-tokens]]
+==== Multi-position tokens
+
+Some token filters can add tokens that span multiple positions. These can
+include tokens for multi-word synonyms, such as using "atm" as a synonym for
+"automatic teller machine."
+
+However, only some token filters, known as _graph token filters_, accurately
+record the `positionLength` for multi-position tokens. This filters include:
+
+* <<analysis-synonym-graph-tokenfilter,`synonym_graph`>>
+* <<analysis-word-delimiter-graph-tokenfilter,`word_delimiter_graph`>>
+
+In the following graph, `domain name system` and its synonym, `dns`, both have a
+position of `0`. However, `dns` has a `positionLength` of `3`. Other tokens in
+the graph have a default `positionLength` of `1`.
+
+image::images/analysis/token-graph-dns-synonym-ex.svg[align="center"]
+
+[[token-graphs-token-graphs-search]]
+===== Using token graphs for search 
+
+<<analysis-index-search-time,Indexing>> ignores the `positionLength` attribute
+and does not support token graphs containing multi-position tokens.
+
+However, queries, such as the <<query-dsl-match-query,`match`>> or
+<<query-dsl-match-query-phrase,`match_phrase`>> query, can use these graphs to
+generate multiple sub-queries from a single query string.
+
+.*Example*
+[%collapsible]
+====
+
+A user runs a search for the following phrase using the `match_phrase` query:
+
+`domain name system is fragile`
+
+During <<analysis-index-search-time,search analysis>>, `dns`, a synonym for
+`domain name system`, is added to the query string's token stream. The `dns`
+token has a `positionLength` of `3`.
+
+image::images/analysis/token-graph-dns-synonym-ex.svg[align="center"]
+
+The `match_phrase` query uses this graph to generate sub-queries for the
+following phrases:
+
+[source,text]
+------
+dns is fragile
+domain name system is fragile
+------
+
+This means the query matches documents containing either `dns is fragile` _or_
+`domain name system is fragile`.
+====
+
+[[token-graphs-invalid-token-graphs]]
+===== Invalid token graphs
+
+The following token filters can add tokens that span multiple positions but
+only record a default `positionLength` of `1`:
+
+* <<analysis-synonym-tokenfilter,`synonym`>>
+* <<analysis-word-delimiter-tokenfilter,`word_delimiter`>>
+
+This means these filters will produce invalid token graphs for streams
+containing such tokens.
+
+In the following graph, `dns` is a multi-position synonym for `domain name
+system`. However, `dns` has the default `positionLength` value of `1`, resulting
+in an invalid graph.
+
+image::images/analysis/token-graph-dns-invalid-ex.svg[align="center"]
+
+Avoid using invalid token graphs for search. Invalid graphs can cause unexpected
+search results.
\ No newline at end of file
diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
index e6bc76e408f23..582ce99b20bf7 100644
--- a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
@@ -8,8 +8,8 @@ The `synonym_graph` token filter allows to easily handle synonyms,
 including multi-word synonyms correctly during the analysis process.
 
 In order to properly handle multi-word synonyms this token filter
-creates a "graph token stream" during processing.  For more information
-on this topic and its various complexities, please read the
+creates a <<token-graphs,graph token stream>> during processing.  For more
+information on this topic and its various complexities, please read the
 http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's TokenStreams are actually graphs] blog post.
 
 ["NOTE",id="synonym-graph-index-note"]
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
index 8581d8cb7ec17..1f2f61a5071dc 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
@@ -440,8 +440,8 @@ that span multiple positions when any of the following parameters are `true`:
 
 However, only the `word_delimiter_graph` filter assigns multi-position tokens a
 `positionLength` attribute, which indicates the number of positions a token
-spans. This ensures the `word_delimiter_graph` filter always produces valid token
-https://en.wikipedia.org/wiki/Directed_acyclic_graph[graphs].
+spans. This ensures the `word_delimiter_graph` filter always produces valid
+<<token-graphs,token graphs>>.
 
 The `word_delimiter` filter does not assign multi-position tokens a
 `positionLength` attribute. This means it produces invalid graphs for streams
diff --git a/docs/reference/images/analysis/token-graph-dns-ex.svg b/docs/reference/images/analysis/token-graph-dns-ex.svg
new file mode 100644
index 0000000000000..0eda4fa54bd20
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-dns-ex.svg
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="810px" height="54px" viewBox="0 0 810 54" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="token-graph-dns-ex" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M167.71559,4.1045 L167.71559,4.1045 C167.71559,2.5803148 168.951186,1.344719 170.475371,1.344719 L212.325881,1.344719 C213.05783,1.344719 213.759795,1.63547523 214.277343,2.1530381 C214.794906,2.67060097 215.085662,3.3725587 215.085662,4.1044999 L215.085662,48.6636799 C215.085662,50.1878651 213.850066,51.4234609 212.325881,51.4234609 L170.475371,51.4234609 C168.951186,51.4234609 167.71559,50.1878652 167.71559,48.6636799 L167.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M167.71559,4.1045 L167.71559,4.1045 C167.71559,2.5803148 168.951186,1.344719 170.475371,1.344719 L212.325881,1.344719 C213.05783,1.344719 213.759795,1.63547523 214.277343,2.1530381 C214.794906,2.67060097 215.085662,3.3725587 215.085662,4.1044999 L215.085662,48.6636799 C215.085662,50.1878651 213.850066,51.4234609 212.325881,51.4234609 L170.475371,51.4234609 C168.951186,51.4234609 167.71559,50.1878652 167.71559,48.6636799 L167.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M193.1659,33.3041 L191.525275,33.3041 L191.525275,22.850967 C191.129442,23.225967 190.608608,23.6061753 189.962775,23.991592 C189.327358,24.366592 188.754442,24.647842 188.244025,24.835342 L188.244025,23.241592 C189.160692,22.8145087 189.962775,22.2936753 190.650275,21.679092 C191.337775,21.0645087 191.82215,20.4707587 192.1034,19.897842 L193.1659,19.897842 L193.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="159.08891 16.85302 178.521985 16.85302 178.521985 38.679795 159.08891 38.679795"></polygon>
+                <line x1="178.52199" y1="27.7664" x2="48.19127" y2="27.2309691" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="160.52213" y1="27.69245" x2="48.19128" y2="27.2309708" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="160.50177 32.64761 174.136307 27.7483867 160.542496 22.7372937"></polygon>
+                <path d="M319.71558,4.1045 L319.71558,4.1045 C319.71558,2.5803148 320.951206,1.344719 322.475376,1.344719 L364.325901,1.344719 C365.057835,1.344719 365.7598,1.63547523 366.277348,2.1530381 C366.794926,2.67060097 367.085667,3.3725587 367.085667,4.1044999 L367.085667,48.6636799 C367.085667,50.1878651 365.850071,51.4234609 364.325901,51.4234609 L322.475376,51.4234609 C320.951206,51.4234609 319.71558,50.1878652 319.71558,48.6636799 L319.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M319.71558,4.1045 L319.71558,4.1045 C319.71558,2.5803148 320.951206,1.344719 322.475376,1.344719 L364.325901,1.344719 C365.057835,1.344719 365.7598,1.63547523 366.277348,2.1530381 C366.794926,2.67060097 367.085667,3.3725587 367.085667,4.1044999 L367.085667,48.6636799 C367.085667,50.1878651 365.850071,51.4234609 364.325901,51.4234609 L322.475376,51.4234609 C320.951206,51.4234609 319.71558,50.1878652 319.71558,48.6636799 L319.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M347.6034,31.72597 L347.6034,33.304095 L338.775275,33.304095 C338.764858,32.9082617 338.827358,32.5280533 338.962775,32.16347 C339.191942,31.5593033 339.551317,30.9655533 340.0409,30.38222 C340.5409,29.7988867 341.25965,29.1270117 342.19715,28.366595 C343.645067,27.1790899 344.624233,26.2363791 345.13465,25.5384624 C345.645067,24.8405457 345.900275,24.1842957 345.900275,23.5697124 C345.900275,22.9134624 345.6659,22.3665874 345.19715,21.9290874 C344.738817,21.4811707 344.13465,21.2572124 343.38465,21.2572124 C342.592983,21.2572124 341.957567,21.4967957 341.4784,21.9759624 C341.00965,22.4447124 340.775275,23.0957541 340.775275,23.9290874 L339.087775,23.7572124 C339.202358,22.4967957 339.63465,21.5384624 340.38465,20.8822124 C341.145067,20.2259624 342.155483,19.8978374 343.4159,19.8978374 C344.69715,19.8978374 345.712775,20.2520041 346.462775,20.9603374 C347.212775,21.6686707 347.587775,22.5488791 347.587775,23.6009624 C347.587775,24.1322124 347.4784,24.6582541 347.25965,25.1790874 C347.0409,25.6999207 346.676317,26.2467957 346.1659,26.8197124 C345.6659,27.3822124 344.82215,28.1634649 343.63465,29.16347 C342.655483,29.9863867 342.025275,30.5488867 341.744025,30.85097 C341.462775,31.1426367 341.233608,31.4343033 341.056525,31.72597 L347.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="207.08891 16.85302 226.52197 16.85302 226.52197 38.679795 207.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="311.0889 16.85302 330.521975 16.85302 330.521975 38.679795 311.0889 38.679795"></polygon>
+                <path d="M487.7156,4.1045 L487.7156,4.1045 C487.7156,2.5803148 488.951196,1.344719 490.475366,1.344719 L532.325891,1.344719 C533.057824,1.344719 533.75979,1.63547523 534.277368,2.1530381 C534.794946,2.67060097 535.085717,3.3725587 535.085717,4.1044999 L535.085717,48.6636799 C535.085717,50.1878651 533.850122,51.4234609 532.325891,51.4234609 L490.475366,51.4234609 C488.951196,51.4234609 487.7156,50.1878652 487.7156,48.6636799 L487.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M487.7156,4.1045 L487.7156,4.1045 C487.7156,2.5803148 488.951196,1.344719 490.475366,1.344719 L532.325891,1.344719 C533.057824,1.344719 533.75979,1.63547523 534.277368,2.1530381 C534.794946,2.67060097 535.085717,3.3725587 535.085717,4.1044999 L535.085717,48.6636799 C535.085717,50.1878651 533.850122,51.4234609 532.325891,51.4234609 L490.475366,51.4234609 C488.951196,51.4234609 487.7156,50.1878652 487.7156,48.6636799 L487.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M506.994,29.77284 L508.634625,29.55409 C508.822125,30.49159 509.139833,31.163465 509.58775,31.569715 C510.046083,31.975965 510.598167,32.17909 511.244,32.17909 C512.014833,32.17909 512.665875,31.913465 513.197125,31.382215 C513.728375,30.850965 513.994,30.1895067 513.994,29.3978324 C513.994,28.64784 513.749208,28.0280458 513.259625,27.5384574 C512.770042,27.0488741 512.145042,26.8040824 511.384625,26.8040824 C511.072125,26.8040824 510.686708,26.8613741 510.228375,26.9759574 L510.415875,25.5384574 C510.520042,25.5488741 510.608583,25.5540824 510.6815,25.5540824 C511.379417,25.5540824 512.004417,25.3717907 512.5565,25.0072074 C513.119,24.6426241 513.40025,24.0853324 513.40025,23.3353324 C513.40025,22.7311657 513.197125,22.2311657 512.790875,21.8353324 C512.384625,21.4394991 511.858583,21.2415824 511.21275,21.2415824 C510.577333,21.2415824 510.046083,21.4447074 509.619,21.8509574 C509.191917,22.2467907 508.921083,22.8457491 508.8065,23.6478324 L507.165875,23.3509574 C507.363792,22.2572074 507.816917,21.4082491 508.52525,20.8040824 C509.233583,20.1999157 510.119,19.8978324 511.1815,19.8978324 C511.910667,19.8978324 512.577333,20.0540824 513.1815,20.3665824 C513.796083,20.6790824 514.264833,21.1061657 514.58775,21.6478324 C514.921083,22.1894991 515.08775,22.7624157 515.08775,23.3665824 C515.08775,23.9394991 514.9315,24.4655407 514.619,24.9447074 C514.3065,25.4134574 513.848167,25.7884574 513.244,26.0697074 C514.035667,26.2572074 514.65025,26.6374157 515.08775,27.2103324 C515.52525,27.7832541 515.744,28.5020067 515.744,29.36659 C515.744,30.52284 515.316917,31.507215 514.46275,32.319715 C513.619,33.132215 512.546083,33.538465 511.244,33.538465 C510.066917,33.538465 509.092958,33.1895067 508.322125,32.49159 C507.551292,31.7936733 507.108583,30.8874233 506.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="479.0889 16.85302 498.521944 16.85302 498.521944 38.679795 479.0889 38.679795"></polygon>
+                <line x1="496.9606" y1="27.00144" x2="369.18106" y2="28.765211" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="478.9623" y1="27.24987" x2="369.18105" y2="28.7652127" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="479.03064 32.20459 492.575257 27.061966 478.89386 22.295143"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="58.385826 10.47244 147.83464 10.47244 147.83464 41.842511 58.385826 41.842511"></polygon>
+                <path d="M80.01599,33.07748 L80.01599,31.85873 C79.4118233,32.8170633 78.5107817,33.29623 77.312865,33.29623 C76.5420317,33.29623 75.8336983,33.0826883 75.187865,32.655605 C74.5420317,32.2285217 74.0420317,31.6347717 73.687865,30.874355 C73.3336983,30.1139383 73.156615,29.2389383 73.156615,28.249355 C73.156615,27.2805999 73.3180733,26.4055974 73.64099,25.6243474 C73.9639067,24.8326807 74.4430733,24.2285141 75.07849,23.8118474 C75.7243233,23.3951807 76.4482817,23.1868474 77.250365,23.1868474 C77.8336983,23.1868474 78.3493233,23.3118474 78.79724,23.5618474 C79.2555733,23.8014307 79.625365,24.1191391 79.906615,24.5149724 L79.906615,19.7180974 L81.54724,19.7180974 L81.54724,33.0774804 L80.01599,33.0774804 L80.01599,33.07748 Z M74.844115,28.249355 C74.844115,29.4889383 75.1045317,30.4160217 75.625365,31.030605 C76.1461983,31.6451883 76.7607817,31.95248 77.469115,31.95248 C78.187865,31.95248 78.79724,31.6608133 79.29724,31.07748 C79.79724,30.48373 80.04724,29.5878967 80.04724,28.38998 C80.04724,27.0670583 79.7920317,26.0983057 79.281615,25.4837224 C78.7711983,24.8587224 78.14099,24.5462224 77.39099,24.5462224 C76.67224,24.5462224 76.0680733,24.8430974 75.57849,25.4368474 C75.0889067,26.0305974 74.844115,26.9680999 74.844115,28.249355 Z M83.516707,28.23373 C83.516707,26.4420583 84.0114987,25.1191391 85.001082,24.2649724 C85.8344153,23.5462224 86.8500403,23.1868474 88.047957,23.1868474 C89.3812903,23.1868474 90.469832,23.6243474 91.313582,24.4993474 C92.157332,25.3639307 92.579207,26.5670583 92.579207,28.10873 C92.579207,29.3483133 92.391707,30.32748 92.016707,31.04623 C91.641707,31.7545633 91.094832,32.3066467 90.376082,32.70248 C89.6677487,33.0983133 88.891707,33.29623 88.047957,33.29623 C86.6937903,33.29623 85.6000403,32.8639383 84.766707,31.999355 C83.9333737,31.124355 83.516707,29.8691467 83.516707,28.23373 Z M85.204207,28.23373 C85.204207,29.4733133 85.469832,30.405605 86.001082,31.030605 C86.5427487,31.6451883 87.2250403,31.95248 88.047957,31.95248 C88.860457,31.95248 89.5375403,31.6451883 90.079207,31.030605 C90.6208737,30.405605 90.891707,29.4576883 90.891707,28.186855 C90.891707,26.9889333 90.6208737,26.0826807 90.079207,25.4680974 C89.5375403,24.8535141 88.860457,24.5462224 88.047957,24.5462224 C87.2250403,24.5462224 86.5427487,24.8535141 86.001082,25.4680974 C85.469832,26.0722641 85.204207,26.9941416 85.204207,28.23373 Z M94.501799,33.07748 L94.501799,23.405597 L95.970549,23.405597 L95.970549,24.764972 C96.2726323,24.2858053 96.673674,23.905597 97.173674,23.624347 C97.6840907,23.3326803 98.2570073,23.186847 98.892424,23.186847 C99.611174,23.186847 100.199716,23.3378887 100.658049,23.639972 C101.116382,23.9316387 101.439299,24.343097 101.626799,24.874347 C102.397632,23.749347 103.392424,23.186847 104.611174,23.186847 C105.579924,23.186847 106.319507,23.452472 106.829924,23.983722 C107.350757,24.514972 107.611174,25.3326803 107.611174,26.436847 L107.611174,33.07748 L105.970549,33.07748 L105.970549,26.983722 C105.970549,26.327472 105.918466,25.858722 105.814299,25.577472 C105.710132,25.2858053 105.517424,25.0514303 105.236174,24.874347 C104.954924,24.6972637 104.626799,24.608722 104.251799,24.608722 C103.574716,24.608722 103.012216,24.8378887 102.564299,25.296222 C102.116382,25.7441387 101.892424,26.4628887 101.892424,27.452472 L101.892424,33.07748 L100.251799,33.07748 L100.251799,26.796222 C100.251799,26.0670553 100.116382,25.5201803 99.845549,25.155597 C99.5747157,24.7910137 99.1372157,24.608722 98.533049,24.608722 C98.0747157,24.608722 97.6476323,24.7285137 97.251799,24.968097 C96.8559657,25.2076803 96.5695073,25.561847 96.392424,26.030597 C96.2257573,26.499347 96.142424,27.1764329 96.142424,28.0618546 L96.142424,33.07748 L94.501799,33.07748 L94.501799,33.07748 Z M116.355101,31.88998 C115.740518,32.4003967 115.151976,32.76498 114.589476,32.98373 C114.037393,33.1920633 113.438434,33.29623 112.792601,33.29623 C111.730101,33.29623 110.912393,33.0358133 110.339476,32.51498 C109.766559,31.9941467 109.480101,31.3326883 109.480101,30.530605 C109.480101,30.0514383 109.589476,29.6191467 109.808226,29.23373 C110.026976,28.8378967 110.308226,28.5253967 110.651976,28.29623 C111.006143,28.0566416 111.401976,27.8743474 111.839476,27.7493474 C112.151976,27.6660141 112.636351,27.5826807 113.292601,27.4993474 C114.615518,27.3430974 115.589476,27.1555974 116.214476,26.9368474 C116.224893,26.7076807 116.230101,26.5670557 116.230101,26.5149724 C116.230101,25.8483057 116.073851,25.3743474 115.761351,25.0930974 C115.344684,24.7285141 114.719684,24.5462224 113.886351,24.5462224 C113.115518,24.5462224 112.547809,24.6816391 112.183226,24.9524724 C111.818643,25.2233057 111.547809,25.6972641 111.370726,26.3743474 L109.761351,26.1555974 C109.907184,25.4785141 110.146768,24.9316391 110.480101,24.5149724 C110.813434,24.0878891 111.297809,23.7597641 111.933226,23.5305974 C112.568643,23.3014307 113.297809,23.1868474 114.120726,23.1868474 C114.954059,23.1868474 115.625934,23.2858057 116.136351,23.4837224 C116.657184,23.6712224 117.037393,23.9160141 117.276976,24.2180974 C117.526976,24.5097641 117.698851,24.8795557 117.792601,25.3274724 C117.844684,25.6087224 117.870726,26.1139307 117.870726,26.8430974 L117.870726,29.030605 C117.870726,30.5514383 117.907184,31.51498 117.980101,31.92123 C118.053018,32.3170633 118.188434,32.70248 118.386351,33.07748 L116.683226,33.07748 C116.506143,32.73373 116.396768,32.3378967 116.355101,31.88998 Z M116.214476,28.218105 C115.620726,28.468105 114.730101,28.6764383 113.542601,28.843105 C112.865518,28.936855 112.386351,29.04623 112.105101,29.17123 C111.823851,29.29623 111.605101,29.4733133 111.448851,29.70248 C111.303018,29.9316467 111.230101,30.1920633 111.230101,30.48373 C111.230101,30.92123 111.396768,31.2858133 111.730101,31.57748 C112.063434,31.8691467 112.547809,32.01498 113.183226,32.01498 C113.808226,32.01498 114.365518,31.8795633 114.855101,31.60873 C115.355101,31.32748 115.719684,30.9472717 115.948851,30.468105 C116.125934,30.093105 116.214476,29.54623 116.214476,28.82748 L116.214476,28.218105 Z M120.418318,21.6087224 L120.418318,19.7180974 L122.058943,19.7180974 L122.058943,21.6087224 L120.418318,21.6087224 Z M120.418318,33.0774804 L120.418318,23.4055974 L122.058943,23.4055974 L122.058943,33.0774804 L120.418318,33.0774804 Z M124.563139,33.0774804 L124.563139,23.4055974 L126.031889,23.4055974 L126.031889,24.7805974 C126.740222,23.7180974 127.766264,23.1868474 129.110014,23.1868474 C129.693347,23.1868474 130.229806,23.2910141 130.719389,23.4993474 C131.208972,23.7076807 131.573556,23.9837224 131.813139,24.3274724 C132.063139,24.6608057 132.235014,25.0618474 132.328764,25.5305974 C132.391264,25.8326807 132.422514,26.3639307 132.422514,27.1243474 L132.422514,33.0774804 L130.781889,33.0774804 L130.781889,27.1868474 C130.781889,26.5201807 130.714181,26.0253891 130.578764,25.7024724 C130.453764,25.3691391 130.229806,25.1035141 129.906889,24.9055974 C129.583972,24.7076807 129.203764,24.6087224 128.766264,24.6087224 C128.068347,24.6087224 127.464181,24.8326807 126.953764,25.2805974 C126.453764,25.7180974 126.203764,26.5566391 126.203764,27.7962224 L126.203764,33.0774804 L124.563139,33.0774804 L124.563139,33.0774804 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="383 10.47244 466.653534 10.47244 466.653534 41.842511 383 41.842511"></polygon>
+                <path d="M395.8628,30.1868524 L397.4878,29.93686 C397.571133,30.5826933 397.821133,31.0826933 398.2378,31.43686 C398.654467,31.78061 399.2378,31.952485 399.9878,31.952485 C400.7378,31.952485 401.295092,31.8014433 401.659675,31.49936 C402.024258,31.1868524 402.20655,30.8222767 402.20655,30.40561 C402.20655,30.0410267 402.045092,29.74936 401.722175,29.53061 C401.503425,29.3847767 400.946133,29.202485 400.0503,28.983735 C398.841967,28.671235 398.003425,28.40561 397.534675,28.1868524 C397.076342,27.9681049 396.727383,27.666019 396.4878,27.2806024 C396.248217,26.8847691 396.128425,26.4472691 396.128425,25.9681024 C396.128425,25.5410191 396.227383,25.1451857 396.4253,24.7806024 C396.623217,24.4056024 396.89405,24.0983107 397.2378,23.8587274 C397.4878,23.6712274 397.83155,23.5149774 398.26905,23.3899774 C398.716967,23.2545607 399.190925,23.1868524 399.690925,23.1868524 C400.451342,23.1868524 401.118008,23.2962274 401.690925,23.5149774 C402.263842,23.7337274 402.685717,24.0306024 402.95655,24.4056024 C403.2378,24.7806024 403.430508,25.2806024 403.534675,25.9056024 L401.9253,26.1243524 C401.852383,25.6243524 401.638842,25.2337274 401.284675,24.9524774 C400.940925,24.6712274 400.451342,24.5306024 399.815925,24.5306024 C399.055508,24.5306024 398.513842,24.6556024 398.190925,24.9056024 C397.878425,25.1556024 397.722175,25.4472691 397.722175,25.7806024 C397.722175,25.9889357 397.784675,26.1816441 397.909675,26.3587274 C398.045092,26.5358107 398.258633,26.6816441 398.5503,26.7962274 C398.70655,26.8587274 399.185717,26.9993524 399.9878,27.2181024 C401.154467,27.5201857 401.966967,27.7701857 402.4253,27.9681024 C402.883633,28.1660242 403.243008,28.452485 403.503425,28.827485 C403.763842,29.202485 403.89405,29.671235 403.89405,30.233735 C403.89405,30.7858183 403.732592,31.3014433 403.409675,31.78061 C403.097175,32.2597767 402.638842,32.6347767 402.034675,32.90561 C401.430508,33.1660267 400.748217,33.296235 399.9878,33.296235 C398.7378,33.296235 397.779467,33.0358183 397.1128,32.514985 C396.45655,31.9941517 396.039883,31.21811 395.8628,30.1868524 Z M405.76905,36.796235 L405.58155,35.264985 C405.946133,35.358735 406.258633,35.405602 406.51905,35.405602 C406.883633,35.405602 407.1753,35.34311 407.39405,35.21811 C407.6128,35.1035267 407.795092,34.93686 407.940925,34.71811 C408.045092,34.5514433 408.211758,34.1451933 408.440925,33.49936 C408.472175,33.405602 408.51905,33.2701933 408.58155,33.09311 L404.909675,23.405602 L406.6753,23.405602 L408.690925,28.9993596 C408.951342,29.7181096 409.185717,30.4681096 409.39405,31.2493596 C409.58155,30.4993596 409.805508,29.7649846 410.065925,29.0462346 L412.14405,23.405602 L413.784675,23.405602 L410.097175,33.233735 C409.701342,34.3066517 409.39405,35.0410267 409.1753,35.43686 C408.883633,35.9785267 408.5503,36.37436 408.1753,36.62436 C407.8003,36.87436 407.352383,36.99936 406.83155,36.99936 C406.508633,36.99936 406.154467,36.9316517 405.76905,36.796235 Z M414.51905,30.1868524 L416.14405,29.93686 C416.227383,30.5826933 416.477383,31.0826933 416.89405,31.43686 C417.310717,31.78061 417.89405,31.952485 418.64405,31.952485 C419.39405,31.952485 419.951342,31.8014433 420.315925,31.49936 C420.680508,31.1868524 420.8628,30.8222767 420.8628,30.40561 C420.8628,30.0410267 420.701342,29.74936 420.378425,29.53061 C420.159675,29.3847767 419.602383,29.202485 418.70655,28.983735 C417.498217,28.671235 416.659675,28.40561 416.190925,28.1868524 C415.732592,27.9681049 415.383633,27.666019 415.14405,27.2806024 C414.904467,26.8847691 414.784675,26.4472691 414.784675,25.9681024 C414.784675,25.5410191 414.883633,25.1451857 415.08155,24.7806024 C415.279467,24.4056024 415.5503,24.0983107 415.89405,23.8587274 C416.14405,23.6712274 416.4878,23.5149774 416.9253,23.3899774 C417.373217,23.2545607 417.847175,23.1868524 418.347175,23.1868524 C419.107592,23.1868524 419.774258,23.2962274 420.347175,23.5149774 C420.920092,23.7337274 421.341967,24.0306024 421.6128,24.4056024 C421.89405,24.7806024 422.086758,25.2806024 422.190925,25.9056024 L420.58155,26.1243524 C420.508633,25.6243524 420.295092,25.2337274 419.940925,24.9524774 C419.597175,24.6712274 419.107592,24.5306024 418.472175,24.5306024 C417.711758,24.5306024 417.170092,24.6556024 416.847175,24.9056024 C416.534675,25.1556024 416.378425,25.4472691 416.378425,25.7806024 C416.378425,25.9889357 416.440925,26.1816441 416.565925,26.3587274 C416.701342,26.5358107 416.914883,26.6816441 417.20655,26.7962274 C417.3628,26.8587274 417.841967,26.9993524 418.64405,27.2181024 C419.810717,27.5201857 420.623217,27.7701857 421.08155,27.9681024 C421.539883,28.1660242 421.899258,28.452485 422.159675,28.827485 C422.420092,29.202485 422.5503,29.671235 422.5503,30.233735 C422.5503,30.7858183 422.388842,31.3014433 422.065925,31.78061 C421.753425,32.2597767 421.295092,32.6347767 420.690925,32.90561 C420.086758,33.1660267 419.404467,33.296235 418.64405,33.296235 C417.39405,33.296235 416.435717,33.0358183 415.76905,32.514985 C415.1128,31.9941517 414.696133,31.21811 414.51905,30.1868524 Z M428.08155,31.608735 L428.315925,33.06186 C427.857592,33.15561 427.446133,33.202485 427.08155,33.202485 C426.4878,33.202485 426.024258,33.108735 425.690925,32.921235 C425.368008,32.733735 425.138842,32.4889433 425.003425,32.18686 C424.868008,31.87436 424.8003,31.2285267 424.8003,30.24936 L424.8003,24.6712274 L423.597175,24.6712274 L423.597175,23.4056024 L424.8003,23.4056024 L424.8003,21.0149774 L426.4253,20.0306024 L426.4253,23.4056024 L428.08155,23.4056024 L428.08155,24.6712274 L426.4253,24.6712274 L426.4253,30.34311 C426.4253,30.8014433 426.451342,31.0983183 426.503425,31.233735 C426.565925,31.3691517 426.659675,31.4785267 426.784675,31.56186 C426.920092,31.6347767 427.1128,31.671235 427.3628,31.671235 C427.539883,31.671235 427.779467,31.6504017 428.08155,31.608735 Z M436.311744,29.96811 L437.999244,30.171235 C437.728411,31.1608183 437.233619,31.9316517 436.514869,32.483735 C435.796119,33.0254017 434.874244,33.296235 433.749244,33.296235 C432.332577,33.296235 431.207577,32.8639433 430.374244,31.99936 C429.551327,31.12436 429.139869,29.9004017 429.139869,28.327485 C429.139869,26.6920633 429.556536,25.4264357 430.389869,24.5306024 C431.233619,23.6347691 432.322161,23.1868524 433.655494,23.1868524 C434.947161,23.1868524 435.999244,23.6295607 436.811744,24.5149774 C437.634661,25.3899774 438.046119,26.6243549 438.046119,28.21811 C438.046119,28.3222767 438.046119,28.46811 438.046119,28.65561 L430.827369,28.65561 C430.889869,29.71811 431.191952,30.5358183 431.733619,31.108735 C432.275286,31.671235 432.947161,31.952485 433.749244,31.952485 C434.353411,31.952485 434.869036,31.796235 435.296119,31.483735 C435.723202,31.1608183 436.061744,30.65561 436.311744,29.96811 Z M430.921119,27.3118524 L436.327369,27.3118524 C436.254452,26.4993524 436.046119,25.8899774 435.702369,25.4837274 C435.181536,24.8483107 434.504452,24.5306024 433.671119,24.5306024 C432.921119,24.5306024 432.285702,24.7858107 431.764869,25.2962274 C431.254452,25.7962274 430.973202,26.4681024 430.921119,27.3118524 Z M440.062446,33.077485 L440.062446,23.405602 L441.531196,23.405602 L441.531196,24.764977 C441.833279,24.2858103 442.234321,23.905602 442.734321,23.624352 C443.244738,23.3326853 443.817654,23.186852 444.453071,23.186852 C445.171821,23.186852 445.760363,23.3378937 446.218696,23.639977 C446.677029,23.9316437 446.999946,24.343102 447.187446,24.874352 C447.958279,23.749352 448.953071,23.186852 450.171821,23.186852 C451.140571,23.186852 451.880154,23.452477 452.390571,23.983727 C452.911404,24.514977 453.171821,25.3326853 453.171821,26.436852 L453.171821,33.077485 L451.531196,33.077485 L451.531196,26.983727 C451.531196,26.327477 451.479113,25.858727 451.374946,25.577477 C451.270779,25.2858103 451.078071,25.0514353 450.796821,24.874352 C450.515571,24.6972687 450.187446,24.608727 449.812446,24.608727 C449.135363,24.608727 448.572863,24.8378937 448.124946,25.296227 C447.677029,25.7441437 447.453071,26.4628937 447.453071,27.452477 L447.453071,33.077485 L445.812446,33.077485 L445.812446,26.796227 C445.812446,26.0670603 445.677029,25.5201853 445.406196,25.155602 C445.135363,24.7910187 444.697863,24.608727 444.093696,24.608727 C443.635363,24.608727 443.208279,24.7285187 442.812446,24.968102 C442.416613,25.2076853 442.130154,25.561852 441.953071,26.030602 C441.786404,26.499352 441.703071,27.1764379 441.703071,28.0618596 L441.703071,33.077485 L440.062446,33.077485 L440.062446,33.077485 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M607.7156,4.1045 L607.7156,4.1045 C607.7156,2.5803148 608.951196,1.344719 610.475366,1.344719 L652.325891,1.344719 C653.057824,1.344719 653.75979,1.63547523 654.277368,2.1530381 C654.794946,2.67060097 655.085717,3.3725587 655.085717,4.1044999 L655.085717,48.6636799 C655.085717,50.1878651 653.850122,51.4234609 652.325891,51.4234609 L610.475366,51.4234609 C608.951196,51.4234609 607.7156,50.1878652 607.7156,48.6636799 L607.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M607.7156,4.1045 L607.7156,4.1045 C607.7156,2.5803148 608.951196,1.344719 610.475366,1.344719 L652.325891,1.344719 C653.057824,1.344719 653.75979,1.63547523 654.277368,2.1530381 C654.794946,2.67060097 655.085717,3.3725587 655.085717,4.1044999 L655.085717,48.6636799 C655.085717,50.1878651 653.850122,51.4234609 652.325891,51.4234609 L610.475366,51.4234609 C608.951196,51.4234609 607.7156,50.1878652 607.7156,48.6636799 L607.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M632.244,33.3041 L632.244,30.100975 L626.447125,30.100975 L626.447125,28.600975 L632.540875,19.944717 L633.884625,19.944717 L633.884625,28.600975 L635.6815,28.600975 L635.6815,30.100975 L633.884625,30.100975 L633.884625,33.3041 L632.244,33.3041 Z M632.244,28.600975 L632.244,22.5853424 L628.0565,28.600975 L632.244,28.600975 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="527.0889 16.85302 546.521944 16.85302 546.521944 38.679795 527.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="599.0889 16.85302 618.521944 16.85302 618.521944 38.679795 599.0889 38.679795"></polygon>
+                <line x1="618.522" y1="27.7664" x2="533.1362" y2="28.7427794" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="600.5232" y1="27.97221" x2="533.136176" y2="28.7427789" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="600.57983 32.9271 614.136532 27.8165577 600.466488 23.0173557"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.8793 12.06562 588.603726 12.06562 588.603726 43.435699 543.8793 43.435699"></polygon>
+                <path d="M560.7394,23.201904 L560.7394,21.311279 L562.380025,21.311279 L562.380025,23.201904 L560.7394,23.201904 Z M560.7394,34.670654 L560.7394,24.998779 L562.380025,24.998779 L562.380025,34.670654 L560.7394,34.670654 Z M564.227986,31.780029 L565.852986,31.530029 C565.93632,32.1758623 566.18632,32.6758623 566.602986,33.030029 C567.019653,33.373779 567.602986,33.545654 568.352986,33.545654 C569.102986,33.545654 569.660278,33.3946123 570.024861,33.092529 C570.389445,32.780029 570.571736,32.4154457 570.571736,31.998779 C570.571736,31.6341957 570.410278,31.342529 570.087361,31.123779 C569.868611,30.9779457 569.31132,30.795654 568.415486,30.576904 C567.207153,30.264404 566.368611,29.998779 565.899861,29.780029 C565.441528,29.561279 565.09257,29.2591957 564.852986,28.873779 C564.613403,28.4779457 564.493611,28.0404457 564.493611,27.561279 C564.493611,27.1341957 564.59257,26.7383623 564.790486,26.373779 C564.988403,25.998779 565.259236,25.6914873 565.602986,25.451904 C565.852986,25.264404 566.196736,25.108154 566.634236,24.983154 C567.082153,24.8477373 567.556111,24.780029 568.056111,24.780029 C568.816528,24.780029 569.483195,24.889404 570.056111,25.108154 C570.629028,25.326904 571.050903,25.623779 571.321736,25.998779 C571.602986,26.373779 571.795695,26.873779 571.899861,27.498779 L570.290486,27.717529 C570.21757,27.217529 570.004028,26.826904 569.649861,26.545654 C569.306111,26.264404 568.816528,26.123779 568.181111,26.123779 C567.420695,26.123779 566.879028,26.248779 566.556111,26.498779 C566.243611,26.748779 566.087361,27.0404457 566.087361,27.373779 C566.087361,27.5821123 566.149861,27.7748207 566.274861,27.951904 C566.410278,28.1289873 566.62382,28.2748207 566.915486,28.389404 C567.071736,28.451904 567.550903,28.592529 568.352986,28.811279 C569.519653,29.1133623 570.332153,29.3633623 570.790486,29.561279 C571.24882,29.7591957 571.608195,30.045654 571.868611,30.420654 C572.129028,30.795654 572.259236,31.264404 572.259236,31.826904 C572.259236,32.3789873 572.097778,32.8946123 571.774861,33.373779 C571.462361,33.8529457 571.004028,34.2279457 570.399861,34.498779 C569.795695,34.7591957 569.113403,34.889404 568.352986,34.889404 C567.102986,34.889404 566.144653,34.6289873 565.477986,34.108154 C564.821736,33.5873207 564.40507,32.811279 564.227986,31.780029 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M759.7156,4.1045 L759.7156,4.1045 C759.7156,2.5803148 760.951196,1.344719 762.475366,1.344719 L804.325891,1.344719 C805.057824,1.344719 805.75979,1.63547523 806.277368,2.1530381 C806.794946,2.67060097 807.085717,3.3725587 807.085717,4.1044999 L807.085717,48.6636799 C807.085717,50.1878651 805.850122,51.4234609 804.325891,51.4234609 L762.475366,51.4234609 C760.951196,51.4234609 759.7156,50.1878652 759.7156,48.6636799 L759.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M759.7156,4.1045 L759.7156,4.1045 C759.7156,2.5803148 760.951196,1.344719 762.475366,1.344719 L804.325891,1.344719 C805.057824,1.344719 805.75979,1.63547523 806.277368,2.1530381 C806.794946,2.67060097 807.085717,3.3725587 807.085717,4.1044999 L807.085717,48.6636799 C807.085717,50.1878651 805.850122,51.4234609 804.325891,51.4234609 L762.475366,51.4234609 C760.951196,51.4234609 759.7156,50.1878652 759.7156,48.6636799 L759.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M778.994,29.8041 L780.71275,29.663475 C780.83775,30.4968083 781.129417,31.1270167 781.58775,31.5541 C782.0565,31.9707667 782.619,32.1791 783.27525,32.1791 C784.066917,32.1791 784.733583,31.882225 785.27525,31.288475 C785.827333,30.694725 786.103375,29.9082667 786.103375,28.9291 C786.103375,27.9915949 785.83775,27.2520091 785.3065,26.7103424 C784.785667,26.1686757 784.098167,25.8978424 783.244,25.8978424 C782.723167,25.8978424 782.249208,26.0176341 781.822125,26.2572174 C781.395042,26.4968007 781.061708,26.8093007 780.822125,27.1947174 L779.27525,26.9915924 L780.572125,20.1322174 L787.21275,20.1322174 L787.21275,21.6947174 L781.884625,21.6947174 L781.165875,25.2884674 C781.967958,24.7259674 782.8065,24.4447174 783.6815,24.4447174 C784.848167,24.4447174 785.832542,24.8509674 786.634625,25.6634674 C787.436708,26.4655507 787.83775,27.5020116 787.83775,28.77285 C787.83775,29.9811833 787.488792,31.0280583 786.790875,31.913475 C785.926292,32.9968083 784.754417,33.538475 783.27525,33.538475 C782.066917,33.538475 781.077333,33.1999333 780.3065,32.52285 C779.535667,31.83535 779.098167,30.9291 778.994,29.8041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="647.0889 16.85302 666.521944 16.85302 666.521944 38.679795 647.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="751.0889 16.85302 770.521944 16.85302 770.521944 38.679795 751.0889 38.679795"></polygon>
+                <line x1="770.522" y1="27.7664" x2="657.38815" y2="29.5616728" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="752.5243" y1="28.052" x2="657.38813" y2="29.5616741" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="752.60284 33.00658 766.136837 27.8359944 752.445614 23.0974309"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="669.7087 10.47244 740.606344 10.47244 740.606344 41.842511 669.7087 41.842511"></polygon>
+                <path d="M681.3761,33.07748 L681.3761,24.671222 L679.922975,24.671222 L679.922975,23.405597 L681.3761,23.405597 L681.3761,22.374347 C681.3761,21.7285137 681.433392,21.2441387 681.547975,20.921222 C681.704225,20.4941387 681.980267,20.1503887 682.3761,19.889972 C682.771933,19.6295553 683.329225,19.499347 684.047975,19.499347 C684.506308,19.499347 685.016725,19.5514303 685.579225,19.655597 L685.329225,21.093097 C684.995892,21.030597 684.678183,20.999347 684.3761,20.999347 C683.8761,20.999347 683.521933,21.108722 683.3136,21.327472 C683.105267,21.5358053 683.0011,21.9316387 683.0011,22.514972 L683.0011,23.405597 L684.891725,23.405597 L684.891725,24.671222 L683.0011,24.671222 L683.0011,33.07748 L681.3761,33.07748 Z M686.153139,33.07748 L686.153139,23.405597 L687.621889,23.405597 L687.621889,24.874347 C687.996889,24.186847 688.340639,23.733722 688.653139,23.514972 C688.976055,23.296222 689.330222,23.186847 689.715639,23.186847 C690.267722,23.186847 690.830222,23.3639303 691.403139,23.718097 L690.840639,25.233722 C690.434389,24.9941387 690.033347,24.874347 689.637514,24.874347 C689.27293,24.874347 688.950014,24.983722 688.668764,25.202472 C688.387514,25.421222 688.184389,25.718097 688.059389,26.093097 C687.871889,26.6764303 687.778139,27.3170579 687.778139,28.0149796 L687.778139,33.07748 L686.153139,33.07748 L686.153139,33.07748 Z M698.693971,31.88998 C698.079388,32.4003967 697.490846,32.76498 696.928346,32.98373 C696.376263,33.1920633 695.777304,33.29623 695.131471,33.29623 C694.068971,33.29623 693.251263,33.0358133 692.678346,32.51498 C692.105429,31.9941467 691.818971,31.3326883 691.818971,30.530605 C691.818971,30.0514383 691.928346,29.6191467 692.147096,29.23373 C692.365846,28.8378967 692.647096,28.5253967 692.990846,28.29623 C693.345013,28.0566416 693.740846,27.8743474 694.178346,27.7493474 C694.490846,27.6660141 694.975221,27.5826807 695.631471,27.4993474 C696.954388,27.3430974 697.928346,27.1555974 698.553346,26.9368474 C698.563763,26.7076807 698.568971,26.5670557 698.568971,26.5149724 C698.568971,25.8483057 698.412721,25.3743474 698.100221,25.0930974 C697.683554,24.7285141 697.058554,24.5462224 696.225221,24.5462224 C695.454388,24.5462224 694.886679,24.6816391 694.522096,24.9524724 C694.157513,25.2233057 693.886679,25.6972641 693.709596,26.3743474 L692.100221,26.1555974 C692.246054,25.4785141 692.485638,24.9316391 692.818971,24.5149724 C693.152304,24.0878891 693.636679,23.7597641 694.272096,23.5305974 C694.907513,23.3014307 695.636679,23.1868474 696.459596,23.1868474 C697.292929,23.1868474 697.964804,23.2858057 698.475221,23.4837224 C698.996054,23.6712224 699.376263,23.9160141 699.615846,24.2180974 C699.865846,24.5097641 700.037721,24.8795557 700.131471,25.3274724 C700.183554,25.6087224 700.209596,26.1139307 700.209596,26.8430974 L700.209596,29.030605 C700.209596,30.5514383 700.246054,31.51498 700.318971,31.92123 C700.391888,32.3170633 700.527304,32.70248 700.725221,33.07748 L699.022096,33.07748 C698.845013,32.73373 698.735638,32.3378967 698.693971,31.88998 Z M698.553346,28.218105 C697.959596,28.468105 697.068971,28.6764383 695.881471,28.843105 C695.204388,28.936855 694.725221,29.04623 694.443971,29.17123 C694.162721,29.29623 693.943971,29.4733133 693.787721,29.70248 C693.641888,29.9316467 693.568971,30.1920633 693.568971,30.48373 C693.568971,30.92123 693.735638,31.2858133 694.068971,31.57748 C694.402304,31.8691467 694.886679,32.01498 695.522096,32.01498 C696.147096,32.01498 696.704388,31.8795633 697.193971,31.60873 C697.693971,31.32748 698.058554,30.9472717 698.287721,30.468105 C698.464804,30.093105 698.553346,29.54623 698.553346,28.82748 L698.553346,28.218105 Z M702.444642,33.874355 L704.038392,34.10873 C704.111309,34.60873 704.298809,34.968105 704.600892,35.1868474 C705.007142,35.4889383 705.564434,35.63998 706.272767,35.63998 C707.033184,35.63998 707.616517,35.4889383 708.022767,35.1868474 C708.439434,34.8847717 708.720684,34.4628967 708.866517,33.92123 C708.949851,33.5878967 708.986309,32.8847717 708.975892,31.811855 C708.267559,32.655605 707.376934,33.07748 706.304017,33.07748 C704.970684,33.07748 703.939434,32.5983133 703.210267,31.63998 C702.481101,30.6816467 702.116517,29.530605 702.116517,28.1868474 C702.116517,27.2597666 702.283184,26.4055974 702.616517,25.6243474 C702.960267,24.8430974 703.444642,24.2441391 704.069642,23.8274724 C704.705059,23.4003891 705.455059,23.1868474 706.319642,23.1868474 C707.455059,23.1868474 708.392559,23.6451807 709.132142,24.5618474 L709.132142,23.4055974 L710.647767,23.4055974 L710.647767,31.7649804 C710.647767,33.2753971 710.491517,34.3431054 710.179017,34.9681054 C709.876934,35.5931054 709.392559,36.0878971 708.725892,36.4524804 C708.069642,36.8170637 707.251934,36.9993554 706.272767,36.9993554 C705.126934,36.9993554 704.194642,36.7389387 703.475892,36.2181054 C702.767559,35.7076887 702.423809,34.9264387 702.444642,33.8743554 L702.444642,33.874355 Z M703.804017,28.061855 C703.804017,29.3326883 704.054017,30.2597717 704.554017,30.843105 C705.064434,31.4264383 705.699851,31.718105 706.460267,31.718105 C707.210267,31.718105 707.840476,31.4316467 708.350892,30.85873 C708.861309,30.2753967 709.116517,29.3639383 709.116517,28.124355 C709.116517,26.9368499 708.850892,26.0462224 708.319642,25.4524724 C707.798809,24.8483057 707.168601,24.5462224 706.429017,24.5462224 C705.699851,24.5462224 705.080059,24.8430974 704.569642,25.4368474 C704.059226,26.0201807 703.804017,26.8951833 703.804017,28.061855 Z M713.132874,21.6087224 L713.132874,19.7180974 L714.773499,19.7180974 L714.773499,21.6087224 L713.132874,21.6087224 Z M713.132874,33.0774804 L713.132874,23.4055974 L714.773499,23.4055974 L714.773499,33.0774804 L713.132874,33.0774804 Z M717.230836,33.0774804 L717.230836,19.7180974 L718.871461,19.7180974 L718.871461,33.0774804 L717.230836,33.0774804 Z M728.047547,29.9681054 L729.735047,30.1712304 C729.464214,31.1608137 728.969422,31.9316471 728.250672,32.4837304 C727.531922,33.0253971 726.610047,33.2962304 725.485047,33.2962304 C724.06838,33.2962304 722.94338,32.8639387 722.110047,31.9993554 C721.28713,31.1243554 720.875672,29.9003971 720.875672,28.3274804 C720.875672,26.6920587 721.292339,25.4264311 722.125672,24.5305978 C722.969422,23.6347645 724.057964,23.1868478 725.391297,23.1868478 C726.682964,23.1868478 727.735047,23.6295561 728.547547,24.5149728 C729.370464,25.3899728 729.781922,26.6243503 729.781922,28.2181054 C729.781922,28.3222721 729.781922,28.4681054 729.781922,28.6556054 L722.563172,28.6556054 C722.625672,29.7181054 722.927755,30.5358137 723.469422,31.1087304 C724.011089,31.6712304 724.682964,31.9524804 725.485047,31.9524804 C726.089214,31.9524804 726.604839,31.7962304 727.031922,31.4837304 C727.459005,31.1608137 727.797547,30.6556054 728.047547,29.9681054 Z M722.656922,27.3118478 L728.063172,27.3118478 C727.990255,26.4993478 727.781922,25.8899728 727.438172,25.4837228 C726.917339,24.8483061 726.240255,24.5305978 725.406922,24.5305978 C724.656922,24.5305978 724.021505,24.7858061 723.500672,25.2962228 C722.990255,25.7962228 722.709005,26.4680978 722.656922,27.3118478 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <line x1="330.52197" y1="27.7664" x2="217.45111" y2="27.7664" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="312.52197" y1="27.7664" x2="217.45111" y2="27.7664" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="312.52197 32.7216 326.136258 27.7664 312.52197 22.8112074"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="229.4882 10.47244 300.385844 10.47244 300.385844 41.842511 229.4882 41.842511"></polygon>
+                <path d="M242.8374,33.07748 L242.8374,23.405597 L244.30615,23.405597 L244.30615,24.780597 C245.014483,23.718097 246.040525,23.186847 247.384275,23.186847 C247.967608,23.186847 248.504067,23.2910137 248.99365,23.499347 C249.483233,23.7076803 249.847817,23.983722 250.0874,24.327472 C250.3374,24.6608053 250.509275,25.061847 250.603025,25.530597 C250.665525,25.8326803 250.696775,26.3639303 250.696775,27.124347 L250.696775,33.07748 L249.05615,33.07748 L249.05615,27.186847 C249.05615,26.5201803 248.988442,26.0253887 248.853025,25.702472 C248.728025,25.3691387 248.504067,25.1035137 248.18115,24.905597 C247.858233,24.7076803 247.478025,24.608722 247.040525,24.608722 C246.342608,24.608722 245.738442,24.8326803 245.228025,25.280597 C244.728025,25.718097 244.478025,26.5566387 244.478025,27.796222 L244.478025,33.07748 L242.8374,33.07748 L242.8374,33.07748 Z M259.525632,31.88998 C258.911049,32.4003967 258.322507,32.76498 257.760007,32.98373 C257.207924,33.1920633 256.608965,33.29623 255.963132,33.29623 C254.900632,33.29623 254.082924,33.0358133 253.510007,32.51498 C252.93709,31.9941467 252.650632,31.3326883 252.650632,30.530605 C252.650632,30.0514383 252.760007,29.6191467 252.978757,29.23373 C253.197507,28.8378967 253.478757,28.5253967 253.822507,28.29623 C254.176674,28.0566416 254.572507,27.8743474 255.010007,27.7493474 C255.322507,27.6660141 255.806882,27.5826807 256.463132,27.4993474 C257.786049,27.3430974 258.760007,27.1555974 259.385007,26.9368474 C259.395424,26.7076807 259.400632,26.5670557 259.400632,26.5149724 C259.400632,25.8483057 259.244382,25.3743474 258.931882,25.0930974 C258.515215,24.7285141 257.890215,24.5462224 257.056882,24.5462224 C256.286049,24.5462224 255.71834,24.6816391 255.353757,24.9524724 C254.989174,25.2233057 254.71834,25.6972641 254.541257,26.3743474 L252.931882,26.1555974 C253.077715,25.4785141 253.317299,24.9316391 253.650632,24.5149724 C253.983965,24.0878891 254.46834,23.7597641 255.103757,23.5305974 C255.739174,23.3014307 256.46834,23.1868474 257.291257,23.1868474 C258.12459,23.1868474 258.796465,23.2858057 259.306882,23.4837224 C259.827715,23.6712224 260.207924,23.9160141 260.447507,24.2180974 C260.697507,24.5097641 260.869382,24.8795557 260.963132,25.3274724 C261.015215,25.6087224 261.041257,26.1139307 261.041257,26.8430974 L261.041257,29.030605 C261.041257,30.5514383 261.077715,31.51498 261.150632,31.92123 C261.223549,32.3170633 261.358965,32.70248 261.556882,33.07748 L259.853757,33.07748 C259.676674,32.73373 259.567299,32.3378967 259.525632,31.88998 Z M259.385007,28.218105 C258.791257,28.468105 257.900632,28.6764383 256.713132,28.843105 C256.036049,28.936855 255.556882,29.04623 255.275632,29.17123 C254.994382,29.29623 254.775632,29.4733133 254.619382,29.70248 C254.473549,29.9316467 254.400632,30.1920633 254.400632,30.48373 C254.400632,30.92123 254.567299,31.2858133 254.900632,31.57748 C255.233965,31.8691467 255.71834,32.01498 256.353757,32.01498 C256.978757,32.01498 257.536049,31.8795633 258.025632,31.60873 C258.525632,31.32748 258.890215,30.9472717 259.119382,30.468105 C259.296465,30.093105 259.385007,29.54623 259.385007,28.82748 L259.385007,28.218105 Z M263.588834,33.07748 L263.588834,23.405597 L265.057584,23.405597 L265.057584,24.764972 C265.359667,24.2858053 265.760709,23.905597 266.260709,23.624347 C266.771126,23.3326803 267.344042,23.186847 267.979459,23.186847 C268.698209,23.186847 269.286751,23.3378887 269.745084,23.639972 C270.203417,23.9316387 270.526334,24.343097 270.713834,24.874347 C271.484667,23.749347 272.479459,23.186847 273.698209,23.186847 C274.666959,23.186847 275.406542,23.452472 275.916959,23.983722 C276.437792,24.514972 276.698209,25.3326803 276.698209,26.436847 L276.698209,33.07748 L275.057584,33.07748 L275.057584,26.983722 C275.057584,26.327472 275.005501,25.858722 274.901334,25.577472 C274.797167,25.2858053 274.604459,25.0514303 274.323209,24.874347 C274.041959,24.6972637 273.713834,24.608722 273.338834,24.608722 C272.661751,24.608722 272.099251,24.8378887 271.651334,25.296222 C271.203417,25.7441387 270.979459,26.4628887 270.979459,27.452472 L270.979459,33.07748 L269.338834,33.07748 L269.338834,26.796222 C269.338834,26.0670553 269.203417,25.5201803 268.932584,25.155597 C268.661751,24.7910137 268.224251,24.608722 267.620084,24.608722 C267.161751,24.608722 266.734667,24.7285137 266.338834,24.968097 C265.943001,25.2076803 265.656542,25.561847 265.479459,26.030597 C265.312792,26.499347 265.229459,27.1764329 265.229459,28.0618546 L265.229459,33.07748 L263.588834,33.07748 L263.588834,33.07748 Z M285.754636,29.968105 L287.442136,30.17123 C287.171303,31.1608133 286.676511,31.9316467 285.957761,32.48373 C285.239011,33.0253967 284.317136,33.29623 283.192136,33.29623 C281.775469,33.29623 280.650469,32.8639383 279.817136,31.999355 C278.994219,31.124355 278.582761,29.9003967 278.582761,28.32748 C278.582761,26.6920583 278.999428,25.4264307 279.832761,24.5305974 C280.676511,23.6347641 281.765053,23.1868474 283.098386,23.1868474 C284.390053,23.1868474 285.442136,23.6295557 286.254636,24.5149724 C287.077553,25.3899724 287.489011,26.6243499 287.489011,28.218105 C287.489011,28.3222717 287.489011,28.468105 287.489011,28.655605 L280.270261,28.655605 C280.332761,29.718105 280.634844,30.5358133 281.176511,31.10873 C281.718178,31.67123 282.390053,31.95248 283.192136,31.95248 C283.796303,31.95248 284.311928,31.79623 284.739011,31.48373 C285.166094,31.1608133 285.504636,30.655605 285.754636,29.968105 Z M280.364011,27.3118474 L285.770261,27.3118474 C285.697344,26.4993474 285.489011,25.8899724 285.145261,25.4837224 C284.624428,24.8483057 283.947344,24.5305974 283.114011,24.5305974 C282.364011,24.5305974 281.728594,24.7858057 281.207761,25.2962224 C280.697344,25.7962224 280.416094,26.4680974 280.364011,27.3118474 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-dns-invalid-ex.svg b/docs/reference/images/analysis/token-graph-dns-invalid-ex.svg
new file mode 100644
index 0000000000000..5614f39bfe35c
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-dns-invalid-ex.svg
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="810px" height="127px" viewBox="0 0 810 127" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="token-graph-dns-invalid-ex" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M167.71559,4.1045 L167.71559,4.1045 C167.71559,2.5803148 168.951186,1.344719 170.475371,1.344719 L212.325881,1.344719 C213.05783,1.344719 213.759795,1.63547523 214.277343,2.1530381 C214.794906,2.67060097 215.085662,3.3725587 215.085662,4.1044999 L215.085662,48.6636799 C215.085662,50.1878651 213.850066,51.4234609 212.325881,51.4234609 L170.475371,51.4234609 C168.951186,51.4234609 167.71559,50.1878652 167.71559,48.6636799 L167.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M167.71559,4.1045 L167.71559,4.1045 C167.71559,2.5803148 168.951186,1.344719 170.475371,1.344719 L212.325881,1.344719 C213.05783,1.344719 213.759795,1.63547523 214.277343,2.1530381 C214.794906,2.67060097 215.085662,3.3725587 215.085662,4.1044999 L215.085662,48.6636799 C215.085662,50.1878651 213.850066,51.4234609 212.325881,51.4234609 L170.475371,51.4234609 C168.951186,51.4234609 167.71559,50.1878652 167.71559,48.6636799 L167.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M193.1659,33.3041 L191.525275,33.3041 L191.525275,22.850967 C191.129442,23.225967 190.608608,23.6061753 189.962775,23.991592 C189.327358,24.366592 188.754442,24.647842 188.244025,24.835342 L188.244025,23.241592 C189.160692,22.8145087 189.962775,22.2936753 190.650275,21.679092 C191.337775,21.0645087 191.82215,20.4707587 192.1034,19.897842 L193.1659,19.897842 L193.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="159.08891 16.85302 178.521985 16.85302 178.521985 38.679795 159.08891 38.679795"></polygon>
+                <line x1="178.52199" y1="27.7664" x2="48.19127" y2="27.2309691" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="160.52213" y1="27.69245" x2="48.19128" y2="27.2309708" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="160.50177 32.64761 174.136307 27.7483867 160.542496 22.7372937"></polygon>
+                <path d="M319.71558,4.1045 L319.71558,4.1045 C319.71558,2.5803148 320.951206,1.344719 322.475376,1.344719 L364.325901,1.344719 C365.057835,1.344719 365.7598,1.63547523 366.277348,2.1530381 C366.794926,2.67060097 367.085667,3.3725587 367.085667,4.1044999 L367.085667,48.6636799 C367.085667,50.1878651 365.850071,51.4234609 364.325901,51.4234609 L322.475376,51.4234609 C320.951206,51.4234609 319.71558,50.1878652 319.71558,48.6636799 L319.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M319.71558,4.1045 L319.71558,4.1045 C319.71558,2.5803148 320.951206,1.344719 322.475376,1.344719 L364.325901,1.344719 C365.057835,1.344719 365.7598,1.63547523 366.277348,2.1530381 C366.794926,2.67060097 367.085667,3.3725587 367.085667,4.1044999 L367.085667,48.6636799 C367.085667,50.1878651 365.850071,51.4234609 364.325901,51.4234609 L322.475376,51.4234609 C320.951206,51.4234609 319.71558,50.1878652 319.71558,48.6636799 L319.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M347.6034,31.72597 L347.6034,33.304095 L338.775275,33.304095 C338.764858,32.9082617 338.827358,32.5280533 338.962775,32.16347 C339.191942,31.5593033 339.551317,30.9655533 340.0409,30.38222 C340.5409,29.7988867 341.25965,29.1270117 342.19715,28.366595 C343.645067,27.1790899 344.624233,26.2363791 345.13465,25.5384624 C345.645067,24.8405457 345.900275,24.1842957 345.900275,23.5697124 C345.900275,22.9134624 345.6659,22.3665874 345.19715,21.9290874 C344.738817,21.4811707 344.13465,21.2572124 343.38465,21.2572124 C342.592983,21.2572124 341.957567,21.4967957 341.4784,21.9759624 C341.00965,22.4447124 340.775275,23.0957541 340.775275,23.9290874 L339.087775,23.7572124 C339.202358,22.4967957 339.63465,21.5384624 340.38465,20.8822124 C341.145067,20.2259624 342.155483,19.8978374 343.4159,19.8978374 C344.69715,19.8978374 345.712775,20.2520041 346.462775,20.9603374 C347.212775,21.6686707 347.587775,22.5488791 347.587775,23.6009624 C347.587775,24.1322124 347.4784,24.6582541 347.25965,25.1790874 C347.0409,25.6999207 346.676317,26.2467957 346.1659,26.8197124 C345.6659,27.3822124 344.82215,28.1634649 343.63465,29.16347 C342.655483,29.9863867 342.025275,30.5488867 341.744025,30.85097 C341.462775,31.1426367 341.233608,31.4343033 341.056525,31.72597 L347.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="207.08891 16.85302 226.52197 16.85302 226.52197 38.679795 207.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="311.0889 16.85302 330.521975 16.85302 330.521975 38.679795 311.0889 38.679795"></polygon>
+                <path d="M487.7156,4.1045 L487.7156,4.1045 C487.7156,2.5803148 488.951196,1.344719 490.475366,1.344719 L532.325891,1.344719 C533.057824,1.344719 533.75979,1.63547523 534.277368,2.1530381 C534.794946,2.67060097 535.085717,3.3725587 535.085717,4.1044999 L535.085717,48.6636799 C535.085717,50.1878651 533.850122,51.4234609 532.325891,51.4234609 L490.475366,51.4234609 C488.951196,51.4234609 487.7156,50.1878652 487.7156,48.6636799 L487.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M487.7156,4.1045 L487.7156,4.1045 C487.7156,2.5803148 488.951196,1.344719 490.475366,1.344719 L532.325891,1.344719 C533.057824,1.344719 533.75979,1.63547523 534.277368,2.1530381 C534.794946,2.67060097 535.085717,3.3725587 535.085717,4.1044999 L535.085717,48.6636799 C535.085717,50.1878651 533.850122,51.4234609 532.325891,51.4234609 L490.475366,51.4234609 C488.951196,51.4234609 487.7156,50.1878652 487.7156,48.6636799 L487.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M506.994,29.77284 L508.634625,29.55409 C508.822125,30.49159 509.139833,31.163465 509.58775,31.569715 C510.046083,31.975965 510.598167,32.17909 511.244,32.17909 C512.014833,32.17909 512.665875,31.913465 513.197125,31.382215 C513.728375,30.850965 513.994,30.1895067 513.994,29.3978324 C513.994,28.64784 513.749208,28.0280458 513.259625,27.5384574 C512.770042,27.0488741 512.145042,26.8040824 511.384625,26.8040824 C511.072125,26.8040824 510.686708,26.8613741 510.228375,26.9759574 L510.415875,25.5384574 C510.520042,25.5488741 510.608583,25.5540824 510.6815,25.5540824 C511.379417,25.5540824 512.004417,25.3717907 512.5565,25.0072074 C513.119,24.6426241 513.40025,24.0853324 513.40025,23.3353324 C513.40025,22.7311657 513.197125,22.2311657 512.790875,21.8353324 C512.384625,21.4394991 511.858583,21.2415824 511.21275,21.2415824 C510.577333,21.2415824 510.046083,21.4447074 509.619,21.8509574 C509.191917,22.2467907 508.921083,22.8457491 508.8065,23.6478324 L507.165875,23.3509574 C507.363792,22.2572074 507.816917,21.4082491 508.52525,20.8040824 C509.233583,20.1999157 510.119,19.8978324 511.1815,19.8978324 C511.910667,19.8978324 512.577333,20.0540824 513.1815,20.3665824 C513.796083,20.6790824 514.264833,21.1061657 514.58775,21.6478324 C514.921083,22.1894991 515.08775,22.7624157 515.08775,23.3665824 C515.08775,23.9394991 514.9315,24.4655407 514.619,24.9447074 C514.3065,25.4134574 513.848167,25.7884574 513.244,26.0697074 C514.035667,26.2572074 514.65025,26.6374157 515.08775,27.2103324 C515.52525,27.7832541 515.744,28.5020067 515.744,29.36659 C515.744,30.52284 515.316917,31.507215 514.46275,32.319715 C513.619,33.132215 512.546083,33.538465 511.244,33.538465 C510.066917,33.538465 509.092958,33.1895067 508.322125,32.49159 C507.551292,31.7936733 507.108583,30.8874233 506.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="479.0889 16.85302 498.521944 16.85302 498.521944 38.679795 479.0889 38.679795"></polygon>
+                <line x1="496.9606" y1="27.00144" x2="369.18106" y2="28.765211" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="478.9623" y1="27.24987" x2="369.18105" y2="28.7652127" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="479.03064 32.20459 492.575257 27.061966 478.89386 22.295143"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="58.385826 10.47244 147.83464 10.47244 147.83464 41.842511 58.385826 41.842511"></polygon>
+                <path d="M80.01599,33.07748 L80.01599,31.85873 C79.4118233,32.8170633 78.5107817,33.29623 77.312865,33.29623 C76.5420317,33.29623 75.8336983,33.0826883 75.187865,32.655605 C74.5420317,32.2285217 74.0420317,31.6347717 73.687865,30.874355 C73.3336983,30.1139383 73.156615,29.2389383 73.156615,28.249355 C73.156615,27.2805999 73.3180733,26.4055974 73.64099,25.6243474 C73.9639067,24.8326807 74.4430733,24.2285141 75.07849,23.8118474 C75.7243233,23.3951807 76.4482817,23.1868474 77.250365,23.1868474 C77.8336983,23.1868474 78.3493233,23.3118474 78.79724,23.5618474 C79.2555733,23.8014307 79.625365,24.1191391 79.906615,24.5149724 L79.906615,19.7180974 L81.54724,19.7180974 L81.54724,33.0774804 L80.01599,33.0774804 L80.01599,33.07748 Z M74.844115,28.249355 C74.844115,29.4889383 75.1045317,30.4160217 75.625365,31.030605 C76.1461983,31.6451883 76.7607817,31.95248 77.469115,31.95248 C78.187865,31.95248 78.79724,31.6608133 79.29724,31.07748 C79.79724,30.48373 80.04724,29.5878967 80.04724,28.38998 C80.04724,27.0670583 79.7920317,26.0983057 79.281615,25.4837224 C78.7711983,24.8587224 78.14099,24.5462224 77.39099,24.5462224 C76.67224,24.5462224 76.0680733,24.8430974 75.57849,25.4368474 C75.0889067,26.0305974 74.844115,26.9680999 74.844115,28.249355 Z M83.516707,28.23373 C83.516707,26.4420583 84.0114987,25.1191391 85.001082,24.2649724 C85.8344153,23.5462224 86.8500403,23.1868474 88.047957,23.1868474 C89.3812903,23.1868474 90.469832,23.6243474 91.313582,24.4993474 C92.157332,25.3639307 92.579207,26.5670583 92.579207,28.10873 C92.579207,29.3483133 92.391707,30.32748 92.016707,31.04623 C91.641707,31.7545633 91.094832,32.3066467 90.376082,32.70248 C89.6677487,33.0983133 88.891707,33.29623 88.047957,33.29623 C86.6937903,33.29623 85.6000403,32.8639383 84.766707,31.999355 C83.9333737,31.124355 83.516707,29.8691467 83.516707,28.23373 Z M85.204207,28.23373 C85.204207,29.4733133 85.469832,30.405605 86.001082,31.030605 C86.5427487,31.6451883 87.2250403,31.95248 88.047957,31.95248 C88.860457,31.95248 89.5375403,31.6451883 90.079207,31.030605 C90.6208737,30.405605 90.891707,29.4576883 90.891707,28.186855 C90.891707,26.9889333 90.6208737,26.0826807 90.079207,25.4680974 C89.5375403,24.8535141 88.860457,24.5462224 88.047957,24.5462224 C87.2250403,24.5462224 86.5427487,24.8535141 86.001082,25.4680974 C85.469832,26.0722641 85.204207,26.9941416 85.204207,28.23373 Z M94.501799,33.07748 L94.501799,23.405597 L95.970549,23.405597 L95.970549,24.764972 C96.2726323,24.2858053 96.673674,23.905597 97.173674,23.624347 C97.6840907,23.3326803 98.2570073,23.186847 98.892424,23.186847 C99.611174,23.186847 100.199716,23.3378887 100.658049,23.639972 C101.116382,23.9316387 101.439299,24.343097 101.626799,24.874347 C102.397632,23.749347 103.392424,23.186847 104.611174,23.186847 C105.579924,23.186847 106.319507,23.452472 106.829924,23.983722 C107.350757,24.514972 107.611174,25.3326803 107.611174,26.436847 L107.611174,33.07748 L105.970549,33.07748 L105.970549,26.983722 C105.970549,26.327472 105.918466,25.858722 105.814299,25.577472 C105.710132,25.2858053 105.517424,25.0514303 105.236174,24.874347 C104.954924,24.6972637 104.626799,24.608722 104.251799,24.608722 C103.574716,24.608722 103.012216,24.8378887 102.564299,25.296222 C102.116382,25.7441387 101.892424,26.4628887 101.892424,27.452472 L101.892424,33.07748 L100.251799,33.07748 L100.251799,26.796222 C100.251799,26.0670553 100.116382,25.5201803 99.845549,25.155597 C99.5747157,24.7910137 99.1372157,24.608722 98.533049,24.608722 C98.0747157,24.608722 97.6476323,24.7285137 97.251799,24.968097 C96.8559657,25.2076803 96.5695073,25.561847 96.392424,26.030597 C96.2257573,26.499347 96.142424,27.1764329 96.142424,28.0618546 L96.142424,33.07748 L94.501799,33.07748 L94.501799,33.07748 Z M116.355101,31.88998 C115.740518,32.4003967 115.151976,32.76498 114.589476,32.98373 C114.037393,33.1920633 113.438434,33.29623 112.792601,33.29623 C111.730101,33.29623 110.912393,33.0358133 110.339476,32.51498 C109.766559,31.9941467 109.480101,31.3326883 109.480101,30.530605 C109.480101,30.0514383 109.589476,29.6191467 109.808226,29.23373 C110.026976,28.8378967 110.308226,28.5253967 110.651976,28.29623 C111.006143,28.0566416 111.401976,27.8743474 111.839476,27.7493474 C112.151976,27.6660141 112.636351,27.5826807 113.292601,27.4993474 C114.615518,27.3430974 115.589476,27.1555974 116.214476,26.9368474 C116.224893,26.7076807 116.230101,26.5670557 116.230101,26.5149724 C116.230101,25.8483057 116.073851,25.3743474 115.761351,25.0930974 C115.344684,24.7285141 114.719684,24.5462224 113.886351,24.5462224 C113.115518,24.5462224 112.547809,24.6816391 112.183226,24.9524724 C111.818643,25.2233057 111.547809,25.6972641 111.370726,26.3743474 L109.761351,26.1555974 C109.907184,25.4785141 110.146768,24.9316391 110.480101,24.5149724 C110.813434,24.0878891 111.297809,23.7597641 111.933226,23.5305974 C112.568643,23.3014307 113.297809,23.1868474 114.120726,23.1868474 C114.954059,23.1868474 115.625934,23.2858057 116.136351,23.4837224 C116.657184,23.6712224 117.037393,23.9160141 117.276976,24.2180974 C117.526976,24.5097641 117.698851,24.8795557 117.792601,25.3274724 C117.844684,25.6087224 117.870726,26.1139307 117.870726,26.8430974 L117.870726,29.030605 C117.870726,30.5514383 117.907184,31.51498 117.980101,31.92123 C118.053018,32.3170633 118.188434,32.70248 118.386351,33.07748 L116.683226,33.07748 C116.506143,32.73373 116.396768,32.3378967 116.355101,31.88998 Z M116.214476,28.218105 C115.620726,28.468105 114.730101,28.6764383 113.542601,28.843105 C112.865518,28.936855 112.386351,29.04623 112.105101,29.17123 C111.823851,29.29623 111.605101,29.4733133 111.448851,29.70248 C111.303018,29.9316467 111.230101,30.1920633 111.230101,30.48373 C111.230101,30.92123 111.396768,31.2858133 111.730101,31.57748 C112.063434,31.8691467 112.547809,32.01498 113.183226,32.01498 C113.808226,32.01498 114.365518,31.8795633 114.855101,31.60873 C115.355101,31.32748 115.719684,30.9472717 115.948851,30.468105 C116.125934,30.093105 116.214476,29.54623 116.214476,28.82748 L116.214476,28.218105 Z M120.418318,21.6087224 L120.418318,19.7180974 L122.058943,19.7180974 L122.058943,21.6087224 L120.418318,21.6087224 Z M120.418318,33.0774804 L120.418318,23.4055974 L122.058943,23.4055974 L122.058943,33.0774804 L120.418318,33.0774804 Z M124.563139,33.0774804 L124.563139,23.4055974 L126.031889,23.4055974 L126.031889,24.7805974 C126.740222,23.7180974 127.766264,23.1868474 129.110014,23.1868474 C129.693347,23.1868474 130.229806,23.2910141 130.719389,23.4993474 C131.208972,23.7076807 131.573556,23.9837224 131.813139,24.3274724 C132.063139,24.6608057 132.235014,25.0618474 132.328764,25.5305974 C132.391264,25.8326807 132.422514,26.3639307 132.422514,27.1243474 L132.422514,33.0774804 L130.781889,33.0774804 L130.781889,27.1868474 C130.781889,26.5201807 130.714181,26.0253891 130.578764,25.7024724 C130.453764,25.3691391 130.229806,25.1035141 129.906889,24.9055974 C129.583972,24.7076807 129.203764,24.6087224 128.766264,24.6087224 C128.068347,24.6087224 127.464181,24.8326807 126.953764,25.2805974 C126.453764,25.7180974 126.203764,26.5566391 126.203764,27.7962224 L126.203764,33.0774804 L124.563139,33.0774804 L124.563139,33.0774804 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="383 10.47244 466.653534 10.47244 466.653534 41.842511 383 41.842511"></polygon>
+                <path d="M395.8628,30.1868524 L397.4878,29.93686 C397.571133,30.5826933 397.821133,31.0826933 398.2378,31.43686 C398.654467,31.78061 399.2378,31.952485 399.9878,31.952485 C400.7378,31.952485 401.295092,31.8014433 401.659675,31.49936 C402.024258,31.1868524 402.20655,30.8222767 402.20655,30.40561 C402.20655,30.0410267 402.045092,29.74936 401.722175,29.53061 C401.503425,29.3847767 400.946133,29.202485 400.0503,28.983735 C398.841967,28.671235 398.003425,28.40561 397.534675,28.1868524 C397.076342,27.9681049 396.727383,27.666019 396.4878,27.2806024 C396.248217,26.8847691 396.128425,26.4472691 396.128425,25.9681024 C396.128425,25.5410191 396.227383,25.1451857 396.4253,24.7806024 C396.623217,24.4056024 396.89405,24.0983107 397.2378,23.8587274 C397.4878,23.6712274 397.83155,23.5149774 398.26905,23.3899774 C398.716967,23.2545607 399.190925,23.1868524 399.690925,23.1868524 C400.451342,23.1868524 401.118008,23.2962274 401.690925,23.5149774 C402.263842,23.7337274 402.685717,24.0306024 402.95655,24.4056024 C403.2378,24.7806024 403.430508,25.2806024 403.534675,25.9056024 L401.9253,26.1243524 C401.852383,25.6243524 401.638842,25.2337274 401.284675,24.9524774 C400.940925,24.6712274 400.451342,24.5306024 399.815925,24.5306024 C399.055508,24.5306024 398.513842,24.6556024 398.190925,24.9056024 C397.878425,25.1556024 397.722175,25.4472691 397.722175,25.7806024 C397.722175,25.9889357 397.784675,26.1816441 397.909675,26.3587274 C398.045092,26.5358107 398.258633,26.6816441 398.5503,26.7962274 C398.70655,26.8587274 399.185717,26.9993524 399.9878,27.2181024 C401.154467,27.5201857 401.966967,27.7701857 402.4253,27.9681024 C402.883633,28.1660242 403.243008,28.452485 403.503425,28.827485 C403.763842,29.202485 403.89405,29.671235 403.89405,30.233735 C403.89405,30.7858183 403.732592,31.3014433 403.409675,31.78061 C403.097175,32.2597767 402.638842,32.6347767 402.034675,32.90561 C401.430508,33.1660267 400.748217,33.296235 399.9878,33.296235 C398.7378,33.296235 397.779467,33.0358183 397.1128,32.514985 C396.45655,31.9941517 396.039883,31.21811 395.8628,30.1868524 Z M405.76905,36.796235 L405.58155,35.264985 C405.946133,35.358735 406.258633,35.405602 406.51905,35.405602 C406.883633,35.405602 407.1753,35.34311 407.39405,35.21811 C407.6128,35.1035267 407.795092,34.93686 407.940925,34.71811 C408.045092,34.5514433 408.211758,34.1451933 408.440925,33.49936 C408.472175,33.405602 408.51905,33.2701933 408.58155,33.09311 L404.909675,23.405602 L406.6753,23.405602 L408.690925,28.9993596 C408.951342,29.7181096 409.185717,30.4681096 409.39405,31.2493596 C409.58155,30.4993596 409.805508,29.7649846 410.065925,29.0462346 L412.14405,23.405602 L413.784675,23.405602 L410.097175,33.233735 C409.701342,34.3066517 409.39405,35.0410267 409.1753,35.43686 C408.883633,35.9785267 408.5503,36.37436 408.1753,36.62436 C407.8003,36.87436 407.352383,36.99936 406.83155,36.99936 C406.508633,36.99936 406.154467,36.9316517 405.76905,36.796235 Z M414.51905,30.1868524 L416.14405,29.93686 C416.227383,30.5826933 416.477383,31.0826933 416.89405,31.43686 C417.310717,31.78061 417.89405,31.952485 418.64405,31.952485 C419.39405,31.952485 419.951342,31.8014433 420.315925,31.49936 C420.680508,31.1868524 420.8628,30.8222767 420.8628,30.40561 C420.8628,30.0410267 420.701342,29.74936 420.378425,29.53061 C420.159675,29.3847767 419.602383,29.202485 418.70655,28.983735 C417.498217,28.671235 416.659675,28.40561 416.190925,28.1868524 C415.732592,27.9681049 415.383633,27.666019 415.14405,27.2806024 C414.904467,26.8847691 414.784675,26.4472691 414.784675,25.9681024 C414.784675,25.5410191 414.883633,25.1451857 415.08155,24.7806024 C415.279467,24.4056024 415.5503,24.0983107 415.89405,23.8587274 C416.14405,23.6712274 416.4878,23.5149774 416.9253,23.3899774 C417.373217,23.2545607 417.847175,23.1868524 418.347175,23.1868524 C419.107592,23.1868524 419.774258,23.2962274 420.347175,23.5149774 C420.920092,23.7337274 421.341967,24.0306024 421.6128,24.4056024 C421.89405,24.7806024 422.086758,25.2806024 422.190925,25.9056024 L420.58155,26.1243524 C420.508633,25.6243524 420.295092,25.2337274 419.940925,24.9524774 C419.597175,24.6712274 419.107592,24.5306024 418.472175,24.5306024 C417.711758,24.5306024 417.170092,24.6556024 416.847175,24.9056024 C416.534675,25.1556024 416.378425,25.4472691 416.378425,25.7806024 C416.378425,25.9889357 416.440925,26.1816441 416.565925,26.3587274 C416.701342,26.5358107 416.914883,26.6816441 417.20655,26.7962274 C417.3628,26.8587274 417.841967,26.9993524 418.64405,27.2181024 C419.810717,27.5201857 420.623217,27.7701857 421.08155,27.9681024 C421.539883,28.1660242 421.899258,28.452485 422.159675,28.827485 C422.420092,29.202485 422.5503,29.671235 422.5503,30.233735 C422.5503,30.7858183 422.388842,31.3014433 422.065925,31.78061 C421.753425,32.2597767 421.295092,32.6347767 420.690925,32.90561 C420.086758,33.1660267 419.404467,33.296235 418.64405,33.296235 C417.39405,33.296235 416.435717,33.0358183 415.76905,32.514985 C415.1128,31.9941517 414.696133,31.21811 414.51905,30.1868524 Z M428.08155,31.608735 L428.315925,33.06186 C427.857592,33.15561 427.446133,33.202485 427.08155,33.202485 C426.4878,33.202485 426.024258,33.108735 425.690925,32.921235 C425.368008,32.733735 425.138842,32.4889433 425.003425,32.18686 C424.868008,31.87436 424.8003,31.2285267 424.8003,30.24936 L424.8003,24.6712274 L423.597175,24.6712274 L423.597175,23.4056024 L424.8003,23.4056024 L424.8003,21.0149774 L426.4253,20.0306024 L426.4253,23.4056024 L428.08155,23.4056024 L428.08155,24.6712274 L426.4253,24.6712274 L426.4253,30.34311 C426.4253,30.8014433 426.451342,31.0983183 426.503425,31.233735 C426.565925,31.3691517 426.659675,31.4785267 426.784675,31.56186 C426.920092,31.6347767 427.1128,31.671235 427.3628,31.671235 C427.539883,31.671235 427.779467,31.6504017 428.08155,31.608735 Z M436.311744,29.96811 L437.999244,30.171235 C437.728411,31.1608183 437.233619,31.9316517 436.514869,32.483735 C435.796119,33.0254017 434.874244,33.296235 433.749244,33.296235 C432.332577,33.296235 431.207577,32.8639433 430.374244,31.99936 C429.551327,31.12436 429.139869,29.9004017 429.139869,28.327485 C429.139869,26.6920633 429.556536,25.4264357 430.389869,24.5306024 C431.233619,23.6347691 432.322161,23.1868524 433.655494,23.1868524 C434.947161,23.1868524 435.999244,23.6295607 436.811744,24.5149774 C437.634661,25.3899774 438.046119,26.6243549 438.046119,28.21811 C438.046119,28.3222767 438.046119,28.46811 438.046119,28.65561 L430.827369,28.65561 C430.889869,29.71811 431.191952,30.5358183 431.733619,31.108735 C432.275286,31.671235 432.947161,31.952485 433.749244,31.952485 C434.353411,31.952485 434.869036,31.796235 435.296119,31.483735 C435.723202,31.1608183 436.061744,30.65561 436.311744,29.96811 Z M430.921119,27.3118524 L436.327369,27.3118524 C436.254452,26.4993524 436.046119,25.8899774 435.702369,25.4837274 C435.181536,24.8483107 434.504452,24.5306024 433.671119,24.5306024 C432.921119,24.5306024 432.285702,24.7858107 431.764869,25.2962274 C431.254452,25.7962274 430.973202,26.4681024 430.921119,27.3118524 Z M440.062446,33.077485 L440.062446,23.405602 L441.531196,23.405602 L441.531196,24.764977 C441.833279,24.2858103 442.234321,23.905602 442.734321,23.624352 C443.244738,23.3326853 443.817654,23.186852 444.453071,23.186852 C445.171821,23.186852 445.760363,23.3378937 446.218696,23.639977 C446.677029,23.9316437 446.999946,24.343102 447.187446,24.874352 C447.958279,23.749352 448.953071,23.186852 450.171821,23.186852 C451.140571,23.186852 451.880154,23.452477 452.390571,23.983727 C452.911404,24.514977 453.171821,25.3326853 453.171821,26.436852 L453.171821,33.077485 L451.531196,33.077485 L451.531196,26.983727 C451.531196,26.327477 451.479113,25.858727 451.374946,25.577477 C451.270779,25.2858103 451.078071,25.0514353 450.796821,24.874352 C450.515571,24.6972687 450.187446,24.608727 449.812446,24.608727 C449.135363,24.608727 448.572863,24.8378937 448.124946,25.296227 C447.677029,25.7441437 447.453071,26.4628937 447.453071,27.452477 L447.453071,33.077485 L445.812446,33.077485 L445.812446,26.796227 C445.812446,26.0670603 445.677029,25.5201853 445.406196,25.155602 C445.135363,24.7910187 444.697863,24.608727 444.093696,24.608727 C443.635363,24.608727 443.208279,24.7285187 442.812446,24.968102 C442.416613,25.2076853 442.130154,25.561852 441.953071,26.030602 C441.786404,26.499352 441.703071,27.1764379 441.703071,28.0618596 L441.703071,33.077485 L440.062446,33.077485 L440.062446,33.077485 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M607.7156,4.1045 L607.7156,4.1045 C607.7156,2.5803148 608.951196,1.344719 610.475366,1.344719 L652.325891,1.344719 C653.057824,1.344719 653.75979,1.63547523 654.277368,2.1530381 C654.794946,2.67060097 655.085717,3.3725587 655.085717,4.1044999 L655.085717,48.6636799 C655.085717,50.1878651 653.850122,51.4234609 652.325891,51.4234609 L610.475366,51.4234609 C608.951196,51.4234609 607.7156,50.1878652 607.7156,48.6636799 L607.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M607.7156,4.1045 L607.7156,4.1045 C607.7156,2.5803148 608.951196,1.344719 610.475366,1.344719 L652.325891,1.344719 C653.057824,1.344719 653.75979,1.63547523 654.277368,2.1530381 C654.794946,2.67060097 655.085717,3.3725587 655.085717,4.1044999 L655.085717,48.6636799 C655.085717,50.1878651 653.850122,51.4234609 652.325891,51.4234609 L610.475366,51.4234609 C608.951196,51.4234609 607.7156,50.1878652 607.7156,48.6636799 L607.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M632.244,33.3041 L632.244,30.100975 L626.447125,30.100975 L626.447125,28.600975 L632.540875,19.944717 L633.884625,19.944717 L633.884625,28.600975 L635.6815,28.600975 L635.6815,30.100975 L633.884625,30.100975 L633.884625,33.3041 L632.244,33.3041 Z M632.244,28.600975 L632.244,22.5853424 L628.0565,28.600975 L632.244,28.600975 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="527.0889 16.85302 546.521944 16.85302 546.521944 38.679795 527.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="599.0889 16.85302 618.521944 16.85302 618.521944 38.679795 599.0889 38.679795"></polygon>
+                <line x1="618.522" y1="27.7664" x2="533.1362" y2="28.7427794" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="600.5232" y1="27.97221" x2="533.136176" y2="28.7427789" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="600.57983 32.9271 614.136532 27.8165577 600.466488 23.0173557"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.8793 12.06562 588.603726 12.06562 588.603726 43.435699 543.8793 43.435699"></polygon>
+                <path d="M560.7394,23.201904 L560.7394,21.311279 L562.380025,21.311279 L562.380025,23.201904 L560.7394,23.201904 Z M560.7394,34.670654 L560.7394,24.998779 L562.380025,24.998779 L562.380025,34.670654 L560.7394,34.670654 Z M564.227986,31.780029 L565.852986,31.530029 C565.93632,32.1758623 566.18632,32.6758623 566.602986,33.030029 C567.019653,33.373779 567.602986,33.545654 568.352986,33.545654 C569.102986,33.545654 569.660278,33.3946123 570.024861,33.092529 C570.389445,32.780029 570.571736,32.4154457 570.571736,31.998779 C570.571736,31.6341957 570.410278,31.342529 570.087361,31.123779 C569.868611,30.9779457 569.31132,30.795654 568.415486,30.576904 C567.207153,30.264404 566.368611,29.998779 565.899861,29.780029 C565.441528,29.561279 565.09257,29.2591957 564.852986,28.873779 C564.613403,28.4779457 564.493611,28.0404457 564.493611,27.561279 C564.493611,27.1341957 564.59257,26.7383623 564.790486,26.373779 C564.988403,25.998779 565.259236,25.6914873 565.602986,25.451904 C565.852986,25.264404 566.196736,25.108154 566.634236,24.983154 C567.082153,24.8477373 567.556111,24.780029 568.056111,24.780029 C568.816528,24.780029 569.483195,24.889404 570.056111,25.108154 C570.629028,25.326904 571.050903,25.623779 571.321736,25.998779 C571.602986,26.373779 571.795695,26.873779 571.899861,27.498779 L570.290486,27.717529 C570.21757,27.217529 570.004028,26.826904 569.649861,26.545654 C569.306111,26.264404 568.816528,26.123779 568.181111,26.123779 C567.420695,26.123779 566.879028,26.248779 566.556111,26.498779 C566.243611,26.748779 566.087361,27.0404457 566.087361,27.373779 C566.087361,27.5821123 566.149861,27.7748207 566.274861,27.951904 C566.410278,28.1289873 566.62382,28.2748207 566.915486,28.389404 C567.071736,28.451904 567.550903,28.592529 568.352986,28.811279 C569.519653,29.1133623 570.332153,29.3633623 570.790486,29.561279 C571.24882,29.7591957 571.608195,30.045654 571.868611,30.420654 C572.129028,30.795654 572.259236,31.264404 572.259236,31.826904 C572.259236,32.3789873 572.097778,32.8946123 571.774861,33.373779 C571.462361,33.8529457 571.004028,34.2279457 570.399861,34.498779 C569.795695,34.7591957 569.113403,34.889404 568.352986,34.889404 C567.102986,34.889404 566.144653,34.6289873 565.477986,34.108154 C564.821736,33.5873207 564.40507,32.811279 564.227986,31.780029 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M759.7156,4.1045 L759.7156,4.1045 C759.7156,2.5803148 760.951196,1.344719 762.475366,1.344719 L804.325891,1.344719 C805.057824,1.344719 805.75979,1.63547523 806.277368,2.1530381 C806.794946,2.67060097 807.085717,3.3725587 807.085717,4.1044999 L807.085717,48.6636799 C807.085717,50.1878651 805.850122,51.4234609 804.325891,51.4234609 L762.475366,51.4234609 C760.951196,51.4234609 759.7156,50.1878652 759.7156,48.6636799 L759.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M759.7156,4.1045 L759.7156,4.1045 C759.7156,2.5803148 760.951196,1.344719 762.475366,1.344719 L804.325891,1.344719 C805.057824,1.344719 805.75979,1.63547523 806.277368,2.1530381 C806.794946,2.67060097 807.085717,3.3725587 807.085717,4.1044999 L807.085717,48.6636799 C807.085717,50.1878651 805.850122,51.4234609 804.325891,51.4234609 L762.475366,51.4234609 C760.951196,51.4234609 759.7156,50.1878652 759.7156,48.6636799 L759.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M778.994,29.8041 L780.71275,29.663475 C780.83775,30.4968083 781.129417,31.1270167 781.58775,31.5541 C782.0565,31.9707667 782.619,32.1791 783.27525,32.1791 C784.066917,32.1791 784.733583,31.882225 785.27525,31.288475 C785.827333,30.694725 786.103375,29.9082667 786.103375,28.9291 C786.103375,27.9915949 785.83775,27.2520091 785.3065,26.7103424 C784.785667,26.1686757 784.098167,25.8978424 783.244,25.8978424 C782.723167,25.8978424 782.249208,26.0176341 781.822125,26.2572174 C781.395042,26.4968007 781.061708,26.8093007 780.822125,27.1947174 L779.27525,26.9915924 L780.572125,20.1322174 L787.21275,20.1322174 L787.21275,21.6947174 L781.884625,21.6947174 L781.165875,25.2884674 C781.967958,24.7259674 782.8065,24.4447174 783.6815,24.4447174 C784.848167,24.4447174 785.832542,24.8509674 786.634625,25.6634674 C787.436708,26.4655507 787.83775,27.5020116 787.83775,28.77285 C787.83775,29.9811833 787.488792,31.0280583 786.790875,31.913475 C785.926292,32.9968083 784.754417,33.538475 783.27525,33.538475 C782.066917,33.538475 781.077333,33.1999333 780.3065,32.52285 C779.535667,31.83535 779.098167,30.9291 778.994,29.8041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="647.0889 16.85302 666.521944 16.85302 666.521944 38.679795 647.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="751.0889 16.85302 770.521944 16.85302 770.521944 38.679795 751.0889 38.679795"></polygon>
+                <line x1="770.522" y1="27.7664" x2="657.38815" y2="29.5616728" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="752.5243" y1="28.052" x2="657.38813" y2="29.5616741" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="752.60284 33.00658 766.136837 27.8359944 752.445614 23.0974309"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="669.7087 10.47244 740.606344 10.47244 740.606344 41.842511 669.7087 41.842511"></polygon>
+                <path d="M681.3761,33.07748 L681.3761,24.671222 L679.922975,24.671222 L679.922975,23.405597 L681.3761,23.405597 L681.3761,22.374347 C681.3761,21.7285137 681.433392,21.2441387 681.547975,20.921222 C681.704225,20.4941387 681.980267,20.1503887 682.3761,19.889972 C682.771933,19.6295553 683.329225,19.499347 684.047975,19.499347 C684.506308,19.499347 685.016725,19.5514303 685.579225,19.655597 L685.329225,21.093097 C684.995892,21.030597 684.678183,20.999347 684.3761,20.999347 C683.8761,20.999347 683.521933,21.108722 683.3136,21.327472 C683.105267,21.5358053 683.0011,21.9316387 683.0011,22.514972 L683.0011,23.405597 L684.891725,23.405597 L684.891725,24.671222 L683.0011,24.671222 L683.0011,33.07748 L681.3761,33.07748 Z M686.153139,33.07748 L686.153139,23.405597 L687.621889,23.405597 L687.621889,24.874347 C687.996889,24.186847 688.340639,23.733722 688.653139,23.514972 C688.976055,23.296222 689.330222,23.186847 689.715639,23.186847 C690.267722,23.186847 690.830222,23.3639303 691.403139,23.718097 L690.840639,25.233722 C690.434389,24.9941387 690.033347,24.874347 689.637514,24.874347 C689.27293,24.874347 688.950014,24.983722 688.668764,25.202472 C688.387514,25.421222 688.184389,25.718097 688.059389,26.093097 C687.871889,26.6764303 687.778139,27.3170579 687.778139,28.0149796 L687.778139,33.07748 L686.153139,33.07748 L686.153139,33.07748 Z M698.693971,31.88998 C698.079388,32.4003967 697.490846,32.76498 696.928346,32.98373 C696.376263,33.1920633 695.777304,33.29623 695.131471,33.29623 C694.068971,33.29623 693.251263,33.0358133 692.678346,32.51498 C692.105429,31.9941467 691.818971,31.3326883 691.818971,30.530605 C691.818971,30.0514383 691.928346,29.6191467 692.147096,29.23373 C692.365846,28.8378967 692.647096,28.5253967 692.990846,28.29623 C693.345013,28.0566416 693.740846,27.8743474 694.178346,27.7493474 C694.490846,27.6660141 694.975221,27.5826807 695.631471,27.4993474 C696.954388,27.3430974 697.928346,27.1555974 698.553346,26.9368474 C698.563763,26.7076807 698.568971,26.5670557 698.568971,26.5149724 C698.568971,25.8483057 698.412721,25.3743474 698.100221,25.0930974 C697.683554,24.7285141 697.058554,24.5462224 696.225221,24.5462224 C695.454388,24.5462224 694.886679,24.6816391 694.522096,24.9524724 C694.157513,25.2233057 693.886679,25.6972641 693.709596,26.3743474 L692.100221,26.1555974 C692.246054,25.4785141 692.485638,24.9316391 692.818971,24.5149724 C693.152304,24.0878891 693.636679,23.7597641 694.272096,23.5305974 C694.907513,23.3014307 695.636679,23.1868474 696.459596,23.1868474 C697.292929,23.1868474 697.964804,23.2858057 698.475221,23.4837224 C698.996054,23.6712224 699.376263,23.9160141 699.615846,24.2180974 C699.865846,24.5097641 700.037721,24.8795557 700.131471,25.3274724 C700.183554,25.6087224 700.209596,26.1139307 700.209596,26.8430974 L700.209596,29.030605 C700.209596,30.5514383 700.246054,31.51498 700.318971,31.92123 C700.391888,32.3170633 700.527304,32.70248 700.725221,33.07748 L699.022096,33.07748 C698.845013,32.73373 698.735638,32.3378967 698.693971,31.88998 Z M698.553346,28.218105 C697.959596,28.468105 697.068971,28.6764383 695.881471,28.843105 C695.204388,28.936855 694.725221,29.04623 694.443971,29.17123 C694.162721,29.29623 693.943971,29.4733133 693.787721,29.70248 C693.641888,29.9316467 693.568971,30.1920633 693.568971,30.48373 C693.568971,30.92123 693.735638,31.2858133 694.068971,31.57748 C694.402304,31.8691467 694.886679,32.01498 695.522096,32.01498 C696.147096,32.01498 696.704388,31.8795633 697.193971,31.60873 C697.693971,31.32748 698.058554,30.9472717 698.287721,30.468105 C698.464804,30.093105 698.553346,29.54623 698.553346,28.82748 L698.553346,28.218105 Z M702.444642,33.874355 L704.038392,34.10873 C704.111309,34.60873 704.298809,34.968105 704.600892,35.1868474 C705.007142,35.4889383 705.564434,35.63998 706.272767,35.63998 C707.033184,35.63998 707.616517,35.4889383 708.022767,35.1868474 C708.439434,34.8847717 708.720684,34.4628967 708.866517,33.92123 C708.949851,33.5878967 708.986309,32.8847717 708.975892,31.811855 C708.267559,32.655605 707.376934,33.07748 706.304017,33.07748 C704.970684,33.07748 703.939434,32.5983133 703.210267,31.63998 C702.481101,30.6816467 702.116517,29.530605 702.116517,28.1868474 C702.116517,27.2597666 702.283184,26.4055974 702.616517,25.6243474 C702.960267,24.8430974 703.444642,24.2441391 704.069642,23.8274724 C704.705059,23.4003891 705.455059,23.1868474 706.319642,23.1868474 C707.455059,23.1868474 708.392559,23.6451807 709.132142,24.5618474 L709.132142,23.4055974 L710.647767,23.4055974 L710.647767,31.7649804 C710.647767,33.2753971 710.491517,34.3431054 710.179017,34.9681054 C709.876934,35.5931054 709.392559,36.0878971 708.725892,36.4524804 C708.069642,36.8170637 707.251934,36.9993554 706.272767,36.9993554 C705.126934,36.9993554 704.194642,36.7389387 703.475892,36.2181054 C702.767559,35.7076887 702.423809,34.9264387 702.444642,33.8743554 L702.444642,33.874355 Z M703.804017,28.061855 C703.804017,29.3326883 704.054017,30.2597717 704.554017,30.843105 C705.064434,31.4264383 705.699851,31.718105 706.460267,31.718105 C707.210267,31.718105 707.840476,31.4316467 708.350892,30.85873 C708.861309,30.2753967 709.116517,29.3639383 709.116517,28.124355 C709.116517,26.9368499 708.850892,26.0462224 708.319642,25.4524724 C707.798809,24.8483057 707.168601,24.5462224 706.429017,24.5462224 C705.699851,24.5462224 705.080059,24.8430974 704.569642,25.4368474 C704.059226,26.0201807 703.804017,26.8951833 703.804017,28.061855 Z M713.132874,21.6087224 L713.132874,19.7180974 L714.773499,19.7180974 L714.773499,21.6087224 L713.132874,21.6087224 Z M713.132874,33.0774804 L713.132874,23.4055974 L714.773499,23.4055974 L714.773499,33.0774804 L713.132874,33.0774804 Z M717.230836,33.0774804 L717.230836,19.7180974 L718.871461,19.7180974 L718.871461,33.0774804 L717.230836,33.0774804 Z M728.047547,29.9681054 L729.735047,30.1712304 C729.464214,31.1608137 728.969422,31.9316471 728.250672,32.4837304 C727.531922,33.0253971 726.610047,33.2962304 725.485047,33.2962304 C724.06838,33.2962304 722.94338,32.8639387 722.110047,31.9993554 C721.28713,31.1243554 720.875672,29.9003971 720.875672,28.3274804 C720.875672,26.6920587 721.292339,25.4264311 722.125672,24.5305978 C722.969422,23.6347645 724.057964,23.1868478 725.391297,23.1868478 C726.682964,23.1868478 727.735047,23.6295561 728.547547,24.5149728 C729.370464,25.3899728 729.781922,26.6243503 729.781922,28.2181054 C729.781922,28.3222721 729.781922,28.4681054 729.781922,28.6556054 L722.563172,28.6556054 C722.625672,29.7181054 722.927755,30.5358137 723.469422,31.1087304 C724.011089,31.6712304 724.682964,31.9524804 725.485047,31.9524804 C726.089214,31.9524804 726.604839,31.7962304 727.031922,31.4837304 C727.459005,31.1608137 727.797547,30.6556054 728.047547,29.9681054 Z M722.656922,27.3118478 L728.063172,27.3118478 C727.990255,26.4993478 727.781922,25.8899728 727.438172,25.4837228 C726.917339,24.8483061 726.240255,24.5305978 725.406922,24.5305978 C724.656922,24.5305978 724.021505,24.7858061 723.500672,25.2962228 C722.990255,25.7962228 722.709005,26.4680978 722.656922,27.3118478 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <line x1="330.52197" y1="27.7664" x2="217.45111" y2="27.7664" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="312.52197" y1="27.7664" x2="217.45111" y2="27.7664" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="312.52197 32.7216 326.136258 27.7664 312.52197 22.8112074"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="229.4882 10.47244 300.385844 10.47244 300.385844 41.842511 229.4882 41.842511"></polygon>
+                <path d="M242.8374,33.07748 L242.8374,23.405597 L244.30615,23.405597 L244.30615,24.780597 C245.014483,23.718097 246.040525,23.186847 247.384275,23.186847 C247.967608,23.186847 248.504067,23.2910137 248.99365,23.499347 C249.483233,23.7076803 249.847817,23.983722 250.0874,24.327472 C250.3374,24.6608053 250.509275,25.061847 250.603025,25.530597 C250.665525,25.8326803 250.696775,26.3639303 250.696775,27.124347 L250.696775,33.07748 L249.05615,33.07748 L249.05615,27.186847 C249.05615,26.5201803 248.988442,26.0253887 248.853025,25.702472 C248.728025,25.3691387 248.504067,25.1035137 248.18115,24.905597 C247.858233,24.7076803 247.478025,24.608722 247.040525,24.608722 C246.342608,24.608722 245.738442,24.8326803 245.228025,25.280597 C244.728025,25.718097 244.478025,26.5566387 244.478025,27.796222 L244.478025,33.07748 L242.8374,33.07748 L242.8374,33.07748 Z M259.525632,31.88998 C258.911049,32.4003967 258.322507,32.76498 257.760007,32.98373 C257.207924,33.1920633 256.608965,33.29623 255.963132,33.29623 C254.900632,33.29623 254.082924,33.0358133 253.510007,32.51498 C252.93709,31.9941467 252.650632,31.3326883 252.650632,30.530605 C252.650632,30.0514383 252.760007,29.6191467 252.978757,29.23373 C253.197507,28.8378967 253.478757,28.5253967 253.822507,28.29623 C254.176674,28.0566416 254.572507,27.8743474 255.010007,27.7493474 C255.322507,27.6660141 255.806882,27.5826807 256.463132,27.4993474 C257.786049,27.3430974 258.760007,27.1555974 259.385007,26.9368474 C259.395424,26.7076807 259.400632,26.5670557 259.400632,26.5149724 C259.400632,25.8483057 259.244382,25.3743474 258.931882,25.0930974 C258.515215,24.7285141 257.890215,24.5462224 257.056882,24.5462224 C256.286049,24.5462224 255.71834,24.6816391 255.353757,24.9524724 C254.989174,25.2233057 254.71834,25.6972641 254.541257,26.3743474 L252.931882,26.1555974 C253.077715,25.4785141 253.317299,24.9316391 253.650632,24.5149724 C253.983965,24.0878891 254.46834,23.7597641 255.103757,23.5305974 C255.739174,23.3014307 256.46834,23.1868474 257.291257,23.1868474 C258.12459,23.1868474 258.796465,23.2858057 259.306882,23.4837224 C259.827715,23.6712224 260.207924,23.9160141 260.447507,24.2180974 C260.697507,24.5097641 260.869382,24.8795557 260.963132,25.3274724 C261.015215,25.6087224 261.041257,26.1139307 261.041257,26.8430974 L261.041257,29.030605 C261.041257,30.5514383 261.077715,31.51498 261.150632,31.92123 C261.223549,32.3170633 261.358965,32.70248 261.556882,33.07748 L259.853757,33.07748 C259.676674,32.73373 259.567299,32.3378967 259.525632,31.88998 Z M259.385007,28.218105 C258.791257,28.468105 257.900632,28.6764383 256.713132,28.843105 C256.036049,28.936855 255.556882,29.04623 255.275632,29.17123 C254.994382,29.29623 254.775632,29.4733133 254.619382,29.70248 C254.473549,29.9316467 254.400632,30.1920633 254.400632,30.48373 C254.400632,30.92123 254.567299,31.2858133 254.900632,31.57748 C255.233965,31.8691467 255.71834,32.01498 256.353757,32.01498 C256.978757,32.01498 257.536049,31.8795633 258.025632,31.60873 C258.525632,31.32748 258.890215,30.9472717 259.119382,30.468105 C259.296465,30.093105 259.385007,29.54623 259.385007,28.82748 L259.385007,28.218105 Z M263.588834,33.07748 L263.588834,23.405597 L265.057584,23.405597 L265.057584,24.764972 C265.359667,24.2858053 265.760709,23.905597 266.260709,23.624347 C266.771126,23.3326803 267.344042,23.186847 267.979459,23.186847 C268.698209,23.186847 269.286751,23.3378887 269.745084,23.639972 C270.203417,23.9316387 270.526334,24.343097 270.713834,24.874347 C271.484667,23.749347 272.479459,23.186847 273.698209,23.186847 C274.666959,23.186847 275.406542,23.452472 275.916959,23.983722 C276.437792,24.514972 276.698209,25.3326803 276.698209,26.436847 L276.698209,33.07748 L275.057584,33.07748 L275.057584,26.983722 C275.057584,26.327472 275.005501,25.858722 274.901334,25.577472 C274.797167,25.2858053 274.604459,25.0514303 274.323209,24.874347 C274.041959,24.6972637 273.713834,24.608722 273.338834,24.608722 C272.661751,24.608722 272.099251,24.8378887 271.651334,25.296222 C271.203417,25.7441387 270.979459,26.4628887 270.979459,27.452472 L270.979459,33.07748 L269.338834,33.07748 L269.338834,26.796222 C269.338834,26.0670553 269.203417,25.5201803 268.932584,25.155597 C268.661751,24.7910137 268.224251,24.608722 267.620084,24.608722 C267.161751,24.608722 266.734667,24.7285137 266.338834,24.968097 C265.943001,25.2076803 265.656542,25.561847 265.479459,26.030597 C265.312792,26.499347 265.229459,27.1764329 265.229459,28.0618546 L265.229459,33.07748 L263.588834,33.07748 L263.588834,33.07748 Z M285.754636,29.968105 L287.442136,30.17123 C287.171303,31.1608133 286.676511,31.9316467 285.957761,32.48373 C285.239011,33.0253967 284.317136,33.29623 283.192136,33.29623 C281.775469,33.29623 280.650469,32.8639383 279.817136,31.999355 C278.994219,31.124355 278.582761,29.9003967 278.582761,28.32748 C278.582761,26.6920583 278.999428,25.4264307 279.832761,24.5305974 C280.676511,23.6347641 281.765053,23.1868474 283.098386,23.1868474 C284.390053,23.1868474 285.442136,23.6295557 286.254636,24.5149724 C287.077553,25.3899724 287.489011,26.6243499 287.489011,28.218105 C287.489011,28.3222717 287.489011,28.468105 287.489011,28.655605 L280.270261,28.655605 C280.332761,29.718105 280.634844,30.5358133 281.176511,31.10873 C281.718178,31.67123 282.390053,31.95248 283.192136,31.95248 C283.796303,31.95248 284.311928,31.79623 284.739011,31.48373 C285.166094,31.1608133 285.504636,30.655605 285.754636,29.968105 Z M280.364011,27.3118474 L285.770261,27.3118474 C285.697344,26.4993474 285.489011,25.8899724 285.145261,25.4837224 C284.624428,24.8483057 283.947344,24.5305974 283.114011,24.5305974 C282.364011,24.5305974 281.728594,24.7858057 281.207761,25.2962224 C280.697344,25.7962224 280.416094,26.4680974 280.364011,27.3118474 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polyline id="Path" fill-opacity="0" fill="#000000" points="192.80577 54.64905 192.80577 111.062975 24.80577 111.062975 24.80577 51.436449"></polyline>
+                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="192.80577 72.64903 192.80577 111.06297 24.80577 111.06297 24.80577 51.436444"></polyline>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="197.76097 72.64905 192.80577 59.034746 187.85057 72.64905"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="67.93664 94.56168 138.83428 94.56168 138.83428 125.931767 67.93664 125.931767"></polygon>
+                <path d="M95.84567,117.16672 L95.84567,115.94797 C95.2415033,116.906303 94.3404617,117.38547 93.142545,117.38547 C92.3717117,117.38547 91.6633783,117.171928 91.017545,116.744845 C90.3717117,116.317762 89.8717117,115.724012 89.517545,114.963595 C89.1633783,114.203178 88.986295,113.328178 88.986295,112.338595 C88.986295,111.369845 89.1477533,110.494845 89.47067,109.713595 C89.7935867,108.921928 90.2727533,108.317762 90.90817,107.901095 C91.5540033,107.484428 92.2779617,107.276095 93.080045,107.276095 C93.6633783,107.276095 94.1790033,107.401095 94.62692,107.651095 C95.0852533,107.890678 95.455045,108.208387 95.736295,108.60422 L95.736295,103.807345 L97.37692,103.807345 L97.37692,117.16672 L95.84567,117.16672 Z M90.673795,112.338595 C90.673795,113.578178 90.9342117,114.505262 91.455045,115.119845 C91.9758783,115.734428 92.5904617,116.04172 93.298795,116.04172 C94.017545,116.04172 94.62692,115.750053 95.12692,115.16672 C95.62692,114.57297 95.87692,113.677137 95.87692,112.47922 C95.87692,111.156303 95.6217117,110.187553 95.111295,109.57297 C94.6008783,108.94797 93.97067,108.63547 93.22067,108.63547 C92.50192,108.63547 91.8977533,108.932345 91.40817,109.526095 C90.9185867,110.119845 90.673795,111.057345 90.673795,112.338595 Z M99.955762,117.16672 L99.955762,107.494845 L101.424512,107.494845 L101.424512,108.869845 C102.132845,107.807345 103.158887,107.276095 104.502637,107.276095 C105.08597,107.276095 105.622429,107.380262 106.112012,107.588595 C106.601595,107.796928 106.966179,108.07297 107.205762,108.41672 C107.455762,108.750053 107.627637,109.151095 107.721387,109.619845 C107.783887,109.921928 107.815137,110.453178 107.815137,111.213595 L107.815137,117.16672 L106.174512,117.16672 L106.174512,111.276095 C106.174512,110.609428 106.106804,110.114637 105.971387,109.79172 C105.846387,109.458387 105.622429,109.192762 105.299512,108.994845 C104.976595,108.796928 104.596387,108.69797 104.158887,108.69797 C103.46097,108.69797 102.856804,108.921928 102.346387,109.369845 C101.846387,109.807345 101.596387,110.645887 101.596387,111.88547 L101.596387,117.16672 L99.955762,117.16672 Z M109.675229,114.276095 L111.300229,114.026095 C111.383562,114.671928 111.633562,115.171928 112.050229,115.526095 C112.466896,115.869845 113.050229,116.04172 113.800229,116.04172 C114.550229,116.04172 115.107521,115.890678 115.472104,115.588595 C115.836687,115.276095 116.018979,114.911512 116.018979,114.494845 C116.018979,114.130262 115.857521,113.838595 115.534604,113.619845 C115.315854,113.474012 114.758562,113.29172 113.862729,113.07297 C112.654396,112.76047 111.815854,112.494845 111.347104,112.276095 C110.888771,112.057345 110.539812,111.755262 110.300229,111.369845 C110.060646,110.974012 109.940854,110.536512 109.940854,110.057345 C109.940854,109.630262 110.039812,109.234428 110.237729,108.869845 C110.435646,108.494845 110.706479,108.187553 111.050229,107.94797 C111.300229,107.76047 111.643979,107.60422 112.081479,107.47922 C112.529396,107.343803 113.003354,107.276095 113.503354,107.276095 C114.263771,107.276095 114.930437,107.38547 115.503354,107.60422 C116.076271,107.82297 116.498146,108.119845 116.768979,108.494845 C117.050229,108.869845 117.242937,109.369845 117.347104,109.994845 L115.737729,110.213595 C115.664812,109.713595 115.451271,109.32297 115.097104,109.04172 C114.753354,108.76047 114.263771,108.619845 113.628354,108.619845 C112.867937,108.619845 112.326271,108.744845 112.003354,108.994845 C111.690854,109.244845 111.534604,109.536512 111.534604,109.869845 C111.534604,110.078178 111.597104,110.270887 111.722104,110.44797 C111.857521,110.625053 112.071062,110.770887 112.362729,110.88547 C112.518979,110.94797 112.998146,111.088595 113.800229,111.307345 C114.966896,111.609428 115.779396,111.859428 116.237729,112.057345 C116.696062,112.255262 117.055437,112.54172 117.315854,112.91672 C117.576271,113.29172 117.706479,113.76047 117.706479,114.32297 C117.706479,114.875053 117.545021,115.390678 117.222104,115.869845 C116.909604,116.349012 116.451271,116.724012 115.847104,116.994845 C115.242937,117.255262 114.560646,117.38547 113.800229,117.38547 C112.550229,117.38547 111.591896,117.125053 110.925229,116.60422 C110.268979,116.083387 109.852312,115.307345 109.675229,114.276095 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="15.089237 39.1447 34.522308 39.1447 34.522308 51.428163 15.089237 51.428163"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="183.08923 42.36557 202.522305 42.36557 202.522305 54.649048 183.08923 54.649048"></polygon>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-dns-synonym-ex.svg b/docs/reference/images/analysis/token-graph-dns-synonym-ex.svg
new file mode 100644
index 0000000000000..cff5b1306b73b
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-dns-synonym-ex.svg
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="810px" height="127px" viewBox="0 0 810 127" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="token-graph-dns-synonym-ex" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M167.71559,4.1045 L167.71559,4.1045 C167.71559,2.5803148 168.951186,1.344719 170.475371,1.344719 L212.325881,1.344719 C213.05783,1.344719 213.759795,1.63547523 214.277343,2.1530381 C214.794906,2.67060097 215.085662,3.3725587 215.085662,4.1044999 L215.085662,48.6636799 C215.085662,50.1878651 213.850066,51.4234609 212.325881,51.4234609 L170.475371,51.4234609 C168.951186,51.4234609 167.71559,50.1878652 167.71559,48.6636799 L167.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M167.71559,4.1045 L167.71559,4.1045 C167.71559,2.5803148 168.951186,1.344719 170.475371,1.344719 L212.325881,1.344719 C213.05783,1.344719 213.759795,1.63547523 214.277343,2.1530381 C214.794906,2.67060097 215.085662,3.3725587 215.085662,4.1044999 L215.085662,48.6636799 C215.085662,50.1878651 213.850066,51.4234609 212.325881,51.4234609 L170.475371,51.4234609 C168.951186,51.4234609 167.71559,50.1878652 167.71559,48.6636799 L167.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M193.1659,33.3041 L191.525275,33.3041 L191.525275,22.850967 C191.129442,23.225967 190.608608,23.6061753 189.962775,23.991592 C189.327358,24.366592 188.754442,24.647842 188.244025,24.835342 L188.244025,23.241592 C189.160692,22.8145087 189.962775,22.2936753 190.650275,21.679092 C191.337775,21.0645087 191.82215,20.4707587 192.1034,19.897842 L193.1659,19.897842 L193.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="159.08891 16.85302 178.521985 16.85302 178.521985 38.679795 159.08891 38.679795"></polygon>
+                <line x1="178.52199" y1="27.7664" x2="48.19127" y2="27.2309691" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="160.52213" y1="27.69245" x2="48.19128" y2="27.2309708" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="160.50177 32.64761 174.136307 27.7483867 160.542496 22.7372937"></polygon>
+                <path d="M319.71558,4.1045 L319.71558,4.1045 C319.71558,2.5803148 320.951206,1.344719 322.475376,1.344719 L364.325901,1.344719 C365.057835,1.344719 365.7598,1.63547523 366.277348,2.1530381 C366.794926,2.67060097 367.085667,3.3725587 367.085667,4.1044999 L367.085667,48.6636799 C367.085667,50.1878651 365.850071,51.4234609 364.325901,51.4234609 L322.475376,51.4234609 C320.951206,51.4234609 319.71558,50.1878652 319.71558,48.6636799 L319.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M319.71558,4.1045 L319.71558,4.1045 C319.71558,2.5803148 320.951206,1.344719 322.475376,1.344719 L364.325901,1.344719 C365.057835,1.344719 365.7598,1.63547523 366.277348,2.1530381 C366.794926,2.67060097 367.085667,3.3725587 367.085667,4.1044999 L367.085667,48.6636799 C367.085667,50.1878651 365.850071,51.4234609 364.325901,51.4234609 L322.475376,51.4234609 C320.951206,51.4234609 319.71558,50.1878652 319.71558,48.6636799 L319.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M347.6034,31.72597 L347.6034,33.304095 L338.775275,33.304095 C338.764858,32.9082617 338.827358,32.5280533 338.962775,32.16347 C339.191942,31.5593033 339.551317,30.9655533 340.0409,30.38222 C340.5409,29.7988867 341.25965,29.1270117 342.19715,28.366595 C343.645067,27.1790899 344.624233,26.2363791 345.13465,25.5384624 C345.645067,24.8405457 345.900275,24.1842957 345.900275,23.5697124 C345.900275,22.9134624 345.6659,22.3665874 345.19715,21.9290874 C344.738817,21.4811707 344.13465,21.2572124 343.38465,21.2572124 C342.592983,21.2572124 341.957567,21.4967957 341.4784,21.9759624 C341.00965,22.4447124 340.775275,23.0957541 340.775275,23.9290874 L339.087775,23.7572124 C339.202358,22.4967957 339.63465,21.5384624 340.38465,20.8822124 C341.145067,20.2259624 342.155483,19.8978374 343.4159,19.8978374 C344.69715,19.8978374 345.712775,20.2520041 346.462775,20.9603374 C347.212775,21.6686707 347.587775,22.5488791 347.587775,23.6009624 C347.587775,24.1322124 347.4784,24.6582541 347.25965,25.1790874 C347.0409,25.6999207 346.676317,26.2467957 346.1659,26.8197124 C345.6659,27.3822124 344.82215,28.1634649 343.63465,29.16347 C342.655483,29.9863867 342.025275,30.5488867 341.744025,30.85097 C341.462775,31.1426367 341.233608,31.4343033 341.056525,31.72597 L347.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="207.08891 16.85302 226.52197 16.85302 226.52197 38.679795 207.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="311.0889 16.85302 330.521975 16.85302 330.521975 38.679795 311.0889 38.679795"></polygon>
+                <path d="M487.7156,4.1045 L487.7156,4.1045 C487.7156,2.5803148 488.951196,1.344719 490.475366,1.344719 L532.325891,1.344719 C533.057824,1.344719 533.75979,1.63547523 534.277368,2.1530381 C534.794946,2.67060097 535.085717,3.3725587 535.085717,4.1044999 L535.085717,48.6636799 C535.085717,50.1878651 533.850122,51.4234609 532.325891,51.4234609 L490.475366,51.4234609 C488.951196,51.4234609 487.7156,50.1878652 487.7156,48.6636799 L487.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M487.7156,4.1045 L487.7156,4.1045 C487.7156,2.5803148 488.951196,1.344719 490.475366,1.344719 L532.325891,1.344719 C533.057824,1.344719 533.75979,1.63547523 534.277368,2.1530381 C534.794946,2.67060097 535.085717,3.3725587 535.085717,4.1044999 L535.085717,48.6636799 C535.085717,50.1878651 533.850122,51.4234609 532.325891,51.4234609 L490.475366,51.4234609 C488.951196,51.4234609 487.7156,50.1878652 487.7156,48.6636799 L487.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M506.994,29.77284 L508.634625,29.55409 C508.822125,30.49159 509.139833,31.163465 509.58775,31.569715 C510.046083,31.975965 510.598167,32.17909 511.244,32.17909 C512.014833,32.17909 512.665875,31.913465 513.197125,31.382215 C513.728375,30.850965 513.994,30.1895067 513.994,29.3978324 C513.994,28.64784 513.749208,28.0280458 513.259625,27.5384574 C512.770042,27.0488741 512.145042,26.8040824 511.384625,26.8040824 C511.072125,26.8040824 510.686708,26.8613741 510.228375,26.9759574 L510.415875,25.5384574 C510.520042,25.5488741 510.608583,25.5540824 510.6815,25.5540824 C511.379417,25.5540824 512.004417,25.3717907 512.5565,25.0072074 C513.119,24.6426241 513.40025,24.0853324 513.40025,23.3353324 C513.40025,22.7311657 513.197125,22.2311657 512.790875,21.8353324 C512.384625,21.4394991 511.858583,21.2415824 511.21275,21.2415824 C510.577333,21.2415824 510.046083,21.4447074 509.619,21.8509574 C509.191917,22.2467907 508.921083,22.8457491 508.8065,23.6478324 L507.165875,23.3509574 C507.363792,22.2572074 507.816917,21.4082491 508.52525,20.8040824 C509.233583,20.1999157 510.119,19.8978324 511.1815,19.8978324 C511.910667,19.8978324 512.577333,20.0540824 513.1815,20.3665824 C513.796083,20.6790824 514.264833,21.1061657 514.58775,21.6478324 C514.921083,22.1894991 515.08775,22.7624157 515.08775,23.3665824 C515.08775,23.9394991 514.9315,24.4655407 514.619,24.9447074 C514.3065,25.4134574 513.848167,25.7884574 513.244,26.0697074 C514.035667,26.2572074 514.65025,26.6374157 515.08775,27.2103324 C515.52525,27.7832541 515.744,28.5020067 515.744,29.36659 C515.744,30.52284 515.316917,31.507215 514.46275,32.319715 C513.619,33.132215 512.546083,33.538465 511.244,33.538465 C510.066917,33.538465 509.092958,33.1895067 508.322125,32.49159 C507.551292,31.7936733 507.108583,30.8874233 506.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="479.0889 16.85302 498.521944 16.85302 498.521944 38.679795 479.0889 38.679795"></polygon>
+                <line x1="496.9606" y1="27.00144" x2="369.18106" y2="28.765211" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="478.9623" y1="27.24987" x2="369.18105" y2="28.7652127" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="479.03064 32.20459 492.575257 27.061966 478.89386 22.295143"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="58.385826 10.47244 147.83464 10.47244 147.83464 41.842511 58.385826 41.842511"></polygon>
+                <path d="M80.01599,33.07748 L80.01599,31.85873 C79.4118233,32.8170633 78.5107817,33.29623 77.312865,33.29623 C76.5420317,33.29623 75.8336983,33.0826883 75.187865,32.655605 C74.5420317,32.2285217 74.0420317,31.6347717 73.687865,30.874355 C73.3336983,30.1139383 73.156615,29.2389383 73.156615,28.249355 C73.156615,27.2805999 73.3180733,26.4055974 73.64099,25.6243474 C73.9639067,24.8326807 74.4430733,24.2285141 75.07849,23.8118474 C75.7243233,23.3951807 76.4482817,23.1868474 77.250365,23.1868474 C77.8336983,23.1868474 78.3493233,23.3118474 78.79724,23.5618474 C79.2555733,23.8014307 79.625365,24.1191391 79.906615,24.5149724 L79.906615,19.7180974 L81.54724,19.7180974 L81.54724,33.0774804 L80.01599,33.0774804 L80.01599,33.07748 Z M74.844115,28.249355 C74.844115,29.4889383 75.1045317,30.4160217 75.625365,31.030605 C76.1461983,31.6451883 76.7607817,31.95248 77.469115,31.95248 C78.187865,31.95248 78.79724,31.6608133 79.29724,31.07748 C79.79724,30.48373 80.04724,29.5878967 80.04724,28.38998 C80.04724,27.0670583 79.7920317,26.0983057 79.281615,25.4837224 C78.7711983,24.8587224 78.14099,24.5462224 77.39099,24.5462224 C76.67224,24.5462224 76.0680733,24.8430974 75.57849,25.4368474 C75.0889067,26.0305974 74.844115,26.9680999 74.844115,28.249355 Z M83.516707,28.23373 C83.516707,26.4420583 84.0114987,25.1191391 85.001082,24.2649724 C85.8344153,23.5462224 86.8500403,23.1868474 88.047957,23.1868474 C89.3812903,23.1868474 90.469832,23.6243474 91.313582,24.4993474 C92.157332,25.3639307 92.579207,26.5670583 92.579207,28.10873 C92.579207,29.3483133 92.391707,30.32748 92.016707,31.04623 C91.641707,31.7545633 91.094832,32.3066467 90.376082,32.70248 C89.6677487,33.0983133 88.891707,33.29623 88.047957,33.29623 C86.6937903,33.29623 85.6000403,32.8639383 84.766707,31.999355 C83.9333737,31.124355 83.516707,29.8691467 83.516707,28.23373 Z M85.204207,28.23373 C85.204207,29.4733133 85.469832,30.405605 86.001082,31.030605 C86.5427487,31.6451883 87.2250403,31.95248 88.047957,31.95248 C88.860457,31.95248 89.5375403,31.6451883 90.079207,31.030605 C90.6208737,30.405605 90.891707,29.4576883 90.891707,28.186855 C90.891707,26.9889333 90.6208737,26.0826807 90.079207,25.4680974 C89.5375403,24.8535141 88.860457,24.5462224 88.047957,24.5462224 C87.2250403,24.5462224 86.5427487,24.8535141 86.001082,25.4680974 C85.469832,26.0722641 85.204207,26.9941416 85.204207,28.23373 Z M94.501799,33.07748 L94.501799,23.405597 L95.970549,23.405597 L95.970549,24.764972 C96.2726323,24.2858053 96.673674,23.905597 97.173674,23.624347 C97.6840907,23.3326803 98.2570073,23.186847 98.892424,23.186847 C99.611174,23.186847 100.199716,23.3378887 100.658049,23.639972 C101.116382,23.9316387 101.439299,24.343097 101.626799,24.874347 C102.397632,23.749347 103.392424,23.186847 104.611174,23.186847 C105.579924,23.186847 106.319507,23.452472 106.829924,23.983722 C107.350757,24.514972 107.611174,25.3326803 107.611174,26.436847 L107.611174,33.07748 L105.970549,33.07748 L105.970549,26.983722 C105.970549,26.327472 105.918466,25.858722 105.814299,25.577472 C105.710132,25.2858053 105.517424,25.0514303 105.236174,24.874347 C104.954924,24.6972637 104.626799,24.608722 104.251799,24.608722 C103.574716,24.608722 103.012216,24.8378887 102.564299,25.296222 C102.116382,25.7441387 101.892424,26.4628887 101.892424,27.452472 L101.892424,33.07748 L100.251799,33.07748 L100.251799,26.796222 C100.251799,26.0670553 100.116382,25.5201803 99.845549,25.155597 C99.5747157,24.7910137 99.1372157,24.608722 98.533049,24.608722 C98.0747157,24.608722 97.6476323,24.7285137 97.251799,24.968097 C96.8559657,25.2076803 96.5695073,25.561847 96.392424,26.030597 C96.2257573,26.499347 96.142424,27.1764329 96.142424,28.0618546 L96.142424,33.07748 L94.501799,33.07748 L94.501799,33.07748 Z M116.355101,31.88998 C115.740518,32.4003967 115.151976,32.76498 114.589476,32.98373 C114.037393,33.1920633 113.438434,33.29623 112.792601,33.29623 C111.730101,33.29623 110.912393,33.0358133 110.339476,32.51498 C109.766559,31.9941467 109.480101,31.3326883 109.480101,30.530605 C109.480101,30.0514383 109.589476,29.6191467 109.808226,29.23373 C110.026976,28.8378967 110.308226,28.5253967 110.651976,28.29623 C111.006143,28.0566416 111.401976,27.8743474 111.839476,27.7493474 C112.151976,27.6660141 112.636351,27.5826807 113.292601,27.4993474 C114.615518,27.3430974 115.589476,27.1555974 116.214476,26.9368474 C116.224893,26.7076807 116.230101,26.5670557 116.230101,26.5149724 C116.230101,25.8483057 116.073851,25.3743474 115.761351,25.0930974 C115.344684,24.7285141 114.719684,24.5462224 113.886351,24.5462224 C113.115518,24.5462224 112.547809,24.6816391 112.183226,24.9524724 C111.818643,25.2233057 111.547809,25.6972641 111.370726,26.3743474 L109.761351,26.1555974 C109.907184,25.4785141 110.146768,24.9316391 110.480101,24.5149724 C110.813434,24.0878891 111.297809,23.7597641 111.933226,23.5305974 C112.568643,23.3014307 113.297809,23.1868474 114.120726,23.1868474 C114.954059,23.1868474 115.625934,23.2858057 116.136351,23.4837224 C116.657184,23.6712224 117.037393,23.9160141 117.276976,24.2180974 C117.526976,24.5097641 117.698851,24.8795557 117.792601,25.3274724 C117.844684,25.6087224 117.870726,26.1139307 117.870726,26.8430974 L117.870726,29.030605 C117.870726,30.5514383 117.907184,31.51498 117.980101,31.92123 C118.053018,32.3170633 118.188434,32.70248 118.386351,33.07748 L116.683226,33.07748 C116.506143,32.73373 116.396768,32.3378967 116.355101,31.88998 Z M116.214476,28.218105 C115.620726,28.468105 114.730101,28.6764383 113.542601,28.843105 C112.865518,28.936855 112.386351,29.04623 112.105101,29.17123 C111.823851,29.29623 111.605101,29.4733133 111.448851,29.70248 C111.303018,29.9316467 111.230101,30.1920633 111.230101,30.48373 C111.230101,30.92123 111.396768,31.2858133 111.730101,31.57748 C112.063434,31.8691467 112.547809,32.01498 113.183226,32.01498 C113.808226,32.01498 114.365518,31.8795633 114.855101,31.60873 C115.355101,31.32748 115.719684,30.9472717 115.948851,30.468105 C116.125934,30.093105 116.214476,29.54623 116.214476,28.82748 L116.214476,28.218105 Z M120.418318,21.6087224 L120.418318,19.7180974 L122.058943,19.7180974 L122.058943,21.6087224 L120.418318,21.6087224 Z M120.418318,33.0774804 L120.418318,23.4055974 L122.058943,23.4055974 L122.058943,33.0774804 L120.418318,33.0774804 Z M124.563139,33.0774804 L124.563139,23.4055974 L126.031889,23.4055974 L126.031889,24.7805974 C126.740222,23.7180974 127.766264,23.1868474 129.110014,23.1868474 C129.693347,23.1868474 130.229806,23.2910141 130.719389,23.4993474 C131.208972,23.7076807 131.573556,23.9837224 131.813139,24.3274724 C132.063139,24.6608057 132.235014,25.0618474 132.328764,25.5305974 C132.391264,25.8326807 132.422514,26.3639307 132.422514,27.1243474 L132.422514,33.0774804 L130.781889,33.0774804 L130.781889,27.1868474 C130.781889,26.5201807 130.714181,26.0253891 130.578764,25.7024724 C130.453764,25.3691391 130.229806,25.1035141 129.906889,24.9055974 C129.583972,24.7076807 129.203764,24.6087224 128.766264,24.6087224 C128.068347,24.6087224 127.464181,24.8326807 126.953764,25.2805974 C126.453764,25.7180974 126.203764,26.5566391 126.203764,27.7962224 L126.203764,33.0774804 L124.563139,33.0774804 L124.563139,33.0774804 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="383 10.47244 466.653534 10.47244 466.653534 41.842511 383 41.842511"></polygon>
+                <path d="M395.8628,30.1868524 L397.4878,29.93686 C397.571133,30.5826933 397.821133,31.0826933 398.2378,31.43686 C398.654467,31.78061 399.2378,31.952485 399.9878,31.952485 C400.7378,31.952485 401.295092,31.8014433 401.659675,31.49936 C402.024258,31.1868524 402.20655,30.8222767 402.20655,30.40561 C402.20655,30.0410267 402.045092,29.74936 401.722175,29.53061 C401.503425,29.3847767 400.946133,29.202485 400.0503,28.983735 C398.841967,28.671235 398.003425,28.40561 397.534675,28.1868524 C397.076342,27.9681049 396.727383,27.666019 396.4878,27.2806024 C396.248217,26.8847691 396.128425,26.4472691 396.128425,25.9681024 C396.128425,25.5410191 396.227383,25.1451857 396.4253,24.7806024 C396.623217,24.4056024 396.89405,24.0983107 397.2378,23.8587274 C397.4878,23.6712274 397.83155,23.5149774 398.26905,23.3899774 C398.716967,23.2545607 399.190925,23.1868524 399.690925,23.1868524 C400.451342,23.1868524 401.118008,23.2962274 401.690925,23.5149774 C402.263842,23.7337274 402.685717,24.0306024 402.95655,24.4056024 C403.2378,24.7806024 403.430508,25.2806024 403.534675,25.9056024 L401.9253,26.1243524 C401.852383,25.6243524 401.638842,25.2337274 401.284675,24.9524774 C400.940925,24.6712274 400.451342,24.5306024 399.815925,24.5306024 C399.055508,24.5306024 398.513842,24.6556024 398.190925,24.9056024 C397.878425,25.1556024 397.722175,25.4472691 397.722175,25.7806024 C397.722175,25.9889357 397.784675,26.1816441 397.909675,26.3587274 C398.045092,26.5358107 398.258633,26.6816441 398.5503,26.7962274 C398.70655,26.8587274 399.185717,26.9993524 399.9878,27.2181024 C401.154467,27.5201857 401.966967,27.7701857 402.4253,27.9681024 C402.883633,28.1660242 403.243008,28.452485 403.503425,28.827485 C403.763842,29.202485 403.89405,29.671235 403.89405,30.233735 C403.89405,30.7858183 403.732592,31.3014433 403.409675,31.78061 C403.097175,32.2597767 402.638842,32.6347767 402.034675,32.90561 C401.430508,33.1660267 400.748217,33.296235 399.9878,33.296235 C398.7378,33.296235 397.779467,33.0358183 397.1128,32.514985 C396.45655,31.9941517 396.039883,31.21811 395.8628,30.1868524 Z M405.76905,36.796235 L405.58155,35.264985 C405.946133,35.358735 406.258633,35.405602 406.51905,35.405602 C406.883633,35.405602 407.1753,35.34311 407.39405,35.21811 C407.6128,35.1035267 407.795092,34.93686 407.940925,34.71811 C408.045092,34.5514433 408.211758,34.1451933 408.440925,33.49936 C408.472175,33.405602 408.51905,33.2701933 408.58155,33.09311 L404.909675,23.405602 L406.6753,23.405602 L408.690925,28.9993596 C408.951342,29.7181096 409.185717,30.4681096 409.39405,31.2493596 C409.58155,30.4993596 409.805508,29.7649846 410.065925,29.0462346 L412.14405,23.405602 L413.784675,23.405602 L410.097175,33.233735 C409.701342,34.3066517 409.39405,35.0410267 409.1753,35.43686 C408.883633,35.9785267 408.5503,36.37436 408.1753,36.62436 C407.8003,36.87436 407.352383,36.99936 406.83155,36.99936 C406.508633,36.99936 406.154467,36.9316517 405.76905,36.796235 Z M414.51905,30.1868524 L416.14405,29.93686 C416.227383,30.5826933 416.477383,31.0826933 416.89405,31.43686 C417.310717,31.78061 417.89405,31.952485 418.64405,31.952485 C419.39405,31.952485 419.951342,31.8014433 420.315925,31.49936 C420.680508,31.1868524 420.8628,30.8222767 420.8628,30.40561 C420.8628,30.0410267 420.701342,29.74936 420.378425,29.53061 C420.159675,29.3847767 419.602383,29.202485 418.70655,28.983735 C417.498217,28.671235 416.659675,28.40561 416.190925,28.1868524 C415.732592,27.9681049 415.383633,27.666019 415.14405,27.2806024 C414.904467,26.8847691 414.784675,26.4472691 414.784675,25.9681024 C414.784675,25.5410191 414.883633,25.1451857 415.08155,24.7806024 C415.279467,24.4056024 415.5503,24.0983107 415.89405,23.8587274 C416.14405,23.6712274 416.4878,23.5149774 416.9253,23.3899774 C417.373217,23.2545607 417.847175,23.1868524 418.347175,23.1868524 C419.107592,23.1868524 419.774258,23.2962274 420.347175,23.5149774 C420.920092,23.7337274 421.341967,24.0306024 421.6128,24.4056024 C421.89405,24.7806024 422.086758,25.2806024 422.190925,25.9056024 L420.58155,26.1243524 C420.508633,25.6243524 420.295092,25.2337274 419.940925,24.9524774 C419.597175,24.6712274 419.107592,24.5306024 418.472175,24.5306024 C417.711758,24.5306024 417.170092,24.6556024 416.847175,24.9056024 C416.534675,25.1556024 416.378425,25.4472691 416.378425,25.7806024 C416.378425,25.9889357 416.440925,26.1816441 416.565925,26.3587274 C416.701342,26.5358107 416.914883,26.6816441 417.20655,26.7962274 C417.3628,26.8587274 417.841967,26.9993524 418.64405,27.2181024 C419.810717,27.5201857 420.623217,27.7701857 421.08155,27.9681024 C421.539883,28.1660242 421.899258,28.452485 422.159675,28.827485 C422.420092,29.202485 422.5503,29.671235 422.5503,30.233735 C422.5503,30.7858183 422.388842,31.3014433 422.065925,31.78061 C421.753425,32.2597767 421.295092,32.6347767 420.690925,32.90561 C420.086758,33.1660267 419.404467,33.296235 418.64405,33.296235 C417.39405,33.296235 416.435717,33.0358183 415.76905,32.514985 C415.1128,31.9941517 414.696133,31.21811 414.51905,30.1868524 Z M428.08155,31.608735 L428.315925,33.06186 C427.857592,33.15561 427.446133,33.202485 427.08155,33.202485 C426.4878,33.202485 426.024258,33.108735 425.690925,32.921235 C425.368008,32.733735 425.138842,32.4889433 425.003425,32.18686 C424.868008,31.87436 424.8003,31.2285267 424.8003,30.24936 L424.8003,24.6712274 L423.597175,24.6712274 L423.597175,23.4056024 L424.8003,23.4056024 L424.8003,21.0149774 L426.4253,20.0306024 L426.4253,23.4056024 L428.08155,23.4056024 L428.08155,24.6712274 L426.4253,24.6712274 L426.4253,30.34311 C426.4253,30.8014433 426.451342,31.0983183 426.503425,31.233735 C426.565925,31.3691517 426.659675,31.4785267 426.784675,31.56186 C426.920092,31.6347767 427.1128,31.671235 427.3628,31.671235 C427.539883,31.671235 427.779467,31.6504017 428.08155,31.608735 Z M436.311744,29.96811 L437.999244,30.171235 C437.728411,31.1608183 437.233619,31.9316517 436.514869,32.483735 C435.796119,33.0254017 434.874244,33.296235 433.749244,33.296235 C432.332577,33.296235 431.207577,32.8639433 430.374244,31.99936 C429.551327,31.12436 429.139869,29.9004017 429.139869,28.327485 C429.139869,26.6920633 429.556536,25.4264357 430.389869,24.5306024 C431.233619,23.6347691 432.322161,23.1868524 433.655494,23.1868524 C434.947161,23.1868524 435.999244,23.6295607 436.811744,24.5149774 C437.634661,25.3899774 438.046119,26.6243549 438.046119,28.21811 C438.046119,28.3222767 438.046119,28.46811 438.046119,28.65561 L430.827369,28.65561 C430.889869,29.71811 431.191952,30.5358183 431.733619,31.108735 C432.275286,31.671235 432.947161,31.952485 433.749244,31.952485 C434.353411,31.952485 434.869036,31.796235 435.296119,31.483735 C435.723202,31.1608183 436.061744,30.65561 436.311744,29.96811 Z M430.921119,27.3118524 L436.327369,27.3118524 C436.254452,26.4993524 436.046119,25.8899774 435.702369,25.4837274 C435.181536,24.8483107 434.504452,24.5306024 433.671119,24.5306024 C432.921119,24.5306024 432.285702,24.7858107 431.764869,25.2962274 C431.254452,25.7962274 430.973202,26.4681024 430.921119,27.3118524 Z M440.062446,33.077485 L440.062446,23.405602 L441.531196,23.405602 L441.531196,24.764977 C441.833279,24.2858103 442.234321,23.905602 442.734321,23.624352 C443.244738,23.3326853 443.817654,23.186852 444.453071,23.186852 C445.171821,23.186852 445.760363,23.3378937 446.218696,23.639977 C446.677029,23.9316437 446.999946,24.343102 447.187446,24.874352 C447.958279,23.749352 448.953071,23.186852 450.171821,23.186852 C451.140571,23.186852 451.880154,23.452477 452.390571,23.983727 C452.911404,24.514977 453.171821,25.3326853 453.171821,26.436852 L453.171821,33.077485 L451.531196,33.077485 L451.531196,26.983727 C451.531196,26.327477 451.479113,25.858727 451.374946,25.577477 C451.270779,25.2858103 451.078071,25.0514353 450.796821,24.874352 C450.515571,24.6972687 450.187446,24.608727 449.812446,24.608727 C449.135363,24.608727 448.572863,24.8378937 448.124946,25.296227 C447.677029,25.7441437 447.453071,26.4628937 447.453071,27.452477 L447.453071,33.077485 L445.812446,33.077485 L445.812446,26.796227 C445.812446,26.0670603 445.677029,25.5201853 445.406196,25.155602 C445.135363,24.7910187 444.697863,24.608727 444.093696,24.608727 C443.635363,24.608727 443.208279,24.7285187 442.812446,24.968102 C442.416613,25.2076853 442.130154,25.561852 441.953071,26.030602 C441.786404,26.499352 441.703071,27.1764379 441.703071,28.0618596 L441.703071,33.077485 L440.062446,33.077485 L440.062446,33.077485 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M607.7156,4.1045 L607.7156,4.1045 C607.7156,2.5803148 608.951196,1.344719 610.475366,1.344719 L652.325891,1.344719 C653.057824,1.344719 653.75979,1.63547523 654.277368,2.1530381 C654.794946,2.67060097 655.085717,3.3725587 655.085717,4.1044999 L655.085717,48.6636799 C655.085717,50.1878651 653.850122,51.4234609 652.325891,51.4234609 L610.475366,51.4234609 C608.951196,51.4234609 607.7156,50.1878652 607.7156,48.6636799 L607.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M607.7156,4.1045 L607.7156,4.1045 C607.7156,2.5803148 608.951196,1.344719 610.475366,1.344719 L652.325891,1.344719 C653.057824,1.344719 653.75979,1.63547523 654.277368,2.1530381 C654.794946,2.67060097 655.085717,3.3725587 655.085717,4.1044999 L655.085717,48.6636799 C655.085717,50.1878651 653.850122,51.4234609 652.325891,51.4234609 L610.475366,51.4234609 C608.951196,51.4234609 607.7156,50.1878652 607.7156,48.6636799 L607.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M632.244,33.3041 L632.244,30.100975 L626.447125,30.100975 L626.447125,28.600975 L632.540875,19.944717 L633.884625,19.944717 L633.884625,28.600975 L635.6815,28.600975 L635.6815,30.100975 L633.884625,30.100975 L633.884625,33.3041 L632.244,33.3041 Z M632.244,28.600975 L632.244,22.5853424 L628.0565,28.600975 L632.244,28.600975 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="527.0889 16.85302 546.521944 16.85302 546.521944 38.679795 527.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="599.0889 16.85302 618.521944 16.85302 618.521944 38.679795 599.0889 38.679795"></polygon>
+                <line x1="618.522" y1="27.7664" x2="533.1362" y2="28.7427794" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="600.5232" y1="27.97221" x2="533.136176" y2="28.7427789" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="600.57983 32.9271 614.136532 27.8165577 600.466488 23.0173557"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.8793 12.06562 588.603726 12.06562 588.603726 43.435699 543.8793 43.435699"></polygon>
+                <path d="M560.7394,23.201904 L560.7394,21.311279 L562.380025,21.311279 L562.380025,23.201904 L560.7394,23.201904 Z M560.7394,34.670654 L560.7394,24.998779 L562.380025,24.998779 L562.380025,34.670654 L560.7394,34.670654 Z M564.227986,31.780029 L565.852986,31.530029 C565.93632,32.1758623 566.18632,32.6758623 566.602986,33.030029 C567.019653,33.373779 567.602986,33.545654 568.352986,33.545654 C569.102986,33.545654 569.660278,33.3946123 570.024861,33.092529 C570.389445,32.780029 570.571736,32.4154457 570.571736,31.998779 C570.571736,31.6341957 570.410278,31.342529 570.087361,31.123779 C569.868611,30.9779457 569.31132,30.795654 568.415486,30.576904 C567.207153,30.264404 566.368611,29.998779 565.899861,29.780029 C565.441528,29.561279 565.09257,29.2591957 564.852986,28.873779 C564.613403,28.4779457 564.493611,28.0404457 564.493611,27.561279 C564.493611,27.1341957 564.59257,26.7383623 564.790486,26.373779 C564.988403,25.998779 565.259236,25.6914873 565.602986,25.451904 C565.852986,25.264404 566.196736,25.108154 566.634236,24.983154 C567.082153,24.8477373 567.556111,24.780029 568.056111,24.780029 C568.816528,24.780029 569.483195,24.889404 570.056111,25.108154 C570.629028,25.326904 571.050903,25.623779 571.321736,25.998779 C571.602986,26.373779 571.795695,26.873779 571.899861,27.498779 L570.290486,27.717529 C570.21757,27.217529 570.004028,26.826904 569.649861,26.545654 C569.306111,26.264404 568.816528,26.123779 568.181111,26.123779 C567.420695,26.123779 566.879028,26.248779 566.556111,26.498779 C566.243611,26.748779 566.087361,27.0404457 566.087361,27.373779 C566.087361,27.5821123 566.149861,27.7748207 566.274861,27.951904 C566.410278,28.1289873 566.62382,28.2748207 566.915486,28.389404 C567.071736,28.451904 567.550903,28.592529 568.352986,28.811279 C569.519653,29.1133623 570.332153,29.3633623 570.790486,29.561279 C571.24882,29.7591957 571.608195,30.045654 571.868611,30.420654 C572.129028,30.795654 572.259236,31.264404 572.259236,31.826904 C572.259236,32.3789873 572.097778,32.8946123 571.774861,33.373779 C571.462361,33.8529457 571.004028,34.2279457 570.399861,34.498779 C569.795695,34.7591957 569.113403,34.889404 568.352986,34.889404 C567.102986,34.889404 566.144653,34.6289873 565.477986,34.108154 C564.821736,33.5873207 564.40507,32.811279 564.227986,31.780029 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M759.7156,4.1045 L759.7156,4.1045 C759.7156,2.5803148 760.951196,1.344719 762.475366,1.344719 L804.325891,1.344719 C805.057824,1.344719 805.75979,1.63547523 806.277368,2.1530381 C806.794946,2.67060097 807.085717,3.3725587 807.085717,4.1044999 L807.085717,48.6636799 C807.085717,50.1878651 805.850122,51.4234609 804.325891,51.4234609 L762.475366,51.4234609 C760.951196,51.4234609 759.7156,50.1878652 759.7156,48.6636799 L759.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M759.7156,4.1045 L759.7156,4.1045 C759.7156,2.5803148 760.951196,1.344719 762.475366,1.344719 L804.325891,1.344719 C805.057824,1.344719 805.75979,1.63547523 806.277368,2.1530381 C806.794946,2.67060097 807.085717,3.3725587 807.085717,4.1044999 L807.085717,48.6636799 C807.085717,50.1878651 805.850122,51.4234609 804.325891,51.4234609 L762.475366,51.4234609 C760.951196,51.4234609 759.7156,50.1878652 759.7156,48.6636799 L759.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M778.994,29.8041 L780.71275,29.663475 C780.83775,30.4968083 781.129417,31.1270167 781.58775,31.5541 C782.0565,31.9707667 782.619,32.1791 783.27525,32.1791 C784.066917,32.1791 784.733583,31.882225 785.27525,31.288475 C785.827333,30.694725 786.103375,29.9082667 786.103375,28.9291 C786.103375,27.9915949 785.83775,27.2520091 785.3065,26.7103424 C784.785667,26.1686757 784.098167,25.8978424 783.244,25.8978424 C782.723167,25.8978424 782.249208,26.0176341 781.822125,26.2572174 C781.395042,26.4968007 781.061708,26.8093007 780.822125,27.1947174 L779.27525,26.9915924 L780.572125,20.1322174 L787.21275,20.1322174 L787.21275,21.6947174 L781.884625,21.6947174 L781.165875,25.2884674 C781.967958,24.7259674 782.8065,24.4447174 783.6815,24.4447174 C784.848167,24.4447174 785.832542,24.8509674 786.634625,25.6634674 C787.436708,26.4655507 787.83775,27.5020116 787.83775,28.77285 C787.83775,29.9811833 787.488792,31.0280583 786.790875,31.913475 C785.926292,32.9968083 784.754417,33.538475 783.27525,33.538475 C782.066917,33.538475 781.077333,33.1999333 780.3065,32.52285 C779.535667,31.83535 779.098167,30.9291 778.994,29.8041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="647.0889 16.85302 666.521944 16.85302 666.521944 38.679795 647.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="751.0889 16.85302 770.521944 16.85302 770.521944 38.679795 751.0889 38.679795"></polygon>
+                <line x1="770.522" y1="27.7664" x2="657.38815" y2="29.5616728" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="752.5243" y1="28.052" x2="657.38813" y2="29.5616741" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="752.60284 33.00658 766.136837 27.8359944 752.445614 23.0974309"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="669.7087 10.47244 740.606344 10.47244 740.606344 41.842511 669.7087 41.842511"></polygon>
+                <path d="M681.3761,33.07748 L681.3761,24.671222 L679.922975,24.671222 L679.922975,23.405597 L681.3761,23.405597 L681.3761,22.374347 C681.3761,21.7285137 681.433392,21.2441387 681.547975,20.921222 C681.704225,20.4941387 681.980267,20.1503887 682.3761,19.889972 C682.771933,19.6295553 683.329225,19.499347 684.047975,19.499347 C684.506308,19.499347 685.016725,19.5514303 685.579225,19.655597 L685.329225,21.093097 C684.995892,21.030597 684.678183,20.999347 684.3761,20.999347 C683.8761,20.999347 683.521933,21.108722 683.3136,21.327472 C683.105267,21.5358053 683.0011,21.9316387 683.0011,22.514972 L683.0011,23.405597 L684.891725,23.405597 L684.891725,24.671222 L683.0011,24.671222 L683.0011,33.07748 L681.3761,33.07748 Z M686.153139,33.07748 L686.153139,23.405597 L687.621889,23.405597 L687.621889,24.874347 C687.996889,24.186847 688.340639,23.733722 688.653139,23.514972 C688.976055,23.296222 689.330222,23.186847 689.715639,23.186847 C690.267722,23.186847 690.830222,23.3639303 691.403139,23.718097 L690.840639,25.233722 C690.434389,24.9941387 690.033347,24.874347 689.637514,24.874347 C689.27293,24.874347 688.950014,24.983722 688.668764,25.202472 C688.387514,25.421222 688.184389,25.718097 688.059389,26.093097 C687.871889,26.6764303 687.778139,27.3170579 687.778139,28.0149796 L687.778139,33.07748 L686.153139,33.07748 L686.153139,33.07748 Z M698.693971,31.88998 C698.079388,32.4003967 697.490846,32.76498 696.928346,32.98373 C696.376263,33.1920633 695.777304,33.29623 695.131471,33.29623 C694.068971,33.29623 693.251263,33.0358133 692.678346,32.51498 C692.105429,31.9941467 691.818971,31.3326883 691.818971,30.530605 C691.818971,30.0514383 691.928346,29.6191467 692.147096,29.23373 C692.365846,28.8378967 692.647096,28.5253967 692.990846,28.29623 C693.345013,28.0566416 693.740846,27.8743474 694.178346,27.7493474 C694.490846,27.6660141 694.975221,27.5826807 695.631471,27.4993474 C696.954388,27.3430974 697.928346,27.1555974 698.553346,26.9368474 C698.563763,26.7076807 698.568971,26.5670557 698.568971,26.5149724 C698.568971,25.8483057 698.412721,25.3743474 698.100221,25.0930974 C697.683554,24.7285141 697.058554,24.5462224 696.225221,24.5462224 C695.454388,24.5462224 694.886679,24.6816391 694.522096,24.9524724 C694.157513,25.2233057 693.886679,25.6972641 693.709596,26.3743474 L692.100221,26.1555974 C692.246054,25.4785141 692.485638,24.9316391 692.818971,24.5149724 C693.152304,24.0878891 693.636679,23.7597641 694.272096,23.5305974 C694.907513,23.3014307 695.636679,23.1868474 696.459596,23.1868474 C697.292929,23.1868474 697.964804,23.2858057 698.475221,23.4837224 C698.996054,23.6712224 699.376263,23.9160141 699.615846,24.2180974 C699.865846,24.5097641 700.037721,24.8795557 700.131471,25.3274724 C700.183554,25.6087224 700.209596,26.1139307 700.209596,26.8430974 L700.209596,29.030605 C700.209596,30.5514383 700.246054,31.51498 700.318971,31.92123 C700.391888,32.3170633 700.527304,32.70248 700.725221,33.07748 L699.022096,33.07748 C698.845013,32.73373 698.735638,32.3378967 698.693971,31.88998 Z M698.553346,28.218105 C697.959596,28.468105 697.068971,28.6764383 695.881471,28.843105 C695.204388,28.936855 694.725221,29.04623 694.443971,29.17123 C694.162721,29.29623 693.943971,29.4733133 693.787721,29.70248 C693.641888,29.9316467 693.568971,30.1920633 693.568971,30.48373 C693.568971,30.92123 693.735638,31.2858133 694.068971,31.57748 C694.402304,31.8691467 694.886679,32.01498 695.522096,32.01498 C696.147096,32.01498 696.704388,31.8795633 697.193971,31.60873 C697.693971,31.32748 698.058554,30.9472717 698.287721,30.468105 C698.464804,30.093105 698.553346,29.54623 698.553346,28.82748 L698.553346,28.218105 Z M702.444642,33.874355 L704.038392,34.10873 C704.111309,34.60873 704.298809,34.968105 704.600892,35.1868474 C705.007142,35.4889383 705.564434,35.63998 706.272767,35.63998 C707.033184,35.63998 707.616517,35.4889383 708.022767,35.1868474 C708.439434,34.8847717 708.720684,34.4628967 708.866517,33.92123 C708.949851,33.5878967 708.986309,32.8847717 708.975892,31.811855 C708.267559,32.655605 707.376934,33.07748 706.304017,33.07748 C704.970684,33.07748 703.939434,32.5983133 703.210267,31.63998 C702.481101,30.6816467 702.116517,29.530605 702.116517,28.1868474 C702.116517,27.2597666 702.283184,26.4055974 702.616517,25.6243474 C702.960267,24.8430974 703.444642,24.2441391 704.069642,23.8274724 C704.705059,23.4003891 705.455059,23.1868474 706.319642,23.1868474 C707.455059,23.1868474 708.392559,23.6451807 709.132142,24.5618474 L709.132142,23.4055974 L710.647767,23.4055974 L710.647767,31.7649804 C710.647767,33.2753971 710.491517,34.3431054 710.179017,34.9681054 C709.876934,35.5931054 709.392559,36.0878971 708.725892,36.4524804 C708.069642,36.8170637 707.251934,36.9993554 706.272767,36.9993554 C705.126934,36.9993554 704.194642,36.7389387 703.475892,36.2181054 C702.767559,35.7076887 702.423809,34.9264387 702.444642,33.8743554 L702.444642,33.874355 Z M703.804017,28.061855 C703.804017,29.3326883 704.054017,30.2597717 704.554017,30.843105 C705.064434,31.4264383 705.699851,31.718105 706.460267,31.718105 C707.210267,31.718105 707.840476,31.4316467 708.350892,30.85873 C708.861309,30.2753967 709.116517,29.3639383 709.116517,28.124355 C709.116517,26.9368499 708.850892,26.0462224 708.319642,25.4524724 C707.798809,24.8483057 707.168601,24.5462224 706.429017,24.5462224 C705.699851,24.5462224 705.080059,24.8430974 704.569642,25.4368474 C704.059226,26.0201807 703.804017,26.8951833 703.804017,28.061855 Z M713.132874,21.6087224 L713.132874,19.7180974 L714.773499,19.7180974 L714.773499,21.6087224 L713.132874,21.6087224 Z M713.132874,33.0774804 L713.132874,23.4055974 L714.773499,23.4055974 L714.773499,33.0774804 L713.132874,33.0774804 Z M717.230836,33.0774804 L717.230836,19.7180974 L718.871461,19.7180974 L718.871461,33.0774804 L717.230836,33.0774804 Z M728.047547,29.9681054 L729.735047,30.1712304 C729.464214,31.1608137 728.969422,31.9316471 728.250672,32.4837304 C727.531922,33.0253971 726.610047,33.2962304 725.485047,33.2962304 C724.06838,33.2962304 722.94338,32.8639387 722.110047,31.9993554 C721.28713,31.1243554 720.875672,29.9003971 720.875672,28.3274804 C720.875672,26.6920587 721.292339,25.4264311 722.125672,24.5305978 C722.969422,23.6347645 724.057964,23.1868478 725.391297,23.1868478 C726.682964,23.1868478 727.735047,23.6295561 728.547547,24.5149728 C729.370464,25.3899728 729.781922,26.6243503 729.781922,28.2181054 C729.781922,28.3222721 729.781922,28.4681054 729.781922,28.6556054 L722.563172,28.6556054 C722.625672,29.7181054 722.927755,30.5358137 723.469422,31.1087304 C724.011089,31.6712304 724.682964,31.9524804 725.485047,31.9524804 C726.089214,31.9524804 726.604839,31.7962304 727.031922,31.4837304 C727.459005,31.1608137 727.797547,30.6556054 728.047547,29.9681054 Z M722.656922,27.3118478 L728.063172,27.3118478 C727.990255,26.4993478 727.781922,25.8899728 727.438172,25.4837228 C726.917339,24.8483061 726.240255,24.5305978 725.406922,24.5305978 C724.656922,24.5305978 724.021505,24.7858061 723.500672,25.2962228 C722.990255,25.7962228 722.709005,26.4680978 722.656922,27.3118478 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <line x1="330.52197" y1="27.7664" x2="217.45111" y2="27.7664" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="312.52197" y1="27.7664" x2="217.45111" y2="27.7664" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="312.52197 32.7216 326.136258 27.7664 312.52197 22.8112074"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="229.4882 10.47244 300.385844 10.47244 300.385844 41.842511 229.4882 41.842511"></polygon>
+                <path d="M242.8374,33.07748 L242.8374,23.405597 L244.30615,23.405597 L244.30615,24.780597 C245.014483,23.718097 246.040525,23.186847 247.384275,23.186847 C247.967608,23.186847 248.504067,23.2910137 248.99365,23.499347 C249.483233,23.7076803 249.847817,23.983722 250.0874,24.327472 C250.3374,24.6608053 250.509275,25.061847 250.603025,25.530597 C250.665525,25.8326803 250.696775,26.3639303 250.696775,27.124347 L250.696775,33.07748 L249.05615,33.07748 L249.05615,27.186847 C249.05615,26.5201803 248.988442,26.0253887 248.853025,25.702472 C248.728025,25.3691387 248.504067,25.1035137 248.18115,24.905597 C247.858233,24.7076803 247.478025,24.608722 247.040525,24.608722 C246.342608,24.608722 245.738442,24.8326803 245.228025,25.280597 C244.728025,25.718097 244.478025,26.5566387 244.478025,27.796222 L244.478025,33.07748 L242.8374,33.07748 L242.8374,33.07748 Z M259.525632,31.88998 C258.911049,32.4003967 258.322507,32.76498 257.760007,32.98373 C257.207924,33.1920633 256.608965,33.29623 255.963132,33.29623 C254.900632,33.29623 254.082924,33.0358133 253.510007,32.51498 C252.93709,31.9941467 252.650632,31.3326883 252.650632,30.530605 C252.650632,30.0514383 252.760007,29.6191467 252.978757,29.23373 C253.197507,28.8378967 253.478757,28.5253967 253.822507,28.29623 C254.176674,28.0566416 254.572507,27.8743474 255.010007,27.7493474 C255.322507,27.6660141 255.806882,27.5826807 256.463132,27.4993474 C257.786049,27.3430974 258.760007,27.1555974 259.385007,26.9368474 C259.395424,26.7076807 259.400632,26.5670557 259.400632,26.5149724 C259.400632,25.8483057 259.244382,25.3743474 258.931882,25.0930974 C258.515215,24.7285141 257.890215,24.5462224 257.056882,24.5462224 C256.286049,24.5462224 255.71834,24.6816391 255.353757,24.9524724 C254.989174,25.2233057 254.71834,25.6972641 254.541257,26.3743474 L252.931882,26.1555974 C253.077715,25.4785141 253.317299,24.9316391 253.650632,24.5149724 C253.983965,24.0878891 254.46834,23.7597641 255.103757,23.5305974 C255.739174,23.3014307 256.46834,23.1868474 257.291257,23.1868474 C258.12459,23.1868474 258.796465,23.2858057 259.306882,23.4837224 C259.827715,23.6712224 260.207924,23.9160141 260.447507,24.2180974 C260.697507,24.5097641 260.869382,24.8795557 260.963132,25.3274724 C261.015215,25.6087224 261.041257,26.1139307 261.041257,26.8430974 L261.041257,29.030605 C261.041257,30.5514383 261.077715,31.51498 261.150632,31.92123 C261.223549,32.3170633 261.358965,32.70248 261.556882,33.07748 L259.853757,33.07748 C259.676674,32.73373 259.567299,32.3378967 259.525632,31.88998 Z M259.385007,28.218105 C258.791257,28.468105 257.900632,28.6764383 256.713132,28.843105 C256.036049,28.936855 255.556882,29.04623 255.275632,29.17123 C254.994382,29.29623 254.775632,29.4733133 254.619382,29.70248 C254.473549,29.9316467 254.400632,30.1920633 254.400632,30.48373 C254.400632,30.92123 254.567299,31.2858133 254.900632,31.57748 C255.233965,31.8691467 255.71834,32.01498 256.353757,32.01498 C256.978757,32.01498 257.536049,31.8795633 258.025632,31.60873 C258.525632,31.32748 258.890215,30.9472717 259.119382,30.468105 C259.296465,30.093105 259.385007,29.54623 259.385007,28.82748 L259.385007,28.218105 Z M263.588834,33.07748 L263.588834,23.405597 L265.057584,23.405597 L265.057584,24.764972 C265.359667,24.2858053 265.760709,23.905597 266.260709,23.624347 C266.771126,23.3326803 267.344042,23.186847 267.979459,23.186847 C268.698209,23.186847 269.286751,23.3378887 269.745084,23.639972 C270.203417,23.9316387 270.526334,24.343097 270.713834,24.874347 C271.484667,23.749347 272.479459,23.186847 273.698209,23.186847 C274.666959,23.186847 275.406542,23.452472 275.916959,23.983722 C276.437792,24.514972 276.698209,25.3326803 276.698209,26.436847 L276.698209,33.07748 L275.057584,33.07748 L275.057584,26.983722 C275.057584,26.327472 275.005501,25.858722 274.901334,25.577472 C274.797167,25.2858053 274.604459,25.0514303 274.323209,24.874347 C274.041959,24.6972637 273.713834,24.608722 273.338834,24.608722 C272.661751,24.608722 272.099251,24.8378887 271.651334,25.296222 C271.203417,25.7441387 270.979459,26.4628887 270.979459,27.452472 L270.979459,33.07748 L269.338834,33.07748 L269.338834,26.796222 C269.338834,26.0670553 269.203417,25.5201803 268.932584,25.155597 C268.661751,24.7910137 268.224251,24.608722 267.620084,24.608722 C267.161751,24.608722 266.734667,24.7285137 266.338834,24.968097 C265.943001,25.2076803 265.656542,25.561847 265.479459,26.030597 C265.312792,26.499347 265.229459,27.1764329 265.229459,28.0618546 L265.229459,33.07748 L263.588834,33.07748 L263.588834,33.07748 Z M285.754636,29.968105 L287.442136,30.17123 C287.171303,31.1608133 286.676511,31.9316467 285.957761,32.48373 C285.239011,33.0253967 284.317136,33.29623 283.192136,33.29623 C281.775469,33.29623 280.650469,32.8639383 279.817136,31.999355 C278.994219,31.124355 278.582761,29.9003967 278.582761,28.32748 C278.582761,26.6920583 278.999428,25.4264307 279.832761,24.5305974 C280.676511,23.6347641 281.765053,23.1868474 283.098386,23.1868474 C284.390053,23.1868474 285.442136,23.6295557 286.254636,24.5149724 C287.077553,25.3899724 287.489011,26.6243499 287.489011,28.218105 C287.489011,28.3222717 287.489011,28.468105 287.489011,28.655605 L280.270261,28.655605 C280.332761,29.718105 280.634844,30.5358133 281.176511,31.10873 C281.718178,31.67123 282.390053,31.95248 283.192136,31.95248 C283.796303,31.95248 284.311928,31.79623 284.739011,31.48373 C285.166094,31.1608133 285.504636,30.655605 285.754636,29.968105 Z M280.364011,27.3118474 L285.770261,27.3118474 C285.697344,26.4993474 285.489011,25.8899724 285.145261,25.4837224 C284.624428,24.8483057 283.947344,24.5305974 283.114011,24.5305974 C282.364011,24.5305974 281.728594,24.7858057 281.207761,25.2962224 C280.697344,25.7962224 280.416094,26.4680974 280.364011,27.3118474 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polyline id="Path" fill-opacity="0" fill="#000000" points="512.8058 54.17213 512.8058 111.858257 24.8058 111.858257 24.8058 51.431972"></polyline>
+                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="512.8058 72.17212 512.8058 111.858262 24.8058 111.858262 24.8058 51.431977"></polyline>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="517.761 72.17213 512.8058 58.557826 507.8506 72.17213"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="227.93665 94.56168 298.834294 94.56168 298.834294 125.931767 227.93665 125.931767"></polygon>
+                <path d="M255.84567,117.16672 L255.84567,115.94797 C255.241503,116.906303 254.340462,117.38547 253.142545,117.38547 C252.371712,117.38547 251.663378,117.171928 251.017545,116.744845 C250.371712,116.317762 249.871712,115.724012 249.517545,114.963595 C249.163378,114.203178 248.986295,113.328178 248.986295,112.338595 C248.986295,111.369845 249.147753,110.494845 249.47067,109.713595 C249.793587,108.921928 250.272753,108.317762 250.90817,107.901095 C251.554003,107.484428 252.277962,107.276095 253.080045,107.276095 C253.663378,107.276095 254.179003,107.401095 254.62692,107.651095 C255.085253,107.890678 255.455045,108.208387 255.736295,108.60422 L255.736295,103.807345 L257.37692,103.807345 L257.37692,117.16672 L255.84567,117.16672 Z M250.673795,112.338595 C250.673795,113.578178 250.934212,114.505262 251.455045,115.119845 C251.975878,115.734428 252.590462,116.04172 253.298795,116.04172 C254.017545,116.04172 254.62692,115.750053 255.12692,115.16672 C255.62692,114.57297 255.87692,113.677137 255.87692,112.47922 C255.87692,111.156303 255.621712,110.187553 255.111295,109.57297 C254.600878,108.94797 253.97067,108.63547 253.22067,108.63547 C252.50192,108.63547 251.897753,108.932345 251.40817,109.526095 C250.918587,110.119845 250.673795,111.057345 250.673795,112.338595 Z M259.955777,117.16672 L259.955777,107.494845 L261.424527,107.494845 L261.424527,108.869845 C262.13286,107.807345 263.158902,107.276095 264.502652,107.276095 C265.085985,107.276095 265.622444,107.380262 266.112027,107.588595 C266.60161,107.796928 266.966194,108.07297 267.205777,108.41672 C267.455777,108.750053 267.627652,109.151095 267.721402,109.619845 C267.783902,109.921928 267.815152,110.453178 267.815152,111.213595 L267.815152,117.16672 L266.174527,117.16672 L266.174527,111.276095 C266.174527,110.609428 266.106819,110.114637 265.971402,109.79172 C265.846402,109.458387 265.622444,109.192762 265.299527,108.994845 C264.97661,108.796928 264.596402,108.69797 264.158902,108.69797 C263.460985,108.69797 262.856819,108.921928 262.346402,109.369845 C261.846402,109.807345 261.596402,110.645887 261.596402,111.88547 L261.596402,117.16672 L259.955777,117.16672 Z M269.675229,114.276095 L271.300229,114.026095 C271.383562,114.671928 271.633562,115.171928 272.050229,115.526095 C272.466896,115.869845 273.050229,116.04172 273.800229,116.04172 C274.550229,116.04172 275.107521,115.890678 275.472104,115.588595 C275.836687,115.276095 276.018979,114.911512 276.018979,114.494845 C276.018979,114.130262 275.857521,113.838595 275.534604,113.619845 C275.315854,113.474012 274.758562,113.29172 273.862729,113.07297 C272.654396,112.76047 271.815854,112.494845 271.347104,112.276095 C270.888771,112.057345 270.539812,111.755262 270.300229,111.369845 C270.060646,110.974012 269.940854,110.536512 269.940854,110.057345 C269.940854,109.630262 270.039812,109.234428 270.237729,108.869845 C270.435646,108.494845 270.706479,108.187553 271.050229,107.94797 C271.300229,107.76047 271.643979,107.60422 272.081479,107.47922 C272.529396,107.343803 273.003354,107.276095 273.503354,107.276095 C274.263771,107.276095 274.930437,107.38547 275.503354,107.60422 C276.076271,107.82297 276.498146,108.119845 276.768979,108.494845 C277.050229,108.869845 277.242937,109.369845 277.347104,109.994845 L275.737729,110.213595 C275.664812,109.713595 275.451271,109.32297 275.097104,109.04172 C274.753354,108.76047 274.263771,108.619845 273.628354,108.619845 C272.867937,108.619845 272.326271,108.744845 272.003354,108.994845 C271.690854,109.244845 271.534604,109.536512 271.534604,109.869845 C271.534604,110.078178 271.597104,110.270887 271.722104,110.44797 C271.857521,110.625053 272.071062,110.770887 272.362729,110.88547 C272.518979,110.94797 272.998146,111.088595 273.800229,111.307345 C274.966896,111.609428 275.779396,111.859428 276.237729,112.057345 C276.696062,112.255262 277.055437,112.54172 277.315854,112.91672 C277.576271,113.29172 277.706479,113.76047 277.706479,114.32297 C277.706479,114.875053 277.545021,115.390678 277.222104,115.869845 C276.909604,116.349012 276.451271,116.724012 275.847104,116.994845 C275.242937,117.255262 274.560646,117.38547 273.800229,117.38547 C272.550229,117.38547 271.591896,117.125053 270.925229,116.60422 C270.268979,116.083387 269.852312,115.307345 269.675229,114.276095 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="15.089237 39.1447 34.522308 39.1447 34.522308 51.428163 15.089237 51.428163"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="503.08923 36.7548 522.522335 36.7548 522.522335 54.172128 503.08923 54.172128"></polygon>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-qbf-ex.svg b/docs/reference/images/analysis/token-graph-qbf-ex.svg
new file mode 100644
index 0000000000000..63970673092d4
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-qbf-ex.svg
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="602px" height="54px" viewBox="0 0 602 54" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Analysis" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
+                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
+                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="75.41746 10.47244 146.3151 10.47244 146.3151 41.842511 75.41746 41.842511"></polygon>
+                <path d="M96.48065,36.7806104 L96.48065,32.046235 C96.23065,32.4108183 95.8764833,32.7129017 95.41815,32.952485 C94.9598167,33.1816517 94.4702333,33.296235 93.9494,33.296235 C92.8035667,33.296235 91.8139833,32.8379017 90.98065,31.921235 C90.1577333,31.0045683 89.746275,29.74936 89.746275,28.15561 C89.746275,27.1764383 89.9129417,26.3014357 90.246275,25.5306024 C90.590025,24.7597691 91.0796083,24.1764357 91.715025,23.7806024 C92.3608583,23.3847691 93.0639833,23.1868524 93.8244,23.1868524 C95.0223167,23.1868524 95.965025,23.6920607 96.652525,24.7024774 L96.652525,23.4056024 L98.121275,23.4056024 L98.121275,36.7806104 L96.48065,36.7806104 L96.48065,36.7806104 Z M91.433775,28.21811 C91.433775,29.4576933 91.6941917,30.389985 92.215025,31.014985 C92.7358583,31.639985 93.3608583,31.952485 94.090025,31.952485 C94.7879417,31.952485 95.3869,31.65561 95.8869,31.06186 C96.3973167,30.46811 96.652525,29.5670683 96.652525,28.358735 C96.652525,27.0670633 96.3869,26.0983107 95.85565,25.4524774 C95.3244,24.8066441 94.6994,24.4837274 93.98065,24.4837274 C93.2723167,24.4837274 92.66815,24.7858107 92.16815,25.3899774 C91.6785667,25.9941441 91.433775,26.9368549 91.433775,28.21811 Z M107.028242,33.077485 L107.028242,31.65561 C106.278242,32.74936 105.257409,33.296235 103.965742,33.296235 C103.392825,33.296235 102.856367,33.18686 102.356367,32.96811 C101.866784,32.74936 101.5022,32.4733183 101.262617,32.139985 C101.023034,31.8066517 100.856367,31.4004017 100.762617,30.921235 C100.6897,30.608735 100.653242,30.0983183 100.653242,29.389985 L100.653242,23.4056024 L102.293867,23.4056024 L102.293867,28.764985 C102.293867,29.6191517 102.330325,30.1972767 102.403242,30.49936 C102.507409,30.9264433 102.726159,31.264985 103.059492,31.514985 C103.392825,31.764985 103.804284,31.889985 104.293867,31.889985 C104.78345,31.889985 105.241784,31.764985 105.668867,31.514985 C106.106367,31.2545683 106.413659,30.9108183 106.590742,30.483735 C106.767825,30.046235 106.856367,29.4160267 106.856367,28.59311 L106.856367,23.4056024 L108.496992,23.4056024 L108.496992,33.0774854 L107.028242,33.0774854 L107.028242,33.077485 Z M111.075834,21.608727 L111.075834,19.718102 L112.716459,19.718102 L112.716459,21.608727 L111.075834,21.608727 Z M111.075834,33.077485 L111.075834,23.405602 L112.716459,23.405602 L112.716459,33.077485 L111.075834,33.077485 Z M121.533155,29.53061 L123.14253,29.74936 C122.965447,30.8535267 122.512322,31.7233183 121.783155,32.358735 C121.064405,32.983735 120.17378,33.296235 119.11128,33.296235 C117.788363,33.296235 116.725863,32.8639433 115.92378,31.99936 C115.121697,31.1347767 114.720655,29.8951933 114.720655,28.28061 C114.720655,27.2285216 114.89253,26.3118524 115.23628,25.5306024 C115.58003,24.7493524 116.106072,24.1660191 116.814405,23.7806024 C117.522738,23.3847691 118.293572,23.1868524 119.126905,23.1868524 C120.178988,23.1868524 121.038363,23.4524774 121.70503,23.9837274 C122.371697,24.5149774 122.79878,25.2701857 122.98628,26.2493524 L121.39253,26.4837274 C121.23628,25.8378941 120.965447,25.3535191 120.58003,25.0306024 C120.194613,24.6972691 119.731072,24.5306024 119.189405,24.5306024 C118.366488,24.5306024 117.694613,24.8274774 117.17378,25.4212274 C116.652947,26.0149774 116.39253,26.9524799 116.39253,28.233735 C116.39253,29.5358183 116.64253,30.483735 117.14253,31.077485 C117.64253,31.6608183 118.293572,31.952485 119.095655,31.952485 C119.741488,31.952485 120.277947,31.7545683 120.70503,31.358735 C121.14253,30.9629017 121.418572,30.3535267 121.533155,29.53061 Z M124.54878,33.077485 L124.54878,19.718102 L126.189405,19.718102 L126.189405,27.343102 L130.08003,23.405602 L132.189405,23.405602 L128.501905,26.999352 L132.564405,33.077485 L130.54878,33.077485 L127.345655,28.1243596 L126.189405,29.2493596 L126.189405,33.077485 L124.54878,33.077485 L124.54878,33.077485 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="261.4882 10.47244 332.385844 10.47244 332.385844 41.842511 261.4882 41.842511"></polygon>
+                <path d="M274.26498,33.07748 L272.749355,33.07748 L272.749355,19.718097 L274.38998,19.718097 L274.38998,24.483722 C275.087897,23.6191387 275.973313,23.186847 277.04623,23.186847 C277.63998,23.186847 278.20248,23.3066387 278.73373,23.546222 C279.26498,23.7858053 279.70248,24.124347 280.04623,24.561847 C280.38998,24.9889303 280.655605,25.5097637 280.843105,26.124347 C281.041022,26.7389303 281.13998,27.3951829 281.13998,28.0931046 C281.13998,29.7493546 280.728522,31.0306046 279.905605,31.9368546 C279.093105,32.8431046 278.10873,33.2962296 276.95248,33.2962296 C275.817063,33.2962296 274.92123,32.8170629 274.26498,31.8587296 L274.26498,33.0774796 L274.26498,33.07748 Z M274.249355,28.17123 C274.249355,29.32748 274.410813,30.1660217 274.73373,30.686855 C275.244147,31.5305974 275.942063,31.95248 276.82748,31.95248 C277.54623,31.95248 278.166022,31.63998 278.686855,31.01498 C279.207688,30.38998 279.468105,29.4628967 279.468105,28.23373 C279.468105,26.9733083 279.218105,26.0410141 278.718105,25.4368474 C278.218105,24.8326807 277.60873,24.5305974 276.88998,24.5305974 C276.181647,24.5305974 275.561855,24.8430974 275.030605,25.4680974 C274.509772,26.0930974 274.249355,26.9941416 274.249355,28.17123 Z M283.125087,33.07748 L283.125087,23.405597 L284.593837,23.405597 L284.593837,24.874347 C284.968837,24.186847 285.312587,23.733722 285.625087,23.514972 C285.948004,23.296222 286.30217,23.186847 286.687587,23.186847 C287.23967,23.186847 287.80217,23.3639303 288.375087,23.718097 L287.812587,25.233722 C287.406337,24.9941387 287.005295,24.874347 286.609462,24.874347 C286.244879,24.874347 285.921962,24.983722 285.640712,25.202472 C285.359462,25.421222 285.156337,25.718097 285.031337,26.093097 C284.843837,26.6764303 284.750087,27.3170579 284.750087,28.0149796 L284.750087,33.07748 L283.125087,33.07748 L283.125087,33.07748 Z M288.744014,28.23373 C288.744014,26.4420583 289.238806,25.1191391 290.228389,24.2649724 C291.061722,23.5462224 292.077347,23.1868474 293.275264,23.1868474 C294.608597,23.1868474 295.697139,23.6243474 296.540889,24.4993474 C297.384639,25.3639307 297.806514,26.5670583 297.806514,28.10873 C297.806514,29.3483133 297.619014,30.32748 297.244014,31.04623 C296.869014,31.7545633 296.322139,32.3066467 295.603389,32.70248 C294.895056,33.0983133 294.119014,33.29623 293.275264,33.29623 C291.921097,33.29623 290.827347,32.8639383 289.994014,31.999355 C289.160681,31.124355 288.744014,29.8691467 288.744014,28.23373 Z M290.431514,28.23373 C290.431514,29.4733133 290.697139,30.405605 291.228389,31.030605 C291.770056,31.6451883 292.452347,31.95248 293.275264,31.95248 C294.087764,31.95248 294.764847,31.6451883 295.306514,31.030605 C295.848181,30.405605 296.119014,29.4576883 296.119014,28.186855 C296.119014,26.9889333 295.848181,26.0826807 295.306514,25.4680974 C294.764847,24.8535141 294.087764,24.5462224 293.275264,24.5462224 C292.452347,24.5462224 291.770056,24.8535141 291.228389,25.4680974 C290.697139,26.0722641 290.431514,26.9941416 290.431514,28.23373 Z M301.510341,33.07748 L298.541591,23.405597 L300.244716,23.405597 L301.775966,28.9837296 L302.354091,31.0618546 C302.385341,30.9576879 302.552008,30.2910213 302.854091,29.0618546 L304.400966,23.405597 L306.088466,23.405597 L307.525966,29.0149796 L308.010341,30.8587296 L308.572841,28.9993546 L310.229091,23.405597 L311.822841,23.405597 L308.791591,33.07748 L307.088466,33.07748 L305.557216,27.2805974 L305.182216,25.6399724 L303.229091,33.07748 L301.510341,33.07748 Z M313.202052,33.07748 L313.202052,23.405597 L314.670802,23.405597 L314.670802,24.780597 C315.379135,23.718097 316.405177,23.186847 317.748927,23.186847 C318.33226,23.186847 318.868719,23.2910137 319.358302,23.499347 C319.847885,23.7076803 320.212469,23.983722 320.452052,24.327472 C320.702052,24.6608053 320.873927,25.061847 320.967677,25.530597 C321.030177,25.8326803 321.061427,26.3639303 321.061427,27.124347 L321.061427,33.07748 L319.420802,33.07748 L319.420802,27.186847 C319.420802,26.5201803 319.353094,26.0253887 319.217677,25.702472 C319.092677,25.3691387 318.868719,25.1035137 318.545802,24.905597 C318.222885,24.7076803 317.842677,24.608722 317.405177,24.608722 C316.70726,24.608722 316.103094,24.8326803 315.592677,25.280597 C315.092677,25.718097 314.842677,26.5566387 314.842677,27.796222 L314.842677,33.07748 L313.202052,33.07748 L313.202052,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="454.6824 10.47244 502.052487 10.47244 502.052487 41.842511 454.6824 41.842511"></polygon>
+                <path d="M467.5489,33.07748 L467.5489,24.671222 L466.095775,24.671222 L466.095775,23.405597 L467.5489,23.405597 L467.5489,22.374347 C467.5489,21.7285137 467.606192,21.2441387 467.720775,20.921222 C467.877025,20.4941387 468.153067,20.1503887 468.5489,19.889972 C468.944733,19.6295553 469.502025,19.499347 470.220775,19.499347 C470.679108,19.499347 471.189525,19.5514303 471.752025,19.655597 L471.502025,21.093097 C471.168692,21.030597 470.850983,20.999347 470.5489,20.999347 C470.0489,20.999347 469.694733,21.108722 469.4864,21.327472 C469.278067,21.5358053 469.1739,21.9316387 469.1739,22.514972 L469.1739,23.405597 L471.064525,23.405597 L471.064525,24.671222 L469.1739,24.671222 L469.1739,33.07748 L467.5489,33.07748 Z M471.732189,28.23373 C471.732189,26.4420583 472.22698,25.1191391 473.216564,24.2649724 C474.049897,23.5462224 475.065522,23.1868474 476.263439,23.1868474 C477.596772,23.1868474 478.685314,23.6243474 479.529064,24.4993474 C480.372814,25.3639307 480.794689,26.5670583 480.794689,28.10873 C480.794689,29.3483133 480.607189,30.32748 480.232189,31.04623 C479.857189,31.7545633 479.310314,32.3066467 478.591564,32.70248 C477.88323,33.0983133 477.107189,33.29623 476.263439,33.29623 C474.909272,33.29623 473.815522,32.8639383 472.982189,31.999355 C472.148855,31.124355 471.732189,29.8691467 471.732189,28.23373 Z M473.419689,28.23373 C473.419689,29.4733133 473.685314,30.405605 474.216564,31.030605 C474.75823,31.6451883 475.440522,31.95248 476.263439,31.95248 C477.075939,31.95248 477.753022,31.6451883 478.294689,31.030605 C478.836355,30.405605 479.107189,29.4576883 479.107189,28.186855 C479.107189,26.9889333 478.836355,26.0826807 478.294689,25.4680974 C477.753022,24.8535141 477.075939,24.5462224 476.263439,24.5462224 C475.440522,24.5462224 474.75823,24.8535141 474.216564,25.4680974 C473.685314,26.0722641 473.419689,26.9941416 473.419689,28.23373 Z M481.623546,33.07748 L485.154796,28.04623 L481.889171,23.4055974 L483.936046,23.4055974 L485.420421,25.6712224 C485.701671,26.0983057 485.925629,26.4576807 486.092296,26.7493474 C486.363129,26.3535141 486.607921,25.9993474 486.826671,25.6868474 L488.467296,23.4055974 L490.420421,23.4055974 L487.076671,27.9524724 L490.670421,33.07748 L488.654796,33.07748 L486.670421,30.07748 L486.154796,29.26498 L483.607921,33.07748 L481.623546,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/analysis/token-graph-qbf-synonym-ex.svg b/docs/reference/images/analysis/token-graph-qbf-synonym-ex.svg
new file mode 100644
index 0000000000000..2baa3d9e63cb5
--- /dev/null
+++ b/docs/reference/images/analysis/token-graph-qbf-synonym-ex.svg
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="602px" height="123px" viewBox="0 0 602 123" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 60.1 (88133) - https://sketch.com -->
+    <title>Slice 1</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="token-graph-qbf-synonym-ex" transform="translate(1.000000, 1.000000)">
+            <g id="Group">
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M0.209972,1.39317 L0.209972,1.39317 C0.209972,0.6962172 0.774963,0.13123 1.471912,0.13123 L46.318115,0.13123 C46.6528013,0.13123 46.9737775,0.26418746 47.2104414,0.50084365 C47.4470975,0.73749984 47.580055,1.05848375 47.580055,1.39317 L47.580055,48.948025 C47.580055,49.6449855 47.0150602,50.2099726 46.318115,50.2099726 L1.471912,50.2099726 C0.774963,50.2099726 0.209972,49.6449854 0.209972,48.948025 L0.209972,1.39317 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M19.488403,25.49685 C19.488403,23.9239333 19.6498613,22.6583083 19.972778,21.699975 C20.2956947,20.731225 20.7748613,19.9864333 21.410278,19.4656 C22.0561113,18.9447667 22.863403,18.68435 23.832153,18.68435 C24.550903,18.68435 25.1811113,18.8301833 25.722778,19.12185 C26.2644447,19.4031 26.707153,19.8197667 27.050903,20.37185 C27.4050697,20.9135167 27.6811113,21.574975 27.879028,22.356225 C28.0873613,23.1270583 28.191528,24.1739333 28.191528,25.49685 C28.191528,27.0697667 28.0300697,28.3406025 27.707153,29.3093576 C27.3842363,30.2676909 26.8998613,31.0124826 26.254028,31.5437326 C25.6186113,32.0645659 24.8113197,32.3249826 23.832153,32.3249826 C22.550903,32.3249826 21.5404863,31.8614409 20.800903,30.9343576 C19.925903,29.8197743 19.488403,28.0072717 19.488403,25.49685 L19.488403,25.49685 Z M21.160278,25.49685 C21.160278,27.6947717 21.4154863,29.1583159 21.925903,29.8874826 C22.4467363,30.6062326 23.082153,30.9656076 23.832153,30.9656076 C24.5925697,30.9656076 25.2279863,30.6010243 25.738403,29.8718576 C26.2488197,29.1426909 26.504028,27.6843551 26.504028,25.49685 C26.504028,23.2989333 26.2488197,21.8406 25.738403,21.12185 C25.2279863,20.4031 24.5873613,20.043725 23.816528,20.043725 C23.066528,20.043725 22.4675697,20.3614333 22.019653,20.99685 C21.4467363,21.80935 21.160278,23.30935 21.160278,25.49685 L21.160278,25.49685 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M183.71559,4.1045 L183.71559,4.1045 C183.71559,2.5803148 184.951186,1.344719 186.475371,1.344719 L228.325896,1.344719 C229.05783,1.344719 229.759795,1.63547523 230.277342,2.1530381 C230.794921,2.67060097 231.085662,3.3725587 231.085662,4.1044999 L231.085662,48.6636799 C231.085662,50.1878651 229.850066,51.4234609 228.325896,51.4234609 L186.475371,51.4234609 C184.951186,51.4234609 183.71559,50.1878652 183.71559,48.6636799 L183.71559,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M209.1659,33.3041 L207.525275,33.3041 L207.525275,22.850967 C207.129442,23.225967 206.608608,23.6061753 205.962775,23.991592 C205.327358,24.366592 204.754442,24.647842 204.244025,24.835342 L204.244025,23.241592 C205.160692,22.8145087 205.962775,22.2936753 206.650275,21.679092 C207.337775,21.0645087 207.82215,20.4707587 208.1034,19.897842 L209.1659,19.897842 L209.1659,33.3041 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="39.08891 16.85302 58.521977 16.85302 58.521977 38.679795 39.08891 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="175.08891 16.85302 194.521985 16.85302 194.521985 38.679795 175.08891 38.679795"></polygon>
+                <line x1="194.52199" y1="27.7664" x2="47.56136" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="176.52206" y1="27.82041" x2="47.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="176.53693 32.77559 190.136295 27.779565 176.507206 22.8652355"></polygon>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M367.71558,4.1045 L367.71558,4.1045 C367.71558,2.5803148 368.951206,1.344719 370.475376,1.344719 L412.325901,1.344719 C413.057835,1.344719 413.7598,1.63547523 414.277348,2.1530381 C414.794926,2.67060097 415.085667,3.3725587 415.085667,4.1044999 L415.085667,48.6636799 C415.085667,50.1878651 413.850071,51.4234609 412.325901,51.4234609 L370.475376,51.4234609 C368.951206,51.4234609 367.71558,50.1878652 367.71558,48.6636799 L367.71558,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M395.6034,31.72597 L395.6034,33.304095 L386.775275,33.304095 C386.764858,32.9082617 386.827358,32.5280533 386.962775,32.16347 C387.191942,31.5593033 387.551317,30.9655533 388.0409,30.38222 C388.5409,29.7988867 389.25965,29.1270117 390.19715,28.366595 C391.645067,27.1790899 392.624233,26.2363791 393.13465,25.5384624 C393.645067,24.8405457 393.900275,24.1842957 393.900275,23.5697124 C393.900275,22.9134624 393.6659,22.3665874 393.19715,21.9290874 C392.738817,21.4811707 392.13465,21.2572124 391.38465,21.2572124 C390.592983,21.2572124 389.957567,21.4967957 389.4784,21.9759624 C389.00965,22.4447124 388.775275,23.0957541 388.775275,23.9290874 L387.087775,23.7572124 C387.202358,22.4967957 387.63465,21.5384624 388.38465,20.8822124 C389.145067,20.2259624 390.155483,19.8978374 391.4159,19.8978374 C392.69715,19.8978374 393.712775,20.2520041 394.462775,20.9603374 C395.212775,21.6686707 395.587775,22.5488791 395.587775,23.6009624 C395.587775,24.1322124 395.4784,24.6582541 395.25965,25.1790874 C395.0409,25.6999207 394.676317,26.2467957 394.1659,26.8197124 C393.6659,27.3822124 392.82215,28.1634649 391.63465,29.16347 C390.655483,29.9863867 390.025275,30.5488867 389.744025,30.85097 C389.462775,31.1426367 389.233608,31.4343033 389.056525,31.72597 L395.6034,31.72597 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="223.0889 16.85302 242.521975 16.85302 242.521975 38.679795 223.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="359.0889 16.85302 378.521975 16.85302 378.521975 38.679795 359.0889 38.679795"></polygon>
+                <line x1="378.52197" y1="27.7664" x2="231.56134" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="360.52206" y1="27.82041" x2="231.56134" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="360.53693 32.77559 374.136295 27.779565 360.507206 22.8652355"></polygon>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" fill-opacity="0" fill="#000000"></path>
+                <path d="M551.7156,4.1045 L551.7156,4.1045 C551.7156,2.5803148 552.951196,1.344719 554.475366,1.344719 L596.325891,1.344719 C597.057824,1.344719 597.75979,1.63547523 598.277368,2.1530381 C598.794946,2.67060097 599.085717,3.3725587 599.085717,4.1044999 L599.085717,48.6636799 C599.085717,50.1878651 597.850122,51.4234609 596.325891,51.4234609 L554.475366,51.4234609 C552.951196,51.4234609 551.7156,50.1878652 551.7156,48.6636799 L551.7156,4.1045 Z" id="Path" stroke="#979CAB" stroke-width="2" stroke-linejoin="round"></path>
+                <path d="M570.994,29.77284 L572.634625,29.55409 C572.822125,30.49159 573.139833,31.163465 573.58775,31.569715 C574.046083,31.975965 574.598167,32.17909 575.244,32.17909 C576.014833,32.17909 576.665875,31.913465 577.197125,31.382215 C577.728375,30.850965 577.994,30.1895067 577.994,29.3978324 C577.994,28.64784 577.749208,28.0280458 577.259625,27.5384574 C576.770042,27.0488741 576.145042,26.8040824 575.384625,26.8040824 C575.072125,26.8040824 574.686708,26.8613741 574.228375,26.9759574 L574.415875,25.5384574 C574.520042,25.5488741 574.608583,25.5540824 574.6815,25.5540824 C575.379417,25.5540824 576.004417,25.3717907 576.5565,25.0072074 C577.119,24.6426241 577.40025,24.0853324 577.40025,23.3353324 C577.40025,22.7311657 577.197125,22.2311657 576.790875,21.8353324 C576.384625,21.4394991 575.858583,21.2415824 575.21275,21.2415824 C574.577333,21.2415824 574.046083,21.4447074 573.619,21.8509574 C573.191917,22.2467907 572.921083,22.8457491 572.8065,23.6478324 L571.165875,23.3509574 C571.363792,22.2572074 571.816917,21.4082491 572.52525,20.8040824 C573.233583,20.1999157 574.119,19.8978324 575.1815,19.8978324 C575.910667,19.8978324 576.577333,20.0540824 577.1815,20.3665824 C577.796083,20.6790824 578.264833,21.1061657 578.58775,21.6478324 C578.921083,22.1894991 579.08775,22.7624157 579.08775,23.3665824 C579.08775,23.9394991 578.9315,24.4655407 578.619,24.9447074 C578.3065,25.4134574 577.848167,25.7884574 577.244,26.0697074 C578.035667,26.2572074 578.65025,26.6374157 579.08775,27.2103324 C579.52525,27.7832541 579.744,28.5020067 579.744,29.36659 C579.744,30.52284 579.316917,31.507215 578.46275,32.319715 C577.619,33.132215 576.546083,33.538465 575.244,33.538465 C574.066917,33.538465 573.092958,33.1895067 572.322125,32.49159 C571.551292,31.7936733 571.108583,30.8874233 570.994,29.77284 Z" id="Path" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="407.0889 16.85302 426.521975 16.85302 426.521975 38.679795 407.0889 38.679795"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="543.0889 16.85302 562.521944 16.85302 562.521944 38.679795 543.0889 38.679795"></polygon>
+                <line x1="562.522" y1="27.7664" x2="415.56137" y2="28.2073485" id="Path" fill-opacity="0" fill="#000000"></line>
+                <line x1="544.52203" y1="27.82041" x2="415.56133" y2="28.20735" id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round"></line>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="544.5369 32.77559 558.136265 27.779565 544.507176 22.8652355"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="75.41746 10.47244 146.3151 10.47244 146.3151 41.842511 75.41746 41.842511"></polygon>
+                <path d="M96.48065,36.7806104 L96.48065,32.046235 C96.23065,32.4108183 95.8764833,32.7129017 95.41815,32.952485 C94.9598167,33.1816517 94.4702333,33.296235 93.9494,33.296235 C92.8035667,33.296235 91.8139833,32.8379017 90.98065,31.921235 C90.1577333,31.0045683 89.746275,29.74936 89.746275,28.15561 C89.746275,27.1764383 89.9129417,26.3014357 90.246275,25.5306024 C90.590025,24.7597691 91.0796083,24.1764357 91.715025,23.7806024 C92.3608583,23.3847691 93.0639833,23.1868524 93.8244,23.1868524 C95.0223167,23.1868524 95.965025,23.6920607 96.652525,24.7024774 L96.652525,23.4056024 L98.121275,23.4056024 L98.121275,36.7806104 L96.48065,36.7806104 L96.48065,36.7806104 Z M91.433775,28.21811 C91.433775,29.4576933 91.6941917,30.389985 92.215025,31.014985 C92.7358583,31.639985 93.3608583,31.952485 94.090025,31.952485 C94.7879417,31.952485 95.3869,31.65561 95.8869,31.06186 C96.3973167,30.46811 96.652525,29.5670683 96.652525,28.358735 C96.652525,27.0670633 96.3869,26.0983107 95.85565,25.4524774 C95.3244,24.8066441 94.6994,24.4837274 93.98065,24.4837274 C93.2723167,24.4837274 92.66815,24.7858107 92.16815,25.3899774 C91.6785667,25.9941441 91.433775,26.9368549 91.433775,28.21811 Z M107.028242,33.077485 L107.028242,31.65561 C106.278242,32.74936 105.257409,33.296235 103.965742,33.296235 C103.392825,33.296235 102.856367,33.18686 102.356367,32.96811 C101.866784,32.74936 101.5022,32.4733183 101.262617,32.139985 C101.023034,31.8066517 100.856367,31.4004017 100.762617,30.921235 C100.6897,30.608735 100.653242,30.0983183 100.653242,29.389985 L100.653242,23.4056024 L102.293867,23.4056024 L102.293867,28.764985 C102.293867,29.6191517 102.330325,30.1972767 102.403242,30.49936 C102.507409,30.9264433 102.726159,31.264985 103.059492,31.514985 C103.392825,31.764985 103.804284,31.889985 104.293867,31.889985 C104.78345,31.889985 105.241784,31.764985 105.668867,31.514985 C106.106367,31.2545683 106.413659,30.9108183 106.590742,30.483735 C106.767825,30.046235 106.856367,29.4160267 106.856367,28.59311 L106.856367,23.4056024 L108.496992,23.4056024 L108.496992,33.0774854 L107.028242,33.0774854 L107.028242,33.077485 Z M111.075834,21.608727 L111.075834,19.718102 L112.716459,19.718102 L112.716459,21.608727 L111.075834,21.608727 Z M111.075834,33.077485 L111.075834,23.405602 L112.716459,23.405602 L112.716459,33.077485 L111.075834,33.077485 Z M121.533155,29.53061 L123.14253,29.74936 C122.965447,30.8535267 122.512322,31.7233183 121.783155,32.358735 C121.064405,32.983735 120.17378,33.296235 119.11128,33.296235 C117.788363,33.296235 116.725863,32.8639433 115.92378,31.99936 C115.121697,31.1347767 114.720655,29.8951933 114.720655,28.28061 C114.720655,27.2285216 114.89253,26.3118524 115.23628,25.5306024 C115.58003,24.7493524 116.106072,24.1660191 116.814405,23.7806024 C117.522738,23.3847691 118.293572,23.1868524 119.126905,23.1868524 C120.178988,23.1868524 121.038363,23.4524774 121.70503,23.9837274 C122.371697,24.5149774 122.79878,25.2701857 122.98628,26.2493524 L121.39253,26.4837274 C121.23628,25.8378941 120.965447,25.3535191 120.58003,25.0306024 C120.194613,24.6972691 119.731072,24.5306024 119.189405,24.5306024 C118.366488,24.5306024 117.694613,24.8274774 117.17378,25.4212274 C116.652947,26.0149774 116.39253,26.9524799 116.39253,28.233735 C116.39253,29.5358183 116.64253,30.483735 117.14253,31.077485 C117.64253,31.6608183 118.293572,31.952485 119.095655,31.952485 C119.741488,31.952485 120.277947,31.7545683 120.70503,31.358735 C121.14253,30.9629017 121.418572,30.3535267 121.533155,29.53061 Z M124.54878,33.077485 L124.54878,19.718102 L126.189405,19.718102 L126.189405,27.343102 L130.08003,23.405602 L132.189405,23.405602 L128.501905,26.999352 L132.564405,33.077485 L130.54878,33.077485 L127.345655,28.1243596 L126.189405,29.2493596 L126.189405,33.077485 L124.54878,33.077485 L124.54878,33.077485 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="261.4882 10.47244 332.385844 10.47244 332.385844 41.842511 261.4882 41.842511"></polygon>
+                <path d="M274.26498,33.07748 L272.749355,33.07748 L272.749355,19.718097 L274.38998,19.718097 L274.38998,24.483722 C275.087897,23.6191387 275.973313,23.186847 277.04623,23.186847 C277.63998,23.186847 278.20248,23.3066387 278.73373,23.546222 C279.26498,23.7858053 279.70248,24.124347 280.04623,24.561847 C280.38998,24.9889303 280.655605,25.5097637 280.843105,26.124347 C281.041022,26.7389303 281.13998,27.3951829 281.13998,28.0931046 C281.13998,29.7493546 280.728522,31.0306046 279.905605,31.9368546 C279.093105,32.8431046 278.10873,33.2962296 276.95248,33.2962296 C275.817063,33.2962296 274.92123,32.8170629 274.26498,31.8587296 L274.26498,33.0774796 L274.26498,33.07748 Z M274.249355,28.17123 C274.249355,29.32748 274.410813,30.1660217 274.73373,30.686855 C275.244147,31.5305974 275.942063,31.95248 276.82748,31.95248 C277.54623,31.95248 278.166022,31.63998 278.686855,31.01498 C279.207688,30.38998 279.468105,29.4628967 279.468105,28.23373 C279.468105,26.9733083 279.218105,26.0410141 278.718105,25.4368474 C278.218105,24.8326807 277.60873,24.5305974 276.88998,24.5305974 C276.181647,24.5305974 275.561855,24.8430974 275.030605,25.4680974 C274.509772,26.0930974 274.249355,26.9941416 274.249355,28.17123 Z M283.125087,33.07748 L283.125087,23.405597 L284.593837,23.405597 L284.593837,24.874347 C284.968837,24.186847 285.312587,23.733722 285.625087,23.514972 C285.948004,23.296222 286.30217,23.186847 286.687587,23.186847 C287.23967,23.186847 287.80217,23.3639303 288.375087,23.718097 L287.812587,25.233722 C287.406337,24.9941387 287.005295,24.874347 286.609462,24.874347 C286.244879,24.874347 285.921962,24.983722 285.640712,25.202472 C285.359462,25.421222 285.156337,25.718097 285.031337,26.093097 C284.843837,26.6764303 284.750087,27.3170579 284.750087,28.0149796 L284.750087,33.07748 L283.125087,33.07748 L283.125087,33.07748 Z M288.744014,28.23373 C288.744014,26.4420583 289.238806,25.1191391 290.228389,24.2649724 C291.061722,23.5462224 292.077347,23.1868474 293.275264,23.1868474 C294.608597,23.1868474 295.697139,23.6243474 296.540889,24.4993474 C297.384639,25.3639307 297.806514,26.5670583 297.806514,28.10873 C297.806514,29.3483133 297.619014,30.32748 297.244014,31.04623 C296.869014,31.7545633 296.322139,32.3066467 295.603389,32.70248 C294.895056,33.0983133 294.119014,33.29623 293.275264,33.29623 C291.921097,33.29623 290.827347,32.8639383 289.994014,31.999355 C289.160681,31.124355 288.744014,29.8691467 288.744014,28.23373 Z M290.431514,28.23373 C290.431514,29.4733133 290.697139,30.405605 291.228389,31.030605 C291.770056,31.6451883 292.452347,31.95248 293.275264,31.95248 C294.087764,31.95248 294.764847,31.6451883 295.306514,31.030605 C295.848181,30.405605 296.119014,29.4576883 296.119014,28.186855 C296.119014,26.9889333 295.848181,26.0826807 295.306514,25.4680974 C294.764847,24.8535141 294.087764,24.5462224 293.275264,24.5462224 C292.452347,24.5462224 291.770056,24.8535141 291.228389,25.4680974 C290.697139,26.0722641 290.431514,26.9941416 290.431514,28.23373 Z M301.510341,33.07748 L298.541591,23.405597 L300.244716,23.405597 L301.775966,28.9837296 L302.354091,31.0618546 C302.385341,30.9576879 302.552008,30.2910213 302.854091,29.0618546 L304.400966,23.405597 L306.088466,23.405597 L307.525966,29.0149796 L308.010341,30.8587296 L308.572841,28.9993546 L310.229091,23.405597 L311.822841,23.405597 L308.791591,33.07748 L307.088466,33.07748 L305.557216,27.2805974 L305.182216,25.6399724 L303.229091,33.07748 L301.510341,33.07748 Z M313.202052,33.07748 L313.202052,23.405597 L314.670802,23.405597 L314.670802,24.780597 C315.379135,23.718097 316.405177,23.186847 317.748927,23.186847 C318.33226,23.186847 318.868719,23.2910137 319.358302,23.499347 C319.847885,23.7076803 320.212469,23.983722 320.452052,24.327472 C320.702052,24.6608053 320.873927,25.061847 320.967677,25.530597 C321.030177,25.8326803 321.061427,26.3639303 321.061427,27.124347 L321.061427,33.07748 L319.420802,33.07748 L319.420802,27.186847 C319.420802,26.5201803 319.353094,26.0253887 319.217677,25.702472 C319.092677,25.3691387 318.868719,25.1035137 318.545802,24.905597 C318.222885,24.7076803 317.842677,24.608722 317.405177,24.608722 C316.70726,24.608722 316.103094,24.8326803 315.592677,25.280597 C315.092677,25.718097 314.842677,26.5566387 314.842677,27.796222 L314.842677,33.07748 L313.202052,33.07748 L313.202052,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="454.6824 10.47244 502.052487 10.47244 502.052487 41.842511 454.6824 41.842511"></polygon>
+                <path d="M467.5489,33.07748 L467.5489,24.671222 L466.095775,24.671222 L466.095775,23.405597 L467.5489,23.405597 L467.5489,22.374347 C467.5489,21.7285137 467.606192,21.2441387 467.720775,20.921222 C467.877025,20.4941387 468.153067,20.1503887 468.5489,19.889972 C468.944733,19.6295553 469.502025,19.499347 470.220775,19.499347 C470.679108,19.499347 471.189525,19.5514303 471.752025,19.655597 L471.502025,21.093097 C471.168692,21.030597 470.850983,20.999347 470.5489,20.999347 C470.0489,20.999347 469.694733,21.108722 469.4864,21.327472 C469.278067,21.5358053 469.1739,21.9316387 469.1739,22.514972 L469.1739,23.405597 L471.064525,23.405597 L471.064525,24.671222 L469.1739,24.671222 L469.1739,33.07748 L467.5489,33.07748 Z M471.732189,28.23373 C471.732189,26.4420583 472.22698,25.1191391 473.216564,24.2649724 C474.049897,23.5462224 475.065522,23.1868474 476.263439,23.1868474 C477.596772,23.1868474 478.685314,23.6243474 479.529064,24.4993474 C480.372814,25.3639307 480.794689,26.5670583 480.794689,28.10873 C480.794689,29.3483133 480.607189,30.32748 480.232189,31.04623 C479.857189,31.7545633 479.310314,32.3066467 478.591564,32.70248 C477.88323,33.0983133 477.107189,33.29623 476.263439,33.29623 C474.909272,33.29623 473.815522,32.8639383 472.982189,31.999355 C472.148855,31.124355 471.732189,29.8691467 471.732189,28.23373 Z M473.419689,28.23373 C473.419689,29.4733133 473.685314,30.405605 474.216564,31.030605 C474.75823,31.6451883 475.440522,31.95248 476.263439,31.95248 C477.075939,31.95248 477.753022,31.6451883 478.294689,31.030605 C478.836355,30.405605 479.107189,29.4576883 479.107189,28.186855 C479.107189,26.9889333 478.836355,26.0826807 478.294689,25.4680974 C477.753022,24.8535141 477.075939,24.5462224 476.263439,24.5462224 C475.440522,24.5462224 474.75823,24.8535141 474.216564,25.4680974 C473.685314,26.0722641 473.419689,26.9941416 473.419689,28.23373 Z M481.623546,33.07748 L485.154796,28.04623 L481.889171,23.4055974 L483.936046,23.4055974 L485.420421,25.6712224 C485.701671,26.0983057 485.925629,26.4576807 486.092296,26.7493474 C486.363129,26.3535141 486.607921,25.9993474 486.826671,25.6868474 L488.467296,23.4055974 L490.420421,23.4055974 L487.076671,27.9524724 L490.670421,33.07748 L488.654796,33.07748 L486.670421,30.07748 L486.154796,29.26498 L483.607921,33.07748 L481.623546,33.07748 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+                <polygon id="Path" fill="#FFFFFF" points="15.088913 39.14516 34.52198 39.14516 34.52198 60.971927 15.088913 60.971927"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="199.08891 39.14516 218.52197 39.14516 218.52197 60.971927 199.08891 60.971927"></polygon>
+                <polyline id="Path" fill-opacity="0" fill="#000000" points="208.80545 39.14516 208.80545 107.61417 24.80545 107.61417 24.80545 39.20815"></polyline>
+                <polyline id="Path" stroke="#00BFB3" stroke-width="3" stroke-linejoin="round" points="208.80545 57.14516 208.80545 107.61417 24.80545 107.61417 24.80545 39.20815"></polyline>
+                <polygon id="Path" stroke="#00BFB3" stroke-width="3" fill="#00BFB3" points="213.76065 57.14516 208.80545 43.530872 203.85025 57.14516"></polygon>
+                <polygon id="Path" fill="#FFFFFF" points="83.77165 90.47244 137.976377 90.47244 137.976377 121.842511 83.77165 121.842511"></polygon>
+                <path d="M97.46379,113.07748 L97.46379,104.67123 L96.010665,104.67123 L96.010665,103.405605 L97.46379,103.405605 L97.46379,102.374355 C97.46379,101.728522 97.5210817,101.244147 97.635665,100.92123 C97.791915,100.494147 98.0679567,100.150397 98.46379,99.88998 C98.8596233,99.6295633 99.416915,99.499355 100.135665,99.499355 C100.593998,99.499355 101.104415,99.5514383 101.666915,99.655605 L101.416915,101.093105 C101.083582,101.030605 100.765873,100.999355 100.46379,100.999355 C99.96379,100.999355 99.6096233,101.10873 99.40129,101.32748 C99.1929567,101.535813 99.08879,101.931647 99.08879,102.51498 L99.08879,103.405605 L100.979415,103.405605 L100.979415,104.67123 L99.08879,104.67123 L99.08879,113.07748 L97.46379,113.07748 Z M108.568969,111.88998 C107.954386,112.400397 107.365844,112.76498 106.803344,112.98373 C106.251261,113.192063 105.652302,113.29623 105.006469,113.29623 C103.943969,113.29623 103.126261,113.035813 102.553344,112.51498 C101.980427,111.994147 101.693969,111.332688 101.693969,110.530605 C101.693969,110.051438 101.803344,109.619147 102.022094,109.23373 C102.240844,108.837897 102.522094,108.525397 102.865844,108.29623 C103.220011,108.056647 103.615844,107.874355 104.053344,107.749355 C104.365844,107.666022 104.850219,107.582688 105.506469,107.499355 C106.829386,107.343105 107.803344,107.155605 108.428344,106.936855 C108.438761,106.707688 108.443969,106.567063 108.443969,106.51498 C108.443969,105.848313 108.287719,105.374355 107.975219,105.093105 C107.558552,104.728522 106.933552,104.54623 106.100219,104.54623 C105.329386,104.54623 104.761677,104.681647 104.397094,104.95248 C104.032511,105.223313 103.761677,105.697272 103.584594,106.374355 L101.975219,106.155605 C102.121052,105.478522 102.360636,104.931647 102.693969,104.51498 C103.027302,104.087897 103.511677,103.759772 104.147094,103.530605 C104.782511,103.301438 105.511677,103.186855 106.334594,103.186855 C107.167927,103.186855 107.839802,103.285813 108.350219,103.48373 C108.871052,103.67123 109.251261,103.916022 109.490844,104.218105 C109.740844,104.509772 109.912719,104.879563 110.006469,105.32748 C110.058552,105.60873 110.084594,106.113938 110.084594,106.843105 L110.084594,109.030605 C110.084594,110.551438 110.121052,111.51498 110.193969,111.92123 C110.266886,112.317063 110.402302,112.70248 110.600219,113.07748 L108.897094,113.07748 C108.720011,112.73373 108.610636,112.337897 108.568969,111.88998 Z M108.428344,108.218105 C107.834594,108.468105 106.943969,108.676438 105.756469,108.843105 C105.079386,108.936855 104.600219,109.04623 104.318969,109.17123 C104.037719,109.29623 103.818969,109.473313 103.662719,109.70248 C103.516886,109.931647 103.443969,110.192063 103.443969,110.48373 C103.443969,110.92123 103.610636,111.285813 103.943969,111.57748 C104.277302,111.869147 104.761677,112.01498 105.397094,112.01498 C106.022094,112.01498 106.579386,111.879563 107.068969,111.60873 C107.568969,111.32748 107.933552,110.947272 108.162719,110.468105 C108.339802,110.093105 108.428344,109.54623 108.428344,108.82748 L108.428344,108.218105 Z M111.975936,110.186855 L113.600936,109.936855 C113.68427,110.582688 113.93427,111.082688 114.350936,111.436855 C114.767603,111.780605 115.350936,111.95248 116.100936,111.95248 C116.850936,111.95248 117.408228,111.801438 117.772811,111.499355 C118.137395,111.186855 118.319686,110.822272 118.319686,110.405605 C118.319686,110.041022 118.158228,109.749355 117.835311,109.530605 C117.616561,109.384772 117.05927,109.20248 116.163436,108.98373 C114.955103,108.67123 114.116561,108.405605 113.647811,108.186855 C113.189478,107.968105 112.84052,107.666022 112.600936,107.280605 C112.361353,106.884772 112.241561,106.447272 112.241561,105.968105 C112.241561,105.541022 112.34052,105.145188 112.538436,104.780605 C112.736353,104.405605 113.007186,104.098313 113.350936,103.85873 C113.600936,103.67123 113.944686,103.51498 114.382186,103.38998 C114.830103,103.254563 115.304061,103.186855 115.804061,103.186855 C116.564478,103.186855 117.231145,103.29623 117.804061,103.51498 C118.376978,103.73373 118.798853,104.030605 119.069686,104.405605 C119.350936,104.780605 119.543645,105.280605 119.647811,105.905605 L118.038436,106.124355 C117.96552,105.624355 117.751978,105.23373 117.397811,104.95248 C117.054061,104.67123 116.564478,104.530605 115.929061,104.530605 C115.168645,104.530605 114.626978,104.655605 114.304061,104.905605 C113.991561,105.155605 113.835311,105.447272 113.835311,105.780605 C113.835311,105.988938 113.897811,106.181647 114.022811,106.35873 C114.158228,106.535813 114.37177,106.681647 114.663436,106.79623 C114.819686,106.85873 115.298853,106.999355 116.100936,107.218105 C117.267603,107.520188 118.080103,107.770188 118.538436,107.968105 C118.99677,108.166022 119.356145,108.45248 119.616561,108.82748 C119.876978,109.20248 120.007186,109.67123 120.007186,110.23373 C120.007186,110.785813 119.845728,111.301438 119.522811,111.780605 C119.210311,112.259772 118.751978,112.634772 118.147811,112.905605 C117.543645,113.166022 116.861353,113.29623 116.100936,113.29623 C114.850936,113.29623 113.892603,113.035813 113.225936,112.51498 C112.569686,111.994147 112.15302,111.218105 111.975936,110.186855 Z M125.538436,111.60873 L125.772811,113.061855 C125.314478,113.155605 124.90302,113.20248 124.538436,113.20248 C123.944686,113.20248 123.481145,113.10873 123.147811,112.92123 C122.824895,112.73373 122.595728,112.488938 122.460311,112.186855 C122.324895,111.874355 122.257186,111.228522 122.257186,110.249355 L122.257186,104.67123 L121.054061,104.67123 L121.054061,103.405605 L122.257186,103.405605 L122.257186,101.01498 L123.882186,100.030605 L123.882186,103.405605 L125.538436,103.405605 L125.538436,104.67123 L123.882186,104.67123 L123.882186,110.343105 C123.882186,110.801438 123.908228,111.098313 123.960311,111.23373 C124.022811,111.369147 124.116561,111.478522 124.241561,111.561855 C124.376978,111.634772 124.569686,111.67123 124.819686,111.67123 C124.99677,111.67123 125.236353,111.650397 125.538436,111.60873 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
+            </g>
+        </g>
+    </g>
+</svg>
\ No newline at end of file

From c6cdd3a4c2ae6b191b49c8412fec16c8d3a80aa2 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Thu, 19 Mar 2020 12:29:42 +0000
Subject: [PATCH 135/285] Revert "Report parser name and location in XContent
 deprecation warnings (#53752)"

This reverts commit 7636930ceb0c20ea34f2b9db42094a0a2af75f03.

There is some randomization in the YAML test suite which means we can't check
for exact xcontentlocation in the deprecation warning headers.
---
 .../org/elasticsearch/common/ParseField.java  | 24 ++--------
 .../common/xcontent/DeprecationHandler.java   | 48 ++++++-------------
 .../common/xcontent/ObjectParser.java         |  2 +-
 .../common/xcontent/XContentLocation.java     |  5 +-
 .../elasticsearch/common/ParseFieldTests.java |  6 +--
 .../common/xcontent/ObjectParserTests.java    |  2 +-
 .../common/ScriptProcessorFactoryTests.java   |  2 +-
 .../xcontent/LoggingDeprecationHandler.java   | 20 +++-----
 .../index/query/BoolQueryBuilderTests.java    |  2 +-
 .../script/StoredScriptTests.java             |  2 +-
 ...LoggingDeprecationAccumulationHandler.java | 29 +++++------
 .../utils/XContentObjectTransformerTests.java |  4 +-
 .../inference/ingest/InferenceProcessor.java  |  2 +-
 .../xpack/security/authc/ApiKeyService.java   | 23 ++++-----
 .../test/ml/data_frame_analytics_crud.yml     |  2 +-
 15 files changed, 58 insertions(+), 115 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
index 3872ef4852274..f4f12ed4f4c81 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
@@ -19,13 +19,11 @@
 package org.elasticsearch.common;
 
 import org.elasticsearch.common.xcontent.DeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentLocation;
 
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Supplier;
 
 /**
  * Holds a field that can be found in a request while parsing and its different
@@ -117,22 +115,6 @@ public ParseField withAllDeprecated() {
      *         names for this {@link ParseField}.
      */
     public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
-        return match(null, () -> XContentLocation.UNKNOWN, fieldName, deprecationHandler);
-    }
-
-    /**
-     * Does {@code fieldName} match this field?
-     * @param parserName
-     *            the name of the parent object holding this field
-     * @param location
-     *            the XContentLocation of the field
-     * @param fieldName
-     *            the field name to match against this {@link ParseField}
-     * @param deprecationHandler called if {@code fieldName} is deprecated
-     * @return true if <code>fieldName</code> matches any of the acceptable
-     *         names for this {@link ParseField}.
-     */
-    public boolean match(String parserName, Supplier<XContentLocation> location, String fieldName, DeprecationHandler deprecationHandler) {
         Objects.requireNonNull(fieldName, "fieldName cannot be null");
         // if this parse field has not been completely deprecated then try to
         // match the preferred name
@@ -145,11 +127,11 @@ public boolean match(String parserName, Supplier<XContentLocation> location, Str
         for (String depName : deprecatedNames) {
             if (fieldName.equals(depName)) {
                 if (fullyDeprecated) {
-                    deprecationHandler.usedDeprecatedField(parserName, location, fieldName);
+                    deprecationHandler.usedDeprecatedField(fieldName);
                 } else if (allReplacedWith == null) {
-                    deprecationHandler.usedDeprecatedName(parserName, location, fieldName, name);
+                    deprecationHandler.usedDeprecatedName(fieldName, name);
                 } else {
-                    deprecationHandler.usedDeprecatedField(parserName, location, fieldName, allReplacedWith);
+                    deprecationHandler.usedDeprecatedField(fieldName, allReplacedWith);
                 }
                 return true;
             }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
index 65f52943f0b86..2cfda82eb4c48 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.common.xcontent;
 
-import java.util.function.Supplier;
-
 /**
  * Callback for notifying the creator of the {@link XContentParser} that
  * parsing hit a deprecated field.
@@ -34,35 +32,20 @@ public interface DeprecationHandler {
      */
     DeprecationHandler THROW_UNSUPPORTED_OPERATION = new DeprecationHandler() {
         @Override
-        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
-            if (parserName != null) {
-                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
-                    + usedName + "] at [" + location.get() + "] which is a deprecated name for [" + replacedWith + "]");
-            } else {
-                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                    + usedName + "] which is a deprecated name for [" + replacedWith + "]");
-            }
+        public void usedDeprecatedField(String usedName, String replacedWith) {
+            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                + usedName + "] which is a deprecated name for [" + replacedWith + "]");
         }
         @Override
-        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
-            if (parserName != null) {
-                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
-                    + usedName + "] at [" + location.get() + "] which has been replaced with [" + modernName + "]");
-            } else {
-                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                    + usedName + "] which has been replaced with [" + modernName + "]");
-            }
+        public void usedDeprecatedName(String usedName, String modernName) {
+            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                + usedName + "] which has been replaced with [" + modernName + "]");
         }
 
         @Override
-        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
-            if (parserName != null) {
-                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
-                    + usedName + "] at [" + location.get() + "] which has been deprecated entirely");
-            } else {
-                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                    + usedName + "] which has been deprecated entirely");
-            }
+        public void usedDeprecatedField(String usedName) {
+            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                + usedName + "] which has been deprecated entirely");
         }
     };
 
@@ -71,17 +54,17 @@ public void usedDeprecatedField(String parserName, Supplier<XContentLocation> lo
      */
     DeprecationHandler IGNORE_DEPRECATIONS = new DeprecationHandler() {
         @Override
-        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+        public void usedDeprecatedName(String usedName, String modernName) {
 
         }
 
         @Override
-        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+        public void usedDeprecatedField(String usedName, String replacedWith) {
 
         }
 
         @Override
-        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+        public void usedDeprecatedField(String usedName) {
 
         }
     };
@@ -91,7 +74,7 @@ public void usedDeprecatedField(String parserName, Supplier<XContentLocation> lo
      * @param usedName the provided field name
      * @param modernName the modern name for the field
      */
-    void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName);
+    void usedDeprecatedName(String usedName, String modernName);
 
     /**
      * Called when the provided field name matches the current field but the entire
@@ -99,13 +82,12 @@ public void usedDeprecatedField(String parserName, Supplier<XContentLocation> lo
      * @param usedName the provided field name
      * @param replacedWith the name of the field that replaced this field
      */
-    void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith);
+    void usedDeprecatedField(String usedName, String replacedWith);
 
     /**
      * Called when the provided field name matches the current field but the entire
      * field has been marked as deprecated with no replacement
      * @param usedName the provided field name
      */
-    void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName);
-
+    void usedDeprecatedField(String usedName);
 }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index 73bbf812f598c..b11f6afd4bb26 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -571,7 +571,7 @@ private class FieldParser {
         }
 
         void assertSupports(String parserName, XContentParser parser, String currentFieldName) {
-            if (parseField.match(parserName, parser::getTokenLocation, currentFieldName, parser.getDeprecationHandler()) == false) {
+            if (parseField.match(currentFieldName, parser.getDeprecationHandler()) == false) {
                 throw new XContentParseException(parser.getTokenLocation(),
                         "[" + parserName  + "] parsefield doesn't accept: " + currentFieldName);
             }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
index 1d5bfd6c2c429..43ab7503cd1dd 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
@@ -25,10 +25,7 @@
  * position of a parsing error to end users and consequently have line and
  * column numbers starting from 1.
  */
-public final class XContentLocation {
-
-    public static final XContentLocation UNKNOWN = new XContentLocation(-1, -1);
-
+public class XContentLocation {
     public final int lineNumber;
     public final int columnNumber;
 
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
index e7420356c8fac..381c7c3e959be 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
@@ -66,11 +66,11 @@ public void testDeprecatedWithNoReplacement() {
         ParseField field = new ParseField(name).withDeprecation(alternatives).withAllDeprecated();
         assertFalse(field.match("not a field name", LoggingDeprecationHandler.INSTANCE));
         assertTrue(field.match("dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [dep] used, this field is unused and will be removed entirely");
+        assertWarnings("Deprecated field [dep] used, which has been removed entirely");
         assertTrue(field.match("old_dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [old_dep] used, this field is unused and will be removed entirely");
+        assertWarnings("Deprecated field [old_dep] used, which has been removed entirely");
         assertTrue(field.match("new_dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [new_dep] used, this field is unused and will be removed entirely");
+        assertWarnings("Deprecated field [new_dep] used, which has been removed entirely");
 
 
     }
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index 9a813c8a24616..c99d1b10d6a4d 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -223,7 +223,7 @@ class TestStruct {
         objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
         objectParser.parse(parser, s, null);
         assertEquals("foo", s.test);
-        assertWarnings("[foo][1:15] Deprecated field [old_test] used, expected [test] instead");
+        assertWarnings("Deprecated field [old_test] used, expected [test] instead");
     }
 
     public void testFailOnValueType() throws IOException {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index c7018d40b18ca..9559c150df68a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -98,7 +98,7 @@ public void testInlineBackcompat() throws Exception {
         configMap.put("inline", "code");
 
         factory.create(null, randomAlphaOfLength(10), configMap);
-        assertWarnings("[script][1:11] Deprecated field [inline] used, expected [source] instead");
+        assertWarnings("Deprecated field [inline] used, expected [source] instead");
     }
 
     public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception {
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
index 5feca4bfaab5a..9a885b55d146c 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
@@ -23,8 +23,6 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.logging.DeprecationLogger;
 
-import java.util.function.Supplier;
-
 /**
  * Logs deprecations to the {@link DeprecationLogger}.
  * <p>
@@ -51,23 +49,17 @@ private LoggingDeprecationHandler() {
     }
 
     @Override
-    public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
-        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-        deprecationLogger.deprecated("{}Deprecated field [{}] used, expected [{}] instead",
-            prefix, usedName, modernName);
+    public void usedDeprecatedName(String usedName, String modernName) {
+        deprecationLogger.deprecated("Deprecated field [{}] used, expected [{}] instead", usedName, modernName);
     }
 
     @Override
-    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
-        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-        deprecationLogger.deprecated("{}Deprecated field [{}] used, replaced by [{}]",
-            prefix, usedName, replacedWith);
+    public void usedDeprecatedField(String usedName, String replacedWith) {
+        deprecationLogger.deprecated("Deprecated field [{}] used, replaced by [{}]", usedName, replacedWith);
     }
 
     @Override
-    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
-        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-        deprecationLogger.deprecated("{}Deprecated field [{}] used, this field is unused and will be removed entirely",
-            prefix, usedName);
+    public void usedDeprecatedField(String usedName) {
+        deprecationLogger.deprecated("Deprecated field [{}] used, which has been removed entirely", usedName);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 03916bac58a84..93d46ed592570 100644
--- a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -302,7 +302,7 @@ public void testDeprecation() throws IOException {
         QueryBuilder q = parseQuery(query);
         QueryBuilder expected = new BoolQueryBuilder().mustNot(new MatchAllQueryBuilder());
         assertEquals(expected, q);
-        assertWarnings("[bool][1:24] Deprecated field [mustNot] used, expected [must_not] instead");
+        assertWarnings("Deprecated field [mustNot] used, expected [must_not] instead");
     }
 
     public void testRewrite() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java b/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java
index 47dc7aaae2a3b..627d67dc833e4 100644
--- a/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java
@@ -104,7 +104,7 @@ public void testSourceParsing() throws Exception {
 
             assertThat(parsed, equalTo(source));
         }
-        assertWarnings("[stored script source][1:33] Deprecated field [code] used, expected [source] instead");
+        assertWarnings("Deprecated field [code] used, expected [source] instead");
 
         // complex script with script object and empty options
         try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
index 323e3a84cdf2e..5e2a4d7a25302 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
@@ -8,12 +8,10 @@
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentLocation;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.function.Supplier;
 
 /**
  * Very similar to {@link org.elasticsearch.common.xcontent.LoggingDeprecationHandler} main differences are:
@@ -28,27 +26,24 @@ public class LoggingDeprecationAccumulationHandler implements DeprecationHandler
     private final List<String> deprecations = new ArrayList<>();
 
     @Override
-    public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(parserName, location, usedName, modernName);
-        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, expected [{}] instead",
-            new Object[]{prefix, usedName, modernName}));
+    public void usedDeprecatedName(String usedName, String modernName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(usedName, modernName);
+        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, expected [{}] instead",
+            new Object[] {usedName, modernName}));
     }
 
     @Override
-    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(parserName, location, usedName, replacedWith);
-        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, replaced by [{}]",
-            new Object[]{prefix, usedName, replacedWith}));
+    public void usedDeprecatedField(String usedName, String replacedWith) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName, replacedWith);
+        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, replaced by [{}]",
+            new Object[] {usedName, replacedWith}));
     }
 
     @Override
-    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(parserName, location, usedName);
-        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, unused and will be removed entirely",
-            new Object[]{prefix, usedName}));
+    public void usedDeprecatedField(String usedName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName);
+        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, which has been deprecated entirely",
+            new Object[]{ usedName }));
     }
 
     /**
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
index 886602395238f..d6ebc9a2e150b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
@@ -127,8 +127,8 @@ public void testToMap() throws IOException {
     public void testDeprecationWarnings() throws IOException {
         XContentObjectTransformer<QueryBuilder> queryBuilderTransformer = new XContentObjectTransformer<>(NamedXContentRegistry.EMPTY,
             (p)-> {
-            p.getDeprecationHandler().usedDeprecatedField(null, null, "oldField", "newField");
-            p.getDeprecationHandler().usedDeprecatedName(null, null, "oldName", "modernName");
+            p.getDeprecationHandler().usedDeprecatedField("oldField", "newField");
+            p.getDeprecationHandler().usedDeprecatedName("oldName", "modernName");
             return new BoolQueryBuilder();
             });
         List<String> deprecations = new ArrayList<>();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index b52b8b5b64e3f..2a4861c5247f7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -242,7 +242,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
                 fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
                 //TODO Remove in 8.x
                 if (fieldMap != null) {
-                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, FIELD_MAPPINGS, FIELD_MAP);
+                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
                 }
             }
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index c363ad1253509..f36ce90660a23 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -49,7 +49,6 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -619,25 +618,21 @@ private ApiKeyLoggingDeprecationHandler(DeprecationLogger logger, String apiKeyI
         }
 
         @Override
-        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
-            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-            deprecationLogger.deprecated("{}Deprecated field [{}] used in api key [{}], expected [{}] instead",
-                prefix, usedName, apiKeyId, modernName);
+        public void usedDeprecatedName(String usedName, String modernName) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], expected [{}] instead",
+                usedName, apiKeyId, modernName);
         }
 
         @Override
-        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
-            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-            deprecationLogger.deprecated("{}Deprecated field [{}] used in api key [{}], replaced by [{}]",
-                prefix, usedName, apiKeyId, replacedWith);
+        public void usedDeprecatedField(String usedName, String replacedWith) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], replaced by [{}]",
+                usedName, apiKeyId, replacedWith);
         }
 
         @Override
-        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
-            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
-            deprecationLogger.deprecated(
-                "{}Deprecated field [{}] used in api key [{}], which is unused and will be removed entirely",
-                prefix, usedName);
+        public void usedDeprecatedField(String usedName) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], which has been removed entirely",
+                usedName);
         }
     }
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index c5df3d9f6cc51..a5a99b30391ea 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -1869,7 +1869,7 @@ setup:
 
   - do:
       allowed_warnings:
-        - '[classification][1:139] Deprecated field [maximum_number_trees] used, expected [max_trees] instead'
+        - 'Deprecated field [maximum_number_trees] used, expected [max_trees] instead'
       ml.put_data_frame_analytics:
         id: "classification-with-maximum-number-trees"
         body: >

From 6eb698bc6d36e56c2e9e0f2506645602c15b157f Mon Sep 17 00:00:00 2001
From: Ignacio Vera <iverase@gmail.com>
Date: Thu, 19 Mar 2020 14:16:08 +0100
Subject: [PATCH 136/285] Add support for distance queries on geo_shape queries
 (#53466)

With the upgrade to Lucene 8.5, LatLonShape field has support for distance queries. This change implements this new feature and removes the limitation.
---
 .../common/geo/GeoShapeUtils.java             |  66 +++++++
 .../index/mapper/GeoShapeIndexer.java         |  17 +-
 .../VectorGeoPointShapeQueryProcessor.java    |   5 +-
 .../query/VectorGeoShapeQueryProcessor.java   | 170 +++++++++---------
 .../search/geo/GeoShapeQueryTests.java        |  40 +++++
 5 files changed, 197 insertions(+), 101 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java

diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java
new file mode 100644
index 0000000000000..7e23385f2adf8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo;
+
+import org.elasticsearch.geometry.Circle;
+import org.elasticsearch.geometry.Line;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
+
+
+/**
+ * Utility class that transforms Elasticsearch geometry objects to the Lucene representation
+ */
+public class GeoShapeUtils {
+
+    public static org.apache.lucene.geo.Polygon toLucenePolygon(Polygon polygon) {
+        org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[polygon.getNumberOfHoles()];
+        for(int i = 0; i<holes.length; i++) {
+            holes[i] = new org.apache.lucene.geo.Polygon(polygon.getHole(i).getY(), polygon.getHole(i).getX());
+        }
+        return new org.apache.lucene.geo.Polygon(polygon.getPolygon().getY(), polygon.getPolygon().getX(), holes);
+    }
+
+    public static org.apache.lucene.geo.Polygon toLucenePolygon(Rectangle r) {
+        return new org.apache.lucene.geo.Polygon(
+            new double[]{r.getMinLat(), r.getMinLat(), r.getMaxLat(), r.getMaxLat(), r.getMinLat()},
+            new double[]{r.getMinLon(), r.getMaxLon(), r.getMaxLon(), r.getMinLon(), r.getMinLon()});
+    }
+
+    public static org.apache.lucene.geo.Rectangle toLuceneRectangle(Rectangle r) {
+        return new org.apache.lucene.geo.Rectangle(r.getMinLat(), r.getMaxLat(), r.getMinLon(), r.getMaxLon());
+    }
+
+    public static org.apache.lucene.geo.Point toLucenePoint(Point point) {
+        return new org.apache.lucene.geo.Point(point.getLat(), point.getLon());
+    }
+
+    public static org.apache.lucene.geo.Line toLuceneLine(Line line) {
+        return new org.apache.lucene.geo.Line(line.getLats(), line.getLons());
+    }
+
+    public static org.apache.lucene.geo.Circle toLuceneCircle(Circle circle) {
+        return new org.apache.lucene.geo.Circle(circle.getLat(), circle.getLon(), circle.getRadiusMeters());
+    }
+
+    private GeoShapeUtils() {
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
index 0b7458de58b16..9a103471a21a3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.geo.GeoLineDecomposer;
 import org.elasticsearch.common.geo.GeoPolygonDecomposer;
+import org.elasticsearch.common.geo.GeoShapeUtils;
 import org.elasticsearch.common.geo.GeoShapeType;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
@@ -213,7 +214,7 @@ public Void visit(GeometryCollection<?> collection) {
 
         @Override
         public Void visit(Line line) {
-            addFields(LatLonShape.createIndexableFields(name, new org.apache.lucene.geo.Line(line.getY(), line.getX())));
+            addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLuceneLine(line)));
             return null;
         }
 
@@ -254,16 +255,13 @@ public Void visit(Point point) {
 
         @Override
         public Void visit(Polygon polygon) {
-            addFields(LatLonShape.createIndexableFields(name, toLucenePolygon(polygon)));
+            addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLucenePolygon(polygon)));
             return null;
         }
 
         @Override
         public Void visit(Rectangle r) {
-            org.apache.lucene.geo.Polygon p = new org.apache.lucene.geo.Polygon(
-                new double[]{r.getMinY(), r.getMinY(), r.getMaxY(), r.getMaxY(), r.getMinY()},
-                new double[]{r.getMinX(), r.getMaxX(), r.getMaxX(), r.getMinX(), r.getMinX()});
-            addFields(LatLonShape.createIndexableFields(name, p));
+            addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLucenePolygon(r)));
             return null;
         }
 
@@ -272,11 +270,4 @@ private void addFields(IndexableField[] fields) {
         }
     }
 
-    public static org.apache.lucene.geo.Polygon toLucenePolygon(Polygon polygon) {
-        org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[polygon.getNumberOfHoles()];
-        for(int i = 0; i<holes.length; i++) {
-            holes[i] = new org.apache.lucene.geo.Polygon(polygon.getHole(i).getY(), polygon.getHole(i).getX());
-        }
-        return new org.apache.lucene.geo.Polygon(polygon.getPolygon().getY(), polygon.getPolygon().getX(), holes);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java b/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java
index d9b8d5a28b1ab..680544c5c7484 100644
--- a/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java
+++ b/server/src/main/java/org/elasticsearch/index/query/VectorGeoPointShapeQueryProcessor.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.geo.GeoPolygonDecomposer;
 import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.GeoShapeUtils;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
@@ -46,8 +47,6 @@
 
 import java.util.ArrayList;
 
-import static org.elasticsearch.index.mapper.GeoShapeIndexer.toLucenePolygon;
-
 public class VectorGeoPointShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
 
     @Override
@@ -145,7 +144,7 @@ private Query visit(ArrayList<Polygon> collector) {
             org.apache.lucene.geo.Polygon[] lucenePolygons =
                 new org.apache.lucene.geo.Polygon[collector.size()];
             for (int i = 0; i < collector.size(); i++) {
-                lucenePolygons[i] = toLucenePolygon(collector.get(i));
+                lucenePolygons[i] = GeoShapeUtils.toLucenePolygon(collector.get(i));
             }
             Query query = LatLonPoint.newPolygonQuery(fieldName, lucenePolygons);
             if (fieldType.hasDocValues()) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java b/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
index 4d79437bf239a..f40d745399579 100644
--- a/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
+++ b/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java
@@ -20,31 +20,32 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.document.LatLonShape;
-import org.apache.lucene.document.ShapeField;
-import org.apache.lucene.geo.Line;
-import org.apache.lucene.geo.Polygon;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.geo.LatLonGeometry;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.geo.GeoLineDecomposer;
+import org.elasticsearch.common.geo.GeoPolygonDecomposer;
+import org.elasticsearch.common.geo.GeoShapeUtils;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
 import org.elasticsearch.geometry.GeometryVisitor;
+import org.elasticsearch.geometry.Line;
 import org.elasticsearch.geometry.LinearRing;
 import org.elasticsearch.geometry.MultiLine;
 import org.elasticsearch.geometry.MultiPoint;
 import org.elasticsearch.geometry.MultiPolygon;
 import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Polygon;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.index.mapper.AbstractSearchableGeometryFieldType;
-import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
-import org.elasticsearch.index.mapper.GeoShapeIndexer;
-import org.elasticsearch.index.mapper.MappedFieldType;
 
-import static org.elasticsearch.index.mapper.GeoShapeIndexer.toLucenePolygon;
+
+import java.util.ArrayList;
+import java.util.List;
+
 
 public class VectorGeoShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
 
@@ -59,127 +60,126 @@ public Query process(Geometry shape, String fieldName, ShapeRelation relation, Q
         return getVectorQueryFromShape(shape, fieldName, relation, context);
     }
 
-    protected Query getVectorQueryFromShape(
-        Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
-        GeoShapeIndexer geometryIndexer = new GeoShapeIndexer(true, fieldName);
-
-        Geometry processedShape = geometryIndexer.prepareForIndexing(queryShape);
-
-        if (processedShape == null) {
+    private Query getVectorQueryFromShape(Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+        final LuceneGeometryCollector visitor = new LuceneGeometryCollector(fieldName, context);
+        queryShape.visit(visitor);
+        final List<LatLonGeometry> geometries = visitor.geometries();
+        if (geometries.size() == 0) {
             return new MatchNoDocsQuery();
         }
-        return processedShape.visit(new ShapeVisitor(context, fieldName, relation));
+        return LatLonShape.newGeometryQuery(fieldName, relation.getLuceneRelation(),
+            geometries.toArray(new LatLonGeometry[geometries.size()]));
     }
 
-    private class ShapeVisitor implements GeometryVisitor<Query, RuntimeException> {
-        QueryShardContext context;
-        MappedFieldType fieldType;
-        String fieldName;
-        ShapeRelation relation;
+    private static class LuceneGeometryCollector implements GeometryVisitor<Void, RuntimeException> {
+        private final List<LatLonGeometry> geometries = new ArrayList<>();
+        private final String name;
+        private final QueryShardContext context;
 
-        ShapeVisitor(QueryShardContext context, String fieldName, ShapeRelation relation) {
+        private LuceneGeometryCollector(String name, QueryShardContext context) {
+            this.name = name;
             this.context = context;
-            this.fieldType = context.fieldMapper(fieldName);
-            this.fieldName = fieldName;
-            this.relation = relation;
         }
 
-        @Override
-        public Query visit(Circle circle) {
-            throw new QueryShardException(context, "Field [" + fieldName + "] found an unknown shape Circle");
+        List<LatLonGeometry> geometries() {
+            return geometries;
         }
 
         @Override
-        public Query visit(GeometryCollection<?> collection) {
-            BooleanQuery.Builder bqb = new BooleanQuery.Builder();
-            visit(bqb, collection);
-            return bqb.build();
-        }
-
-        private void visit(BooleanQuery.Builder bqb, GeometryCollection<?> collection) {
-            BooleanClause.Occur occur;
-            if (relation == ShapeRelation.CONTAINS || relation == ShapeRelation.DISJOINT) {
-                // all shapes must be disjoint / must be contained in relation to the indexed shape.
-                occur = BooleanClause.Occur.MUST;
-            } else {
-                // at least one shape must intersect / contain the indexed shape.
-                occur = BooleanClause.Occur.SHOULD;
+        public Void visit(Circle circle) {
+            if (circle.isEmpty() == false) {
+                geometries.add(GeoShapeUtils.toLuceneCircle(circle));
             }
+            return null;
+        }
+
+        @Override
+        public Void visit(GeometryCollection<?> collection) {
             for (Geometry shape : collection) {
-                bqb.add(shape.visit(this), occur);
+                shape.visit(this);
             }
+            return null;
         }
 
         @Override
-        public Query visit(org.elasticsearch.geometry.Line line) {
-            validateIsGeoShapeFieldType();
-            return LatLonShape.newLineQuery(fieldName, relation.getLuceneRelation(), new Line(line.getY(), line.getX()));
+        public Void visit(org.elasticsearch.geometry.Line line) {
+            if (line.isEmpty() == false) {
+                List<org.elasticsearch.geometry.Line> collector = new ArrayList<>();
+                GeoLineDecomposer.decomposeLine(line, collector);
+                collectLines(collector);
+            }
+            return null;
         }
 
         @Override
-        public Query visit(LinearRing ring) {
-            throw new QueryShardException(context, "Field [" + fieldName + "] found an unsupported shape LinearRing");
+        public Void visit(LinearRing ring) {
+            throw new QueryShardException(context, "Field [" + name + "] found and unsupported shape LinearRing");
         }
 
         @Override
-        public Query visit(MultiLine multiLine) {
-            validateIsGeoShapeFieldType();
-            Line[] lines = new Line[multiLine.size()];
-            for (int i = 0; i < multiLine.size(); i++) {
-                lines[i] = new Line(multiLine.get(i).getY(), multiLine.get(i).getX());
-            }
-            return LatLonShape.newLineQuery(fieldName, relation.getLuceneRelation(), lines);
+        public Void visit(MultiLine multiLine) {
+            List<org.elasticsearch.geometry.Line> collector = new ArrayList<>();
+            GeoLineDecomposer.decomposeMultiLine(multiLine, collector);
+            collectLines(collector);
+            return null;
         }
 
         @Override
-        public Query visit(MultiPoint multiPoint) {
-            double[][] points = new double[multiPoint.size()][2];
-            for (int i = 0; i < multiPoint.size(); i++) {
-                points[i] = new double[] {multiPoint.get(i).getLat(), multiPoint.get(i).getLon()};
+        public Void visit(MultiPoint multiPoint) {
+            for (Point point : multiPoint) {
+                visit(point);
             }
-            return LatLonShape.newPointQuery(fieldName, relation.getLuceneRelation(), points);
+            return null;
         }
 
         @Override
-        public Query visit(MultiPolygon multiPolygon) {
-            Polygon[] polygons = new Polygon[multiPolygon.size()];
-            for (int i = 0; i < multiPolygon.size(); i++) {
-                polygons[i] = toLucenePolygon(multiPolygon.get(i));
+        public Void visit(MultiPolygon multiPolygon) {
+            if (multiPolygon.isEmpty() == false) {
+                List<org.elasticsearch.geometry.Polygon> collector = new ArrayList<>();
+                GeoPolygonDecomposer.decomposeMultiPolygon(multiPolygon, true, collector);
+                collectPolygons(collector);
             }
-            return LatLonShape.newPolygonQuery(fieldName, relation.getLuceneRelation(), polygons);
+            return null;
         }
 
         @Override
-        public Query visit(Point point) {
-            validateIsGeoShapeFieldType();
-            ShapeField.QueryRelation luceneRelation = relation.getLuceneRelation();
-            if (luceneRelation == ShapeField.QueryRelation.CONTAINS) {
-                // contains and intersects are equivalent but the implementation of
-                // intersects is more efficient.
-                luceneRelation = ShapeField.QueryRelation.INTERSECTS;
+        public Void visit(Point point) {
+            if (point.isEmpty() == false) {
+                geometries.add(GeoShapeUtils.toLucenePoint(point));
             }
-            return LatLonShape.newPointQuery(fieldName, luceneRelation,
-                new double[] {point.getY(), point.getX()});
+            return null;
+
         }
 
         @Override
-        public Query visit(org.elasticsearch.geometry.Polygon polygon) {
-            return LatLonShape.newPolygonQuery(fieldName, relation.getLuceneRelation(), toLucenePolygon(polygon));
+        public Void visit(org.elasticsearch.geometry.Polygon polygon) {
+            if (polygon.isEmpty() == false) {
+                List<org.elasticsearch.geometry.Polygon> collector = new ArrayList<>();
+                GeoPolygonDecomposer.decomposePolygon(polygon, true, collector);
+                collectPolygons(collector);
+            }
+            return null;
         }
 
         @Override
-        public Query visit(Rectangle r) {
-            return LatLonShape.newBoxQuery(fieldName, relation.getLuceneRelation(),
-                r.getMinY(), r.getMaxY(), r.getMinX(), r.getMaxX());
+        public Void visit(Rectangle r) {
+            if (r.isEmpty() == false) {
+                geometries.add(GeoShapeUtils.toLuceneRectangle(r));
+            }
+            return null;
         }
 
-        private void validateIsGeoShapeFieldType() {
-            if (fieldType instanceof GeoShapeFieldMapper.GeoShapeFieldType == false) {
-                throw new QueryShardException(context, "Expected " + GeoShapeFieldMapper.CONTENT_TYPE
-                    + " field type for Field [" + fieldName + "] but found " + fieldType.typeName());
+        private void collectLines(List<org.elasticsearch.geometry.Line> geometryLines) {
+            for (Line line: geometryLines) {
+                geometries.add(GeoShapeUtils.toLuceneLine(line));
             }
         }
-    }
 
+        private void collectPolygons(List<org.elasticsearch.geometry.Polygon> geometryPolygons) {
+            for (Polygon polygon : geometryPolygons) {
+                geometries.add(GeoShapeUtils.toLucenePolygon(polygon));
+            }
+        }
+    }
 }
 
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index afb87348a541f..032c6d68766be 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.SpatialStrategy;
+import org.elasticsearch.common.geo.builders.CircleBuilder;
 import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
 import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
 import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
@@ -37,6 +38,7 @@
 import org.elasticsearch.common.geo.builders.PolygonBuilder;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -751,4 +753,42 @@ public void testShapeFilterWithDefinedGeoCollection() throws Exception {
         assertSearchResponse(result);
         assertHitCount(result, 0);
     }
+
+    public void testDistanceQuery() throws Exception {
+        String mapping = Strings.toString(createRandomMapping());
+        client().admin().indices().prepareCreate("test_distance").setMapping(mapping).get();
+        ensureGreen();
+
+        CircleBuilder circleBuilder = new CircleBuilder().center(new Coordinate(1, 0)).radius(350, DistanceUnit.KILOMETERS);
+
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("geo", new PointBuilder(2, 2)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("geo", new PointBuilder(3, 1)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("geo", new PointBuilder(-20, -30)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("geo", new PointBuilder(20, 30)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+
+        SearchResponse response = client().prepareSearch("test_distance")
+            .setQuery(QueryBuilders.geoShapeQuery("geo", circleBuilder.buildGeometry()).relation(ShapeRelation.WITHIN))
+            .get();
+        assertEquals(2, response.getHits().getTotalHits().value);
+        response = client().prepareSearch("test_distance")
+            .setQuery(QueryBuilders.geoShapeQuery("geo", circleBuilder.buildGeometry()).relation(ShapeRelation.INTERSECTS))
+            .get();
+        assertEquals(2, response.getHits().getTotalHits().value);
+        response = client().prepareSearch("test_distance")
+            .setQuery(QueryBuilders.geoShapeQuery("geo", circleBuilder.buildGeometry()).relation(ShapeRelation.DISJOINT))
+            .get();
+        assertEquals(2, response.getHits().getTotalHits().value);
+        response = client().prepareSearch("test_distance")
+            .setQuery(QueryBuilders.geoShapeQuery("geo", circleBuilder.buildGeometry()).relation(ShapeRelation.CONTAINS))
+            .get();
+        assertEquals(0, response.getHits().getTotalHits().value);
+    }
 }

From 112ae9cb1796737e6b4d92caf17965c7cfed0f6f Mon Sep 17 00:00:00 2001
From: Ignacio Vera <iverase@gmail.com>
Date: Thu, 19 Mar 2020 14:27:38 +0100
Subject: [PATCH 137/285] Add support for distance queries on shape queries
 (#53468)

With the upgrade to Lucene 8.5, XYShape field has support for distance queries. This change implements this new feature and removes the limitation.
---
 .../spatial/index/mapper/ShapeIndexer.java    |  37 +----
 .../spatial/index/mapper/ShapeUtils.java      |  68 ++++++++
 .../index/query/ShapeQueryBuilder.java        |   3 +-
 .../index/query/ShapeQueryProcessor.java      | 155 ++++++++----------
 .../xpack/spatial/search/ShapeQueryTests.java |  40 +++++
 5 files changed, 186 insertions(+), 117 deletions(-)
 create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeUtils.java

diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java
index 2df89bee955ba..7346a94c5f180 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java
@@ -6,8 +6,6 @@
 package org.elasticsearch.xpack.spatial.index.mapper;
 
 import org.apache.lucene.document.XYShape;
-import org.apache.lucene.geo.XYLine;
-import org.apache.lucene.geo.XYPolygon;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
@@ -75,8 +73,7 @@ public Void visit(GeometryCollection<?> collection) {
 
         @Override
         public Void visit(Line line) {
-            float[][] vertices = lineToFloatArray(line.getX(), line.getY());
-            addFields(XYShape.createIndexableFields(name, new XYLine(vertices[0], vertices[1])));
+            addFields(XYShape.createIndexableFields(name, ShapeUtils.toLuceneXYLine(line)));
             return null;
         }
 
@@ -111,22 +108,19 @@ public Void visit(MultiPolygon multiPolygon) {
 
         @Override
         public Void visit(Point point) {
-            addFields(XYShape.createIndexableFields(name, (float)point.getX(), (float)point.getY()));
+            addFields(XYShape.createIndexableFields(name, (float) point.getX(), (float) point.getY()));
             return null;
         }
 
         @Override
         public Void visit(Polygon polygon) {
-            addFields(XYShape.createIndexableFields(name, toLucenePolygon(polygon)));
+            addFields(XYShape.createIndexableFields(name, ShapeUtils.toLuceneXYPolygon(polygon)));
             return null;
         }
 
         @Override
         public Void visit(Rectangle r) {
-            XYPolygon p = new XYPolygon(
-                new float[]{(float)r.getMinX(), (float)r.getMaxX(), (float)r.getMaxX(), (float)r.getMinX(), (float)r.getMinX()},
-                new float[]{(float)r.getMinY(), (float)r.getMinY(), (float)r.getMaxY(), (float)r.getMaxY(), (float)r.getMinY()});
-            addFields(XYShape.createIndexableFields(name, p));
+            addFields(XYShape.createIndexableFields(name, ShapeUtils.toLuceneXYPolygon(r)));
             return null;
         }
 
@@ -134,27 +128,4 @@ private void addFields(IndexableField[] fields) {
             this.fields.addAll(Arrays.asList(fields));
         }
     }
-
-    public static XYPolygon toLucenePolygon(Polygon polygon) {
-        XYPolygon[] holes = new XYPolygon[polygon.getNumberOfHoles()];
-        LinearRing ring;
-        float[][] vertices;
-        for(int i = 0; i<holes.length; i++) {
-            ring = polygon.getHole(i);
-            vertices = lineToFloatArray(ring.getX(), ring.getY());
-            holes[i] = new XYPolygon(vertices[0], vertices[1]);
-        }
-        ring = polygon.getPolygon();
-        vertices = lineToFloatArray(ring.getX(), ring.getY());
-        return new XYPolygon(vertices[0], vertices[1], holes);
-    }
-
-    private static float[][] lineToFloatArray(double[] x, double[] y) {
-        float[][] result = new float[2][x.length];
-        for (int i = 0; i < x.length; ++i) {
-            result[0][i] = (float)x[i];
-            result[1][i] = (float)y[i];
-        }
-        return result;
-    }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeUtils.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeUtils.java
new file mode 100644
index 0000000000000..a5bf308e28fcf
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeUtils.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.spatial.index.mapper;
+
+import org.elasticsearch.geometry.Circle;
+import org.elasticsearch.geometry.Line;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
+
+
+/**
+ * Utility class that transforms Elasticsearch geometry objects to the Lucene representation
+ */
+public class ShapeUtils {
+
+    public static org.apache.lucene.geo.XYPolygon toLuceneXYPolygon(Polygon polygon) {
+        org.apache.lucene.geo.XYPolygon[] holes = new org.apache.lucene.geo.XYPolygon[polygon.getNumberOfHoles()];
+        for(int i = 0; i<holes.length; i++) {
+            holes[i] = new org.apache.lucene.geo.XYPolygon(
+                doubleArrayToFloatArray(polygon.getHole(i).getX()),
+                doubleArrayToFloatArray(polygon.getHole(i).getY()));
+        }
+        return new org.apache.lucene.geo.XYPolygon(
+            doubleArrayToFloatArray(polygon.getPolygon().getX()),
+            doubleArrayToFloatArray(polygon.getPolygon().getY()), holes);
+    }
+
+    public static org.apache.lucene.geo.XYPolygon toLuceneXYPolygon(Rectangle r) {
+        return new org.apache.lucene.geo.XYPolygon(
+            new float[]{(float) r.getMinX(), (float) r.getMaxX(), (float) r.getMaxX(), (float) r.getMinX(), (float) r.getMinX()},
+            new float[]{(float) r.getMinY(), (float) r.getMinY(), (float) r.getMaxY(), (float) r.getMaxY(), (float) r.getMinY()});
+    }
+
+    public static org.apache.lucene.geo.XYRectangle toLuceneXYRectangle(Rectangle r) {
+        return new org.apache.lucene.geo.XYRectangle((float) r.getMinX(), (float) r.getMaxX(),
+                                                     (float) r.getMinY(), (float) r.getMaxY());
+    }
+
+    public static org.apache.lucene.geo.XYPoint toLuceneXYPoint(Point point) {
+        return new org.apache.lucene.geo.XYPoint((float) point.getX(), (float) point.getY());
+    }
+
+    public static org.apache.lucene.geo.XYLine toLuceneXYLine(Line line) {
+        return new org.apache.lucene.geo.XYLine(
+            doubleArrayToFloatArray(line.getX()),
+            doubleArrayToFloatArray(line.getY()));
+    }
+
+    public static org.apache.lucene.geo.XYCircle toLuceneXYCircle(Circle circle) {
+        return new org.apache.lucene.geo.XYCircle((float) circle.getX(), (float) circle.getY(), (float) circle.getRadiusMeters());
+    }
+
+    private ShapeUtils() {
+    }
+
+    private static float[] doubleArrayToFloatArray(double[] array) {
+        float[] result = new float[array.length];
+        for (int i = 0; i < array.length; ++i) {
+            result[i] = (float) array[i];
+        }
+        return result;
+    }
+
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
index a92e827b8f66e..1989b4ab18019 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.spatial.index.query;
 
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
@@ -121,7 +122,7 @@ public Query buildShapeQuery(QueryShardContext context, MappedFieldType fieldTyp
         }
 
         final AbstractGeometryFieldMapper.AbstractGeometryFieldType ft = (AbstractGeometryFieldMapper.AbstractGeometryFieldType) fieldType;
-        return ft.geometryQueryBuilder().process(shape, ft.name(), relation, context);
+        return  new ConstantScoreQuery(ft.geometryQueryBuilder().process(shape, ft.name(), relation, context));
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
index da9bc32355bac..b3f4d5fcdb033 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
@@ -5,13 +5,8 @@
  */
 package org.elasticsearch.xpack.spatial.index.query;
 
-import org.apache.lucene.document.ShapeField;
 import org.apache.lucene.document.XYShape;
-import org.apache.lucene.geo.XYLine;
-import org.apache.lucene.geo.XYPolygon;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.geo.XYGeometry;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
@@ -33,24 +28,26 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper;
+import org.elasticsearch.xpack.spatial.index.mapper.ShapeUtils;
+
+import java.util.ArrayList;
+import java.util.List;
 
-import static org.elasticsearch.xpack.spatial.index.mapper.ShapeIndexer.toLucenePolygon;
 
 public class ShapeQueryProcessor implements AbstractSearchableGeometryFieldType.QueryProcessor {
 
     @Override
     public Query process(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) {
         validateIsShapeFieldType(fieldName, context);
-        if (shape == null) {
-            return new MatchNoDocsQuery();
-        }
         // CONTAINS queries are not supported by VECTOR strategy for indices created before version 7.5.0 (Lucene 8.3.0);
         if (relation == ShapeRelation.CONTAINS && context.indexVersionCreated().before(Version.V_7_5_0)) {
             throw new QueryShardException(context,
                 ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + "].");
         }
-        // wrap geometry Query as a ConstantScoreQuery
-        return new ConstantScoreQuery(shape.visit(new ShapeVisitor(context, fieldName, relation)));
+        if (shape == null) {
+            return new MatchNoDocsQuery();
+        }
+        return getVectorQueryFromShape(shape, fieldName, relation, context);
     }
 
     private void validateIsShapeFieldType(String fieldName, QueryShardContext context) {
@@ -61,115 +58,107 @@ private void validateIsShapeFieldType(String fieldName, QueryShardContext contex
         }
     }
 
-    private class ShapeVisitor implements GeometryVisitor<Query, RuntimeException> {
-        QueryShardContext context;
-        String fieldName;
-        ShapeRelation relation;
+    private Query getVectorQueryFromShape(Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+        final LuceneGeometryCollector visitor = new LuceneGeometryCollector(fieldName, context);
+        queryShape.visit(visitor);
+        final List<XYGeometry> geometries = visitor.geometries();
+        if (geometries.size() == 0) {
+            return new MatchNoDocsQuery();
+        }
+        return XYShape.newGeometryQuery(fieldName, relation.getLuceneRelation(),
+            geometries.toArray(new XYGeometry[geometries.size()]));
+    }
+
+    private static class LuceneGeometryCollector implements GeometryVisitor<Void, RuntimeException> {
+        private final List<XYGeometry> geometries = new ArrayList<>();
+        private final String name;
+        private final QueryShardContext context;
 
-        ShapeVisitor(QueryShardContext context, String fieldName, ShapeRelation relation) {
+        private LuceneGeometryCollector(String name, QueryShardContext context) {
+            this.name = name;
             this.context = context;
-            this.fieldName = fieldName;
-            this.relation = relation;
         }
 
-        @Override
-        public Query visit(Circle circle) {
-            throw new QueryShardException(context, "Field [" + fieldName + "] found and unknown shape Circle");
+        List<XYGeometry> geometries() {
+            return geometries;
         }
 
         @Override
-        public Query visit(GeometryCollection<?> collection) {
-            BooleanQuery.Builder bqb = new BooleanQuery.Builder();
-            visit(bqb, collection);
-            return bqb.build();
-        }
-
-        private void visit(BooleanQuery.Builder bqb, GeometryCollection<?> collection) {
-            BooleanClause.Occur occur;
-            if (relation == ShapeRelation.CONTAINS || relation == ShapeRelation.DISJOINT) {
-                // all shapes must be disjoint / must be contained in relation to the indexed shape.
-                occur = BooleanClause.Occur.MUST;
-            } else {
-                // at least one shape must intersect / contain the indexed shape.
-                occur = BooleanClause.Occur.SHOULD;
+        public Void visit(Circle circle) {
+            if (circle.isEmpty() == false) {
+                geometries.add(ShapeUtils.toLuceneXYCircle(circle));
             }
+            return null;
+        }
+
+        @Override
+        public Void visit(GeometryCollection<?> collection) {
             for (Geometry shape : collection) {
-                bqb.add(shape.visit(this), occur);
+                shape.visit(this);
             }
+            return null;
         }
 
         @Override
-        public Query visit(Line line) {
-            return XYShape.newLineQuery(fieldName, relation.getLuceneRelation(),
-                new XYLine(doubleArrayToFloatArray(line.getX()), doubleArrayToFloatArray(line.getY())));
+        public Void visit(Line line) {
+            if (line.isEmpty() == false) {
+                geometries.add(ShapeUtils.toLuceneXYLine(line));
+            }
+            return null;
         }
 
         @Override
-        public Query visit(LinearRing ring) {
-            throw new QueryShardException(context, "Field [" + fieldName + "] found and unsupported shape LinearRing");
+        public Void visit(LinearRing ring) {
+            throw new QueryShardException(context, "Field [" + name + "] found and unsupported shape LinearRing");
         }
 
         @Override
-        public Query visit(MultiLine multiLine) {
-            XYLine[] lines = new XYLine[multiLine.size()];
-            for (int i=0; i<multiLine.size(); i++) {
-                lines[i] = new XYLine(doubleArrayToFloatArray(multiLine.get(i).getX()),
-                    doubleArrayToFloatArray(multiLine.get(i).getY()));
+        public Void visit(MultiLine multiLine) {
+            for (Line line : multiLine) {
+                visit(line);
             }
-            return XYShape.newLineQuery(fieldName, relation.getLuceneRelation(), lines);
+            return null;
         }
 
         @Override
-        public Query visit(MultiPoint multiPoint) {
-            float[][] points = new float[multiPoint.size()][2];
-            for (int i = 0; i < multiPoint.size(); i++) {
-                points[i] = new float[] {(float) multiPoint.get(i).getX(), (float) multiPoint.get(i).getY()};
+        public Void visit(MultiPoint multiPoint) {
+            for (Point point : multiPoint) {
+                visit(point);
             }
-            return XYShape.newPointQuery(fieldName, relation.getLuceneRelation(), points);
+            return null;
         }
 
         @Override
-        public Query visit(MultiPolygon multiPolygon) {
-            XYPolygon[] polygons = new XYPolygon[multiPolygon.size()];
-            for (int i=0; i<multiPolygon.size(); i++) {
-                polygons[i] = toLucenePolygon(multiPolygon.get(i));
+        public Void visit(MultiPolygon multiPolygon) {
+            for (Polygon polygon : multiPolygon) {
+                visit(polygon);
             }
-            return visitMultiPolygon(polygons);
-        }
-
-        private Query visitMultiPolygon(XYPolygon... polygons) {
-            return XYShape.newPolygonQuery(fieldName, relation.getLuceneRelation(), polygons);
+            return null;
         }
 
         @Override
-        public Query visit(Point point) {
-            ShapeField.QueryRelation luceneRelation = relation.getLuceneRelation();
-            if (luceneRelation == ShapeField.QueryRelation.CONTAINS) {
-                // contains and intersects are equivalent but the implementation of
-                // intersects is more efficient.
-                luceneRelation = ShapeField.QueryRelation.INTERSECTS;
+        public Void visit(Point point) {
+            if (point.isEmpty() == false) {
+                geometries.add(ShapeUtils.toLuceneXYPoint(point));
             }
-            float[][] pointArray  = new float[][] {{(float)point.getX(), (float)point.getY()}};
-            return XYShape.newPointQuery(fieldName, luceneRelation, pointArray);
-        }
+            return null;
 
-        @Override
-        public Query visit(Polygon polygon) {
-            return XYShape.newPolygonQuery(fieldName, relation.getLuceneRelation(), toLucenePolygon(polygon));
         }
 
         @Override
-        public Query visit(Rectangle r) {
-            return XYShape.newBoxQuery(fieldName, relation.getLuceneRelation(),
-                (float)r.getMinX(), (float)r.getMaxX(), (float)r.getMinY(), (float)r.getMaxY());
+        public Void visit(Polygon polygon) {
+            if (polygon.isEmpty() == false) {
+                geometries.add(ShapeUtils.toLuceneXYPolygon(polygon));
+            }
+            return null;
         }
-    }
 
-    private static float[] doubleArrayToFloatArray(double[] array) {
-        float[] result = new float[array.length];
-        for (int i = 0; i < array.length; ++i) {
-            result[i] = (float) array[i];
+        @Override
+        public Void visit(Rectangle r) {
+            if (r.isEmpty() == false) {
+                geometries.add(ShapeUtils.toLuceneXYRectangle(r));
+            }
+            return null;
         }
-        return result;
     }
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java
index 77c48e9f51fc8..98e149f1a8362 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java
@@ -10,11 +10,13 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.GeoJson;
 import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.builders.CircleBuilder;
 import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
 import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
 import org.elasticsearch.common.geo.builders.MultiPointBuilder;
 import org.elasticsearch.common.geo.builders.PointBuilder;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -336,4 +338,42 @@ public void testGeometryCollectionRelations() throws IOException {
             assertEquals(1, response.getHits().getTotalHits().value);
         }
     }
+
+    public void testDistanceQuery() throws Exception {
+        client().admin().indices().prepareCreate("test_distance").setMapping("location", "type=shape")
+            .execute().actionGet();
+        ensureGreen();
+
+        CircleBuilder circleBuilder = new CircleBuilder().center(new Coordinate(1, 0)).radius(10, DistanceUnit.METERS);
+
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("location", new PointBuilder(2, 2)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("location", new PointBuilder(3, 1)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("location", new PointBuilder(-20, -30)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+        client().index(new IndexRequest("test_distance")
+            .source(jsonBuilder().startObject().field("location", new PointBuilder(20, 30)).endObject())
+            .setRefreshPolicy(IMMEDIATE)).actionGet();
+
+        SearchResponse response = client().prepareSearch("test_distance")
+            .setQuery(new ShapeQueryBuilder("location", circleBuilder.buildGeometry()).relation(ShapeRelation.WITHIN))
+            .get();
+        assertEquals(2, response.getHits().getTotalHits().value);
+        response = client().prepareSearch("test_distance")
+            .setQuery(new ShapeQueryBuilder("location", circleBuilder.buildGeometry()).relation(ShapeRelation.INTERSECTS))
+            .get();
+        assertEquals(2, response.getHits().getTotalHits().value);
+        response = client().prepareSearch("test_distance")
+            .setQuery(new ShapeQueryBuilder("location", circleBuilder.buildGeometry()).relation(ShapeRelation.DISJOINT))
+            .get();
+        assertEquals(2, response.getHits().getTotalHits().value);
+        response = client().prepareSearch("test_distance")
+            .setQuery(new ShapeQueryBuilder("location", circleBuilder.buildGeometry()).relation(ShapeRelation.CONTAINS))
+            .get();
+        assertEquals(0, response.getHits().getTotalHits().value);
+    }
 }

From d47d74a55889e3b0fecb358d086d82e3df6a13ff Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 19 Mar 2020 08:00:06 -0600
Subject: [PATCH 138/285] Fix feature flag setting for ComponentTemplate APIs
 (#53758)

The feature flag was set for *most* of the builds, but there are a couple where it was missing.

Resolves #53708
---
 qa/smoke-test-multinode/build.gradle                      | 8 ++++++++
 .../test/cluster.component_template/10_basic.yml          | 3 ---
 x-pack/qa/core-rest-tests-with-security/build.gradle      | 8 ++++++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index 244b5d1e8af6e..1a2317fbb1667 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import org.elasticsearch.gradle.info.BuildParams
+
 apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
@@ -46,3 +48,9 @@ integTest.runner {
     ].join(',')
   }
 }
+
+testClusters.integTest {
+  if (BuildParams.isSnapshotBuild() == false) {
+    systemProperty 'es.itv2_feature_flag_registered', 'true'
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
index 177678a3da81d..51b1fa1bd6d7d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
@@ -1,8 +1,5 @@
 ---
 "Basic CRUD":
-  - skip:
-      version: "all"
-      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/53708"
 
   - do:
       cluster.put_component_template:
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 4fc4428c8ed6f..139fb7cd2cabe 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.info.BuildParams
+
 apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
@@ -37,3 +39,9 @@ testClusters.integTest {
   user username: System.getProperty('tests.rest.cluster.username', 'test_user'),
     password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
 }
+
+testClusters.integTest {
+  if (BuildParams.isSnapshotBuild() == false) {
+    systemProperty 'es.itv2_feature_flag_registered', 'true'
+  }
+}

From c1dc5238da165be524e33a20ac95b5c4bf6566f3 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 19 Mar 2020 14:13:52 +0000
Subject: [PATCH 139/285] Apply cluster states in system context (#53785)

Today cluster states are sometimes (rarely) applied in the default context
rather than system context, which means that any appliers which capture their
contexts cannot do things like remote transport actions when security is
enabled.

There are at least two ways that we end up applying the cluster state in the
default context:

1. locally applying a cluster state that indicates that the master has failed
2. the elected master times out while waiting for a response from another node

This commit ensures that cluster states are always applied in the system
context.

Mitigates #53751
---
 .../PublicationTransportHandler.java          |  2 +
 .../service/ClusterApplierService.java        |  5 +-
 .../cluster/coordination/NodeJoinTests.java   |  5 +-
 .../snapshots/SnapshotResiliencyTests.java    |  8 +--
 .../AbstractCoordinatorTestCase.java          | 30 ++++++-----
 .../MockSinglePrioritizingExecutor.java       |  5 +-
 .../ClusterApplierAssertionPlugin.java        | 50 +++++++++++++++++++
 .../service/FakeThreadPoolMasterService.java  | 22 ++++----
 .../elasticsearch/test/ESIntegTestCase.java   |  4 ++
 .../MockSinglePrioritizingExecutorTests.java  |  2 +-
 .../FakeThreadPoolMasterServiceTests.java     | 10 +++-
 11 files changed, 103 insertions(+), 40 deletions(-)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java

diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index a4b2f509b2ed8..1a6db35a8091e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -161,6 +161,7 @@ public PublicationContext newPublicationContext(ClusterChangedEvent clusterChang
             public void sendPublishRequest(DiscoveryNode destination, PublishRequest publishRequest,
                                            ActionListener<PublishWithJoinResponse> originalListener) {
                 assert publishRequest.getAcceptedState() == clusterChangedEvent.state() : "state got switched on us";
+                assert transportService.getThreadPool().getThreadContext().isSystemContext();
                 final ActionListener<PublishWithJoinResponse> responseActionListener;
                 if (destination.equals(nodes.getLocalNode())) {
                     // if publishing to self, use original request instead (see currentPublishRequestToSelf for explanation)
@@ -197,6 +198,7 @@ public void onFailure(Exception e) {
             @Override
             public void sendApplyCommit(DiscoveryNode destination, ApplyCommitRequest applyCommitRequest,
                                         ActionListener<TransportResponse.Empty> responseActionListener) {
+                assert transportService.getThreadPool().getThreadContext().isSystemContext();
                 transportService.sendRequest(destination, COMMIT_STATE_ACTION_NAME, applyCommitRequest, stateRequestOptions,
                     new TransportResponseHandler<TransportResponse.Empty>() {
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index 574919f6751ab..4912c8f17a8e7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -347,7 +348,9 @@ private void submitStateUpdateTask(final String source, final ClusterStateTaskCo
         if (!lifecycle.started()) {
             return;
         }
-        try {
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+            threadContext.markAsSystemContext();
             UpdateTask updateTask = new UpdateTask(config.priority(), source, new SafeClusterApplyListener(listener, logger), executor);
             if (config.timeout() != null) {
                 threadPoolExecutor.execute(updateTask, config.timeout(),
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index ac3075597dc57..6557d647c75fa 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -122,9 +122,10 @@ private static ClusterState initialState(DiscoveryNode localNode, long term, lon
     private void setupFakeMasterServiceAndCoordinator(long term, ClusterState initialState) {
         deterministicTaskQueue
             = new DeterministicTaskQueue(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), random());
+        final ThreadPool fakeThreadPool = deterministicTaskQueue.getThreadPool();
         FakeThreadPoolMasterService fakeMasterService = new FakeThreadPoolMasterService("test_node","test",
-                deterministicTaskQueue::scheduleNow);
-        setupMasterServiceAndCoordinator(term, initialState, fakeMasterService, deterministicTaskQueue.getThreadPool(), Randomness.get());
+            fakeThreadPool, deterministicTaskQueue::scheduleNow);
+        setupMasterServiceAndCoordinator(term, initialState, fakeMasterService, fakeThreadPool, Randomness.get());
         fakeMasterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
             coordinator.handlePublishRequest(new PublishRequest(event.state()));
             publishListener.onResponse(null);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 062e64ba55b74..2139268fe3888 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1163,15 +1163,15 @@ private final class TestClusterNode {
             TestClusterNode(DiscoveryNode node) throws IOException {
                 this.node = node;
                 final Environment environment = createEnvironment(node.getName());
-                masterService = new FakeThreadPoolMasterService(node.getName(), "test", deterministicTaskQueue::scheduleNow);
+                threadPool = deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable));
+                masterService = new FakeThreadPoolMasterService(node.getName(), "test", threadPool, deterministicTaskQueue::scheduleNow);
                 final Settings settings = environment.settings();
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-                threadPool = deterministicTaskQueue.getThreadPool();
                 clusterService = new ClusterService(settings, clusterSettings, masterService,
                     new ClusterApplierService(node.getName(), settings, clusterSettings, threadPool) {
                         @Override
                         protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-                            return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue);
+                            return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool);
                         }
 
                         @Override
@@ -1211,7 +1211,7 @@ protected NamedWriteableRegistry writeableRegistry() {
                     }
                 };
                 transportService = mockTransport.createTransportService(
-                    settings, deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable)),
+                    settings, threadPool,
                     new TransportInterceptor() {
                         @Override
                         public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
index 5239827f2b171..491bd7d858091 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -911,6 +911,7 @@ class ClusterNode {
             }
 
             private void setUp() {
+                final ThreadPool threadPool = deterministicTaskQueue.getThreadPool(this::onNode);
                 mockTransport = new DisruptableMockTransport(localNode, logger) {
                     @Override
                     protected void execute(Runnable runnable) {
@@ -928,24 +929,20 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                             .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
                     }
                 };
-
                 final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
                     nodeSettings : Settings.builder().put(nodeSettings)
                     .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
                         ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
-                transportService = mockTransport.createTransportService(
-                    settings, deterministicTaskQueue.getThreadPool(this::onNode),
-                    getTransportInterceptor(localNode, deterministicTaskQueue.getThreadPool(this::onNode)),
-                    a -> localNode, null, emptySet());
-                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
+                transportService = mockTransport.createTransportService(settings, threadPool,
+                    getTransportInterceptor(localNode, threadPool), a -> localNode, null, emptySet());
+                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test", threadPool,
                     runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
                 clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
-                    deterministicTaskQueue, this::onNode);
+                    deterministicTaskQueue, threadPool);
                 clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
                 clusterService.setNodeConnectionsService(
-                    new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
-                        transportService));
+                    new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService));
                 final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
                     Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
                 final AllocationService allocationService = ESAllocationTestCase.createAllocationService(Settings.EMPTY);
@@ -954,8 +951,8 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get(), (s, p, r) -> {},
                     getElectionStrategy());
                 masterService.setClusterStatePublisher(coordinator);
-                final GatewayService gatewayService = new GatewayService(settings, allocationService, clusterService,
-                    deterministicTaskQueue.getThreadPool(this::onNode), coordinator, null);
+                final GatewayService gatewayService
+                    = new GatewayService(settings, allocationService, clusterService, threadPool, coordinator, null);
 
                 logger.trace("starting up [{}]", localNode);
                 transportService.start();
@@ -1292,8 +1289,9 @@ static class AckedFakeThreadPoolMasterService extends FakeThreadPoolMasterServic
 
         AckCollector nextAckCollector = new AckCollector();
 
-        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
-            super(nodeName, serviceName, onTaskAvailableToRun);
+        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, ThreadPool threadPool,
+                                         Consumer<Runnable> onTaskAvailableToRun) {
+            super(nodeName, serviceName, threadPool, onTaskAvailableToRun);
         }
 
         @Override
@@ -1323,8 +1321,8 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
         private boolean applicationMayFail;
 
         DisruptableClusterApplierService(String nodeName, Settings settings, ClusterSettings clusterSettings,
-                                         DeterministicTaskQueue deterministicTaskQueue, Function<Runnable, Runnable> runnableWrapper) {
-            super(nodeName, settings, clusterSettings, deterministicTaskQueue.getThreadPool(runnableWrapper));
+                                         DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
+            super(nodeName, settings, clusterSettings, threadPool);
             this.nodeName = nodeName;
             this.deterministicTaskQueue = deterministicTaskQueue;
             addStateApplier(event -> {
@@ -1344,7 +1342,7 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
 
         @Override
         protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue);
+            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue, threadPool);
         }
 
         @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
index add5014e555fc..553188ac24539 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.TimeUnit;
 
@@ -29,7 +30,7 @@
  */
 public class MockSinglePrioritizingExecutor extends PrioritizedEsThreadPoolExecutor {
 
-    public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue) {
+    public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
         super(name, 0, 1, 0L, TimeUnit.MILLISECONDS,
             r -> new Thread() {
                 @Override
@@ -51,7 +52,7 @@ public String toString() {
                     });
                 }
             },
-            deterministicTaskQueue.getThreadPool().getThreadContext(), deterministicTaskQueue.getThreadPool().scheduler());
+            threadPool.getThreadContext(), threadPool.scheduler());
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java b/test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java
new file mode 100644
index 0000000000000..b44760ae37195
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.service;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public class ClusterApplierAssertionPlugin extends Plugin {
+    @Override
+    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry, Environment environment,
+                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
+                                               IndexNameExpressionResolver indexNameExpressionResolver) {
+        clusterService.addStateApplier(event -> {
+            assert threadPool.getThreadContext().isSystemContext();
+        });
+        clusterService.addListener(event -> {
+            assert threadPool.getThreadContext().isSystemContext();
+        });
+        return Collections.emptyList();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java b/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
index 74d9cc17a5325..100662f24a6ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
@@ -41,8 +41,6 @@
 
 import static org.apache.lucene.util.LuceneTestCase.random;
 import static org.elasticsearch.test.ESTestCase.randomInt;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class FakeThreadPoolMasterService extends MasterService {
     private static final Logger logger = LogManager.getLogger(FakeThreadPoolMasterService.class);
@@ -54,21 +52,14 @@ public class FakeThreadPoolMasterService extends MasterService {
     private boolean taskInProgress = false;
     private boolean waitForPublish = false;
 
-    public FakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
+    public FakeThreadPoolMasterService(String nodeName, String serviceName, ThreadPool threadPool,
+                                       Consumer<Runnable> onTaskAvailableToRun) {
         super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build(),
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            createMockThreadPool());
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool);
         this.name = serviceName;
         this.onTaskAvailableToRun = onTaskAvailableToRun;
     }
 
-    private static ThreadPool createMockThreadPool() {
-        final ThreadContext context = new ThreadContext(Settings.EMPTY);
-        final ThreadPool mockThreadPool = mock(ThreadPool.class);
-        when(mockThreadPool.getThreadContext()).thenReturn(context);
-        return mockThreadPool;
-    }
-
     @Override
     protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
         return new PrioritizedEsThreadPoolExecutor(name, 1, 1, 1, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(name),
@@ -110,7 +101,11 @@ public void run() {
                     final Runnable task = pendingTasks.remove(taskIndex);
                     taskInProgress = true;
                     scheduledNextTask = false;
-                    task.run();
+                    final ThreadContext threadContext = threadPool.getThreadContext();
+                    try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                        threadContext.markAsSystemContext();
+                        task.run();
+                    }
                     if (waitForPublish == false) {
                         taskInProgress = false;
                     }
@@ -168,4 +163,5 @@ public void onFailure(Exception e) {
     protected AckListener wrapAckListener(AckListener ackListener) {
         return ackListener;
     }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 3116792c502d6..4a482d6d412cf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -72,6 +72,7 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterApplierAssertionPlugin;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
@@ -1784,6 +1785,9 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
             if (randomBoolean()) {
                 mocks.add(MockFieldFilterPlugin.class);
             }
+            if (randomBoolean()) {
+                mocks.add(ClusterApplierAssertionPlugin.class);
+            }
         }
 
         if (addMockTransportService()) {
diff --git a/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java b/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
index 427f0d32c1270..66424de7f75d6 100644
--- a/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
@@ -29,7 +29,7 @@ public class MockSinglePrioritizingExecutorTests extends ESTestCase {
 
     public void testPrioritizedEsThreadPoolExecutor() {
         final DeterministicTaskQueue taskQueue = DeterministicTaskQueueTests.newTaskQueue();
-        final PrioritizedEsThreadPoolExecutor executor = new MockSinglePrioritizingExecutor("test", taskQueue);
+        final PrioritizedEsThreadPoolExecutor executor = new MockSinglePrioritizingExecutor("test", taskQueue, taskQueue.getThreadPool());
         final AtomicBoolean called1 = new AtomicBoolean();
         final AtomicBoolean called2 = new AtomicBoolean();
         executor.execute(new PrioritizedRunnable(Priority.NORMAL) {
diff --git a/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java b/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
index f97477707e461..c3e162254421b 100644
--- a/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
@@ -27,7 +27,10 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -37,6 +40,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class FakeThreadPoolMasterServiceTests extends ESTestCase {
 
@@ -48,7 +53,10 @@ public void testFakeMasterService() {
         lastClusterStateRef.set(ClusterStateCreationUtils.state(discoveryNode, discoveryNode));
         long firstClusterStateVersion = lastClusterStateRef.get().version();
         AtomicReference<ActionListener<Void>> publishingCallback = new AtomicReference<>();
-        FakeThreadPoolMasterService masterService = new FakeThreadPoolMasterService("test_node","test", runnableTasks::add);
+        final ThreadContext context = new ThreadContext(Settings.EMPTY);
+        final ThreadPool mockThreadPool = mock(ThreadPool.class);
+        when(mockThreadPool.getThreadContext()).thenReturn(context);
+        FakeThreadPoolMasterService masterService = new FakeThreadPoolMasterService("test_node","test", mockThreadPool, runnableTasks::add);
         masterService.setClusterStateSupplier(lastClusterStateRef::get);
         masterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
             lastClusterStateRef.set(event.state());

From 57321124eab95a9587c38a8fcb7ea5a8d2334131 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Thu, 19 Mar 2020 15:40:53 +0100
Subject: [PATCH 140/285] [DOCS] Changes seconds to milliseconds since the
 Epoch in AD docs. (#53797)

---
 docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc     | 2 +-
 .../reference/ml/anomaly-detection/apis/start-datafeed.asciidoc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc
index 510e5b9a7b863..9bac3347dccaa 100644
--- a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc
@@ -98,7 +98,7 @@ on January 1, 2018:
 --------------------------------------------------
 POST _ml/anomaly_detectors/total-requests/_flush
 {
-  "advance_time": "1514804400"
+  "advance_time": "1514804400000"
 }
 --------------------------------------------------
 // TEST[skip:setup:server_metrics_openjob]
diff --git a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
index 2dffa5e62481e..66a946e27fc3e 100644
--- a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
@@ -49,7 +49,7 @@ following formats: +
 
 - ISO 8601 format with milliseconds, for example `2017-01-22T06:00:00.000Z`
 - ISO 8601 format without milliseconds, for example `2017-01-22T06:00:00+00:00`
-- Seconds from the Epoch, for example `1390370400`
+- Milliseconds from the Epoch, for example `1485061200000`
 
 Date-time arguments using either of the ISO 8601 formats must have a time zone
 designator, where Z is accepted as an abbreviation for UTC time.

From f58680bad33d5ce4139157a69a4d9f5f286bc3c4 Mon Sep 17 00:00:00 2001
From: Costin Leau <costin@users.noreply.github.com>
Date: Thu, 19 Mar 2020 16:58:54 +0200
Subject: [PATCH 141/285] EQL: Add Substring function with Python semantics
 (#53688)

Does not reuse substring from SQL due to the difference in semantics and
the accepted arguments.
Currently it is missing full integration tests as, due to the usage of
scripting, requires an actual integration test against a proper cluster (and
likely its own QA project).
---
 x-pack/plugin/eql/build.gradle                |   6 -
 .../xpack/eql/analysis/Analyzer.java          |  43 ++++--
 .../xpack/eql/execution/PlanExecutor.java     |   3 +-
 .../function/EqlFunctionRegistry.java         |  20 +++
 .../function/scalar/string/StringUtils.java   |  50 +++++++
 .../function/scalar/string/Substring.java     | 129 ++++++++++++++++++
 .../scalar/string/SubstringFunctionPipe.java  | 111 +++++++++++++++
 .../string/SubstringFunctionProcessor.java    | 108 +++++++++++++++
 .../whitelist/InternalEqlScriptUtils.java     |  24 ++++
 .../eql/plugin/EqlPainlessExtension.java      |  41 ++++++
 ...asticsearch.painless.spi.PainlessExtension |   1 +
 .../xpack/eql/plugin/eql_whitelist.txt        |  53 +++++++
 .../eql/action/AbstractEqlIntegTestCase.java  |   4 +-
 .../xpack/eql/action/EqlActionIT.java         |   3 +-
 .../xpack/eql/analysis/VerifierTests.java     |   4 +-
 .../scalar/string/StringUtilsTests.java       |  75 ++++++++++
 .../xpack/eql/planner/QueryFolderOkTests.java |  12 ++
 .../expression/function/FunctionRegistry.java |   8 +-
 .../whitelist/InternalQlScriptUtils.java      |  26 +++-
 .../ql/expression/gen/script/Scripts.java     |  10 +-
 .../scalar/string/StringFunctionUtils.java    |  14 +-
 .../whitelist/InternalSqlScriptUtils.java     |  28 ----
 22 files changed, 704 insertions(+), 69 deletions(-)
 create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java
 create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java
 create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java
 create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java
 create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java
 create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java
 create mode 100644 x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
 create mode 100644 x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
 create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java

diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle
index 2d720f170d83c..97713aa8f7ee2 100644
--- a/x-pack/plugin/eql/build.gradle
+++ b/x-pack/plugin/eql/build.gradle
@@ -46,12 +46,6 @@ dependencies {
 
   // TOML parser for EqlActionIT tests
   testCompile 'io.ous:jtoml:2.0.0'
-
-  // JSON parser for tests input data
-  testCompile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  testCompile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  testCompile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-
 }
 
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java
index 82ce55ac74f3d..aecd531984606 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java
@@ -10,9 +10,11 @@
 import org.elasticsearch.xpack.ql.expression.Attribute;
 import org.elasticsearch.xpack.ql.expression.NamedExpression;
 import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
+import org.elasticsearch.xpack.ql.expression.function.Function;
+import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.ql.rule.Rule;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
 
 import java.util.ArrayList;
@@ -35,7 +37,8 @@ public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) {
     @Override
     protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
         Batch resolution = new Batch("Resolution",
-                new ResolveRefs());
+                new ResolveRefs(),
+                new ResolveFunctions());
         
         return asList(resolution);
     }
@@ -52,7 +55,7 @@ private LogicalPlan verify(LogicalPlan plan) {
         return plan;
     }
 
-    private static class ResolveRefs extends AnalyzeRule<LogicalPlan> {
+    private static class ResolveRefs extends AnalyzerRule<LogicalPlan> {
 
         @Override
         protected LogicalPlan rule(LogicalPlan plan) {
@@ -87,20 +90,34 @@ protected LogicalPlan rule(LogicalPlan plan) {
         }
     }
 
-    abstract static class AnalyzeRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
+    private class ResolveFunctions extends AnalyzerRule<LogicalPlan> {
 
-        // transformUp (post-order) - that is first children and then the node
-        // but with a twist; only if the tree is not resolved or analyzed
         @Override
-        public final LogicalPlan apply(LogicalPlan plan) {
-            return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
-        }
+        protected LogicalPlan rule(LogicalPlan plan) {
+            return plan.transformExpressionsUp(e -> {
+                if (e instanceof UnresolvedFunction) {
+                    UnresolvedFunction uf = (UnresolvedFunction) e;
 
-        @Override
-        protected abstract LogicalPlan rule(SubPlan plan);
+                    if (uf.analyzed()) {
+                        return uf;
+                    }
+
+                    String name = uf.name();
 
-        protected boolean skipResolved() {
-            return true;
+                    if (uf.childrenResolved() == false) {
+                        return uf;
+                    }
+
+                    String functionName = functionRegistry.resolveAlias(name);
+                    if (functionRegistry.functionExists(functionName) == false) {
+                        return uf.missing(functionName, functionRegistry.listFunctions());
+                    }
+                    FunctionDefinition def = functionRegistry.resolveFunction(functionName);
+                    Function f = uf.buildResolved(null, def);
+                    return f;
+                }
+                return e;
+            });
         }
     }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java
index 650d5b750a8a0..cc6ba5d018f6a 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.eql.analysis.Analyzer;
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.Verifier;
+import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
 import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.planner.Planner;
@@ -44,7 +45,7 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe
         this.writableRegistry = writeableRegistry;
 
         this.indexResolver = indexResolver;
-        this.functionRegistry = null;
+        this.functionRegistry = new EqlFunctionRegistry();
 
         this.metrics = new Metrics();
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java
index a219d4482d0a0..119e12fa9f39c 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java
@@ -6,10 +6,30 @@
 
 package org.elasticsearch.xpack.eql.expression.function;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
+import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 
+import java.util.Locale;
+
 public class EqlFunctionRegistry extends FunctionRegistry {
 
     public EqlFunctionRegistry() {
+        super(functions());
+    }
+    
+    private static FunctionDefinition[][] functions() {
+        return new FunctionDefinition[][] {
+        // Scalar functions
+        // String
+            new FunctionDefinition[] {
+                def(Substring.class, Substring::new, "substring"),
+            },
+        };
+    }
+
+    @Override
+    protected String normalize(String name) {
+        return name.toLowerCase(Locale.ROOT);
     }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java
new file mode 100644
index 0000000000000..11657872c7f69
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.Strings;
+
+import static org.elasticsearch.common.Strings.hasLength;
+
+final class StringUtils {
+
+    private StringUtils() {}
+
+    /**
+     * Returns a substring using the Python slice semantics, meaning
+     * start and end can be negative
+     */
+    static String substringSlice(String string, int start, int end) {
+        if (hasLength(string) == false) {
+            return string;
+        }
+
+        int length = string.length();
+        
+        // handle first negative values
+        if (start < 0) {
+            start += length;
+        }
+        if (start < 0) {
+            start = 0;
+        }
+        if (end < 0) {
+            end += length;
+        }
+        if (end < 0) {
+            end = 0;
+        } else if (end > length) {
+            end = length;
+        }
+
+        if (start >= end) {
+            return org.elasticsearch.xpack.ql.util.StringUtils.EMPTY;
+        }
+
+        return Strings.substring(string, start, end);
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java
new file mode 100644
index 0000000000000..5828a74532522
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.function.OptionalArgument;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isInteger;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * EQL specific substring function - similar to the one in Python.
+ * Note this is different than the one in SQL.
+ */
+public class Substring extends ScalarFunction implements OptionalArgument {
+
+    private final Expression source, start, end;
+
+    public Substring(Source source, Expression src, Expression start, Expression end) {
+        super(source, Arrays.asList(src, start, end != null ? end : new Literal(source, null, DataTypes.NULL)));
+        this.source = src;
+        this.start = start;
+        this.end = arguments().get(2);
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = isStringAndExact(source, sourceText(), ParamOrdinal.FIRST);
+        if (sourceResolution.unresolved()) {
+            return sourceResolution;
+        }
+
+        TypeResolution startResolution = isInteger(start, sourceText(), ParamOrdinal.SECOND);
+        if (startResolution.unresolved()) {
+            return startResolution;
+        }
+
+        return isInteger(end, sourceText(), ParamOrdinal.THIRD);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new SubstringFunctionPipe(source(), this, Expressions.pipe(source), Expressions.pipe(start), Expressions.pipe(end));
+    }
+
+    @Override
+    public boolean foldable() {
+        return source.foldable() && start.foldable() && end.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold(), start.fold(), end.fold());
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Substring::new, source, start, end);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+        ScriptTemplate startScript = asScript(start);
+        ScriptTemplate endScript = asScript(end);
+
+        return asScriptFrom(sourceScript, startScript, endScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate startScript, ScriptTemplate endScript) {
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s,%s)"),
+                "substring",
+                sourceScript.template(),
+                startScript.template(),
+                endScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params())
+                    .script(startScript.params())
+                    .script(endScript.params())
+                    .build(), dataType());
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(processScript("doc[{}].value"),
+                paramsBuilder().variable(field.exactAttribute().name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Substring(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java
new file mode 100644
index 0000000000000..a6959432edb0e
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class SubstringFunctionPipe extends Pipe {
+
+    private final Pipe source, start, end;
+
+    public SubstringFunctionPipe(Source source, Expression expression, Pipe src, Pipe start, Pipe end) {
+        super(source, expression, Arrays.asList(src, start, end));
+        this.source = src;
+        this.start = start;
+        this.end = end;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        if (newChildren.size() != 3) {
+            throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newSource = source.resolveAttributes(resolver);
+        Pipe newStart = start.resolveAttributes(resolver);
+        Pipe newEnd = end.resolveAttributes(resolver);
+        if (newSource == source && newStart == start && newEnd == end) {
+            return this;
+        }
+        return replaceChildren(newSource, newStart, newEnd);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery() && start.supportedByAggsOnlyQuery() && end.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved() && start.resolved() && end.resolved();
+    }
+
+    protected Pipe replaceChildren(Pipe newSource, Pipe newStart, Pipe newEnd) {
+        return new SubstringFunctionPipe(source(), expression(), newSource, newStart, newEnd);
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+        start.collectFields(sourceBuilder);
+        end.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<SubstringFunctionPipe> info() {
+        return NodeInfo.create(this, SubstringFunctionPipe::new, expression(), source, start, end);
+    }
+
+    @Override
+    public SubstringFunctionProcessor asProcessor() {
+        return new SubstringFunctionProcessor(source.asProcessor(), start.asProcessor(), end.asProcessor());
+    }
+    
+    public Pipe src() {
+        return source;
+    }
+    
+    public Pipe start() {
+        return start;
+    }
+    
+    public Pipe end() {
+        return end;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source, start, end);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        SubstringFunctionPipe other = (SubstringFunctionPipe) obj;
+        return Objects.equals(source, other.source)
+                && Objects.equals(start, other.start)
+                && Objects.equals(end, other.end);
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java
new file mode 100644
index 0000000000000..c4820232e3f26
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SubstringFunctionProcessor implements Processor {
+
+    public static final String NAME = "ssub";
+
+    private final Processor source, start, end;
+
+    public SubstringFunctionProcessor(Processor source, Processor start, Processor end) {
+        this.source = source;
+        this.start = start;
+        this.end = end;
+    }
+
+    public SubstringFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        start = in.readNamedWriteable(Processor.class);
+        end = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+        out.writeNamedWriteable(start);
+        out.writeNamedWriteable(end);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input), start.process(input), end.process(input));
+    }
+
+    public static Object doProcess(Object source, Object start, Object end) {
+        if (source == null) {
+            return null;
+        }
+        if (!(source instanceof String || source instanceof Character)) {
+            throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (start == null) {
+            return source;
+        }
+        if ((start instanceof Number) == false) {
+            throw new EqlIllegalArgumentException("A number is required; received [{}]", start);
+        }
+        if (end != null && (end instanceof Number) == false) {
+            throw new EqlIllegalArgumentException("A number is required; received [{}]", end);
+        }
+
+        String str = source.toString();
+        int startIndex = ((Number) start).intValue();
+        int endIndex = end == null ? str.length() : ((Number) end).intValue();
+                
+        return StringUtils.substringSlice(str, startIndex, endIndex);
+    }
+    
+    protected Processor source() {
+        return source;
+    }
+    
+    protected Processor start() {
+        return start;
+    }
+    
+    protected Processor end() {
+        return end;
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        SubstringFunctionProcessor other = (SubstringFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+                && Objects.equals(start(), other.start())
+                && Objects.equals(end(), other.end());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), start(), end());
+    }
+    
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java
new file mode 100644
index 0000000000000..959334a73d8b0
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
+
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
+import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
+
+/*
+ * Whitelisted class for EQL scripts.
+ * Acts as a registry of the various static methods used <b>internally</b> by the scalar functions
+ * (to simplify the whitelist definition).
+ */
+public class InternalEqlScriptUtils extends InternalQlScriptUtils {
+
+    InternalEqlScriptUtils() {}
+
+    public static String substring(String s, Number start, Number end) {
+        return (String) SubstringFunctionProcessor.doProcess(s, start, end);
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java
new file mode 100644
index 0000000000000..011aefd56a855
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.plugin;
+
+import org.elasticsearch.painless.spi.PainlessExtension;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.AggregationScript;
+import org.elasticsearch.script.BucketAggregationSelectorScript;
+import org.elasticsearch.script.FieldScript;
+import org.elasticsearch.script.FilterScript;
+import org.elasticsearch.script.NumberSortScript;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.StringSortScript;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+
+public class EqlPainlessExtension implements PainlessExtension {
+
+    private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(EqlPainlessExtension.class, "eql_whitelist.txt");
+
+    @Override
+    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        Map<ScriptContext<?>, List<Whitelist>> whitelist = new HashMap<>();
+        List<Whitelist> list = singletonList(WHITELIST);
+        whitelist.put(FilterScript.CONTEXT, list);
+        whitelist.put(AggregationScript.CONTEXT, list);
+        whitelist.put(FieldScript.CONTEXT, list);
+        whitelist.put(NumberSortScript.CONTEXT, list);
+        whitelist.put(StringSortScript.CONTEXT, list);
+        whitelist.put(BucketAggregationSelectorScript.CONTEXT, list);
+        return whitelist;
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
new file mode 100644
index 0000000000000..70e279c04909a
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
@@ -0,0 +1 @@
+org.elasticsearch.xpack.eql.plugin.EqlPainlessExtension
\ No newline at end of file
diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
new file mode 100644
index 0000000000000..8fb4a57958299
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
@@ -0,0 +1,53 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# This file contains a whitelist for EQL specific utilities and classes available inside EQL scripting
+
+#### Classes
+
+class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils {
+}
+
+class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils {
+#
+# Utilities
+# 
+  def docValue(java.util.Map, String)
+  boolean nullSafeFilter(Boolean)
+  double nullSafeSortNumeric(Number)
+  String nullSafeSortString(Object)
+
+#
+# Comparison
+#
+  Boolean eq(Object, Object)
+  Boolean nulleq(Object, Object)
+  Boolean neq(Object, Object)
+  Boolean lt(Object, Object)
+  Boolean lte(Object, Object)
+  Boolean gt(Object, Object)
+  Boolean gte(Object, Object)
+  Boolean in(Object, java.util.List)
+
+#
+# Logical
+#
+  Boolean and(Boolean, Boolean)
+  Boolean or(Boolean, Boolean)
+  Boolean not(Boolean)
+  Boolean isNull(Object)
+  Boolean isNotNull(Object)
+
+#
+# Regex
+#
+  Boolean regex(String, String)
+
+#
+# ASCII Functions
+# 
+  String  substring(String, Number, Number)
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java
index a7f1cf5099766..386c0e337b310 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
 
 import java.util.Collection;
-import java.util.Collections;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 
 @ESIntegTestCase.ClusterScope(scope = SUITE, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0)
@@ -35,7 +35,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singletonList(LocalStateEqlXPackPlugin.class);
+        return singletonList(LocalStateEqlXPackPlugin.class);
     }
 }
 
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
index cf313acd1a1b6..b2e5c2c89abe1 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.eql.action;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.fasterxml.jackson.databind.ObjectMapper;
+
 import org.elasticsearch.Build;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -43,7 +43,6 @@ public static void checkForSnapshot() {
     @SuppressWarnings("unchecked")
     public void setUpData() throws Exception {
         // Insert test data
-        ObjectMapper mapper = new ObjectMapper();
         BulkRequestBuilder bulkBuilder = client().prepareBulk();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, EqlActionIT.class.getResourceAsStream("/test_data.json"))) {
             List<Object> list = parser.list();
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
index e6d398b2ea30f..1570184ced1e0 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
@@ -131,8 +131,6 @@ public void testFunctionParsingUnknown() {
 
     // Test the known EQL functions that are not supported
     public void testFunctionVerificationUnknown() {
-        assertEquals("1:26: Unknown function [substring]",
-                error("foo where user_domain == substring('abcdfeg', 0, 5)"));
         assertEquals("1:25: Unknown function [endsWith]",
                 error("file where opcode=0 and endsWith(file_name, 'loREr.exe')"));
         assertEquals("1:25: Unknown function [startsWith]",
@@ -143,7 +141,7 @@ public void testFunctionVerificationUnknown() {
                 error("file where opcode=0 and indexOf(file_name, 'plore') == 2"));
         assertEquals("1:15: Unknown function [add]",
                 error("process where add(serial_event_id, 0) == 1"));
-        assertEquals("1:15: Unknown function [subtract]",
+        assertEquals("1:15: Unknown function [subtract], did you mean [substring]?",
                 error("process where subtract(serial_event_id, -5) == 6"));
         assertEquals("1:15: Unknown function [multiply]",
                 error("process where multiply(6, serial_event_id) == 30"));
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java
new file mode 100644
index 0000000000000..0abf9c5eb755d
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.StringUtils.substringSlice;
+
+public class StringUtilsTests extends ESTestCase {
+
+    public void testSubstringSlicePositive() {
+        String str = randomAlphaOfLength(10);
+        assertEquals(str.substring(1, 7), substringSlice(str, 1, 7));
+    }
+
+    public void testSubstringSliceNegative() {
+        String str = randomAlphaOfLength(10);
+        assertEquals(str.substring(5, 9), substringSlice(str, -5, -1));
+    }
+
+    public void testSubstringSliceNegativeOverLength() {
+        String str = randomAlphaOfLength(10);
+        assertEquals("", substringSlice(str, -15, -11));
+    }
+
+    public void testSubstringSlicePositiveOverLength() {
+        String str = randomAlphaOfLength(10);
+        assertEquals("", substringSlice(str, 11, 14));
+    }
+
+    public void testSubstringHigherEndThanStartNegative() {
+        String str = randomAlphaOfLength(10);
+        assertEquals("", substringSlice(str, -20, -11));
+    }
+
+    public void testSubstringRandomSlicePositive() {
+        String str = randomAlphaOfLength(10);
+        int start = randomInt(5);
+        int end = start + randomInt(3);
+        assertEquals(str.substring(start, end), substringSlice(str, start, end));
+    }
+
+    public void testSubstringRandomSliceNegative() {
+        String str = randomAlphaOfLength(10);
+        int end = 1 + randomInt(3);
+        int start = end + randomInt(5);
+        assertEquals(str.substring(10 - start, 10 - end), substringSlice(str, -start, -end));
+    }
+
+    public void testStartNegativeHigherThanLength() {
+        String str = randomAlphaOfLength(10);
+        int start = 10 + randomInt(10);
+        assertEquals(str.substring(0, 10 - 1), substringSlice(str, -start, -1));
+    }
+
+    public void testEndHigherThanLength() {
+        String str = randomAlphaOfLength(10);
+        int end = 10 + randomInt(10);
+        assertEquals(str, substringSlice(str, 0, end));
+    }
+
+    public void testSubstringRandomSliceSameStartEnd() {
+        String str = randomAlphaOfLength(10);
+        int start = randomInt();
+        assertEquals("", substringSlice(str, start, start));
+    }
+
+    public void testNullValue() {
+        assertNull(substringSlice(null, 0, 0));
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
index acbe328a4740a..b774ff916414c 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.eql.planner;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
 
@@ -59,6 +60,17 @@ public class QueryFolderOkTests extends AbstractQueryFolderTestCase {
                             "\"term\":{\"opcode\":{\"value\":3",
                     }
             },
+            {"substringFunction", "process where substring(file_name, -4) == '.exe'",
+                    new Object[]{
+                            "{\"script\":{\"source\":\""
+                            + "InternalSqlScriptUtils.nullSafeFilter("
+                            + "InternalSqlScriptUtils.eq("
+                            + "InternalSqlScriptUtils.substring("
+                            + "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))",
+                            "\"params\":{\"v0\":\"file_name.keyword\",\"v1\":-4,\"v2\":null,\"v3\":\".exe\"}"
+                            
+                    }
+            }
     };
 
     private final String name;
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java
index 04ecf703908d9..6b90a59289f07 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java
@@ -86,8 +86,12 @@ public FunctionDefinition resolveFunction(String functionName) {
         return def;
     }
 
+    protected String normalize(String name) {
+        return name.toUpperCase(Locale.ROOT);
+    }
+
     public String resolveAlias(String alias) {
-        String upperCase = alias.toUpperCase(Locale.ROOT);
+        String upperCase = normalize(alias);
         return aliases.getOrDefault(upperCase, upperCase);
     }
 
@@ -102,7 +106,7 @@ public Collection<FunctionDefinition> listFunctions() {
 
     public Collection<FunctionDefinition> listFunctions(String pattern) {
         // It is worth double checking if we need this copy. These are immutable anyway.
-        Pattern p = Strings.hasText(pattern) ? Pattern.compile(pattern.toUpperCase(Locale.ROOT)) : null;
+        Pattern p = Strings.hasText(pattern) ? Pattern.compile(normalize(pattern)) : null;
         return defs.entrySet().stream()
                 .filter(e -> p == null || p.matcher(e.getKey()).matches())
                 .map(e -> new FunctionDefinition(e.getKey(), emptyList(),
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
index 26c4bea1cf592..1f63bd7ac34c4 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
@@ -9,12 +9,16 @@
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
+import java.util.List;
 import java.util.Map;
 
-public abstract class InternalQlScriptUtils {
+public class InternalQlScriptUtils {
 
     //
     // Utilities
@@ -79,6 +83,10 @@ public static Boolean gte(Object left, Object right) {
         return BinaryComparisonOperation.GTE.apply(left, right);
     }
 
+    public static Boolean in(Object value, List<Object> values) {
+        return InProcessor.apply(value, values);
+    }
+
     public static Boolean and(Boolean left, Boolean right) {
         return BinaryLogicOperation.AND.apply(left, right);
     }
@@ -90,4 +98,20 @@ public static Boolean or(Boolean left, Boolean right) {
     public static Boolean not(Boolean expression) {
         return NotProcessor.apply(expression);
     }
+
+    public static Boolean isNull(Object expression) {
+        return CheckNullOperation.IS_NULL.apply(expression);
+    }
+
+    public static Boolean isNotNull(Object expression) {
+        return CheckNullOperation.IS_NOT_NULL.apply(expression);
+    }
+
+    //
+    // Regex
+    //
+    public static Boolean regex(String value, String pattern) {
+        // TODO: this needs to be improved to avoid creating the pattern on every call
+        return RegexOperation.match(value, pattern);
+    }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java
index a743500337f45..337957089e6dd 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java
@@ -25,6 +25,8 @@
 public final class Scripts {
 
     public static final String DOC_VALUE = "doc[{}].value";
+    public static final String QL_SCRIPTS = "{ql}";
+    public static final String EQL_SCRIPTS = "{eql}";
     public static final String SQL_SCRIPTS = "{sql}";
     public static final String PARAM = "{}";
     // FIXME: this needs to be either renamed (drop Sql) or find a pluggable approach (through ScriptWeaver)
@@ -33,7 +35,9 @@ public final class Scripts {
     private Scripts() {}
 
     static final Map<Pattern, String> FORMATTING_PATTERNS = unmodifiableMap(Stream.of(
-            new SimpleEntry<>(DOC_VALUE, SQL_SCRIPTS + ".docValue(doc,{})"),
+            new SimpleEntry<>(DOC_VALUE, QL_SCRIPTS + ".docValue(doc,{})"),
+            new SimpleEntry<>(QL_SCRIPTS, INTERNAL_SCRIPT_UTILS),
+            new SimpleEntry<>(EQL_SCRIPTS, INTERNAL_SCRIPT_UTILS),
             new SimpleEntry<>(SQL_SCRIPTS, INTERNAL_SCRIPT_UTILS),
             new SimpleEntry<>(PARAM, "params.%s"))
             .collect(toMap(e -> Pattern.compile(e.getKey(), Pattern.LITERAL), Map.Entry::getValue, (a, b) -> a, LinkedHashMap::new)));
@@ -56,7 +60,7 @@ public static String formatTemplate(String template) {
 
     public static ScriptTemplate nullSafeFilter(ScriptTemplate script) {
         return new ScriptTemplate(formatTemplate(
-                format(Locale.ROOT, "{sql}.nullSafeFilter(%s)", script.template())),
+                format(Locale.ROOT, "{ql}.nullSafeFilter(%s)", script.template())),
                 script.params(),
                 DataTypes.BOOLEAN);
     }
@@ -64,7 +68,7 @@ public static ScriptTemplate nullSafeFilter(ScriptTemplate script) {
     public static ScriptTemplate nullSafeSort(ScriptTemplate script) {
         String methodName = script.outputType().isNumeric() ? "nullSafeSortNumeric" : "nullSafeSortString";
         return new ScriptTemplate(formatTemplate(
-                format(Locale.ROOT, "{sql}.%s(%s)", methodName, script.template())),
+                format(Locale.ROOT, "{ql}.%s(%s)", methodName, script.template())),
                 script.params(),
                 script.outputType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
index 7fc55770bd3f8..d09e3ee317c2d 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.sql.expression.function.scalar.string;
 
+import static org.elasticsearch.common.Strings.hasLength;
+
 abstract class StringFunctionUtils {
 
     /**
@@ -20,11 +22,13 @@ static String substring(String s, int start, int length) {
             return s;
         }
         
-        if (start < 0)
+        if (start < 0) {
             start = 0;
+        }
         
-        if (start + 1 > s.length() || length < 0)
+        if (start + 1 > s.length() || length < 0) {
             return "";
+        }
         
         return (start + length > s.length()) ? s.substring(start) : s.substring(start, start + length);
     }
@@ -66,10 +70,4 @@ static String trimLeadingWhitespaces(String s) {
         }
         return sb.toString();
     }
-
-    private static boolean hasLength(String s) {
-        return (s != null && s.length() > 0);
-    }
-    
-
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
index 82a41fb2024da..d303c9fa03c01 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
@@ -9,10 +9,7 @@
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.script.JodaCompatibleZonedDateTime;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
-import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation;
-import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateAddProcessor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateDiffProcessor;
@@ -66,23 +63,6 @@ public class InternalSqlScriptUtils extends InternalQlScriptUtils {
 
     InternalSqlScriptUtils() {}
 
-
-    //
-    // Logical
-    //
-
-    public static Boolean isNull(Object expression) {
-        return CheckNullOperation.IS_NULL.apply(expression);
-    }
-
-    public static Boolean isNotNull(Object expression) {
-        return CheckNullOperation.IS_NOT_NULL.apply(expression);
-    }
-
-    public static Boolean in(Object value, List<Object> values) {
-        return InProcessor.apply(value, values);
-    }
-
     //
     // Conditional
     //
@@ -106,14 +86,6 @@ public static Object nullif(Object left, Object right) {
         return NullIfProcessor.apply(left, right);
     }
 
-    //
-    // Regex
-    //
-    public static Boolean regex(String value, String pattern) {
-        // TODO: this needs to be improved to avoid creating the pattern on every call
-        return RegexOperation.match(value, pattern);
-    }
-
     //
     // Math
     //

From 783df3f9610a1f29149a10c1024758db566c74e3 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Thu, 19 Mar 2020 11:15:55 -0400
Subject: [PATCH 142/285] [ML] only retry persistence failures when the failure
 is intermittent and stop retrying when analytics job is stopping (#53725)

This fixes two issues:


- Results persister would retry actions even if they are not intermittent. An example of an persistent failure is a doc mapping problem.
- Data frame analytics would continue to retry to persist results even after the job is stopped.

closes https://github.com/elastic/elasticsearch/issues/53687
---
 .../process/AnalyticsResultProcessor.java     |  5 +-
 .../process/DataFrameRowsJoiner.java          | 14 ++++-
 .../persistence/ResultsPersisterService.java  | 53 +++++++++++++++++-
 .../ResultsPersisterServiceTests.java         | 55 ++++++++++++++++++-
 4 files changed, 118 insertions(+), 9 deletions(-)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
index 34be13967b5d2..d5873eb29aa23 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -108,6 +108,7 @@ public void awaitForCompletion() {
     }
 
     public void cancel() {
+        dataFrameRowsJoiner.cancel();
         isCancelled = true;
     }
 
@@ -265,12 +266,12 @@ private void indexStatsResult(ToXContentObject result, Function<String, String>
                 new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
                 WriteRequest.RefreshPolicy.IMMEDIATE,
                 docIdSupplier.apply(analytics.getId()),
-                () -> true,
+                () -> isCancelled == false,
                 errorMsg -> auditor.error(analytics.getId(),
                     "failed to persist result with id [" + docIdSupplier.apply(analytics.getId()) + "]; " + errorMsg)
             );
         } catch (IOException ioe) {
-            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed indexing stats result", analytics.getId()), ioe);
+            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed serializing stats result", analytics.getId()), ioe);
         } catch (Exception e) {
             LOGGER.error(() -> new ParameterizedMessage("[{}] Failed indexing stats result", analytics.getId()), e);
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java
index 93c28c8a8d32b..0296b16e488d2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java
@@ -40,9 +40,9 @@ class DataFrameRowsJoiner implements AutoCloseable {
     private final Iterator<DataFrameDataExtractor.Row> dataFrameRowsIterator;
     private LinkedList<RowResults> currentResults;
     private volatile String failure;
+    private volatile boolean isCancelled;
 
-    DataFrameRowsJoiner(String analyticsId, DataFrameDataExtractor dataExtractor,
-                        ResultsPersisterService resultsPersisterService) {
+    DataFrameRowsJoiner(String analyticsId, DataFrameDataExtractor dataExtractor, ResultsPersisterService resultsPersisterService) {
         this.analyticsId = Objects.requireNonNull(analyticsId);
         this.dataExtractor = Objects.requireNonNull(dataExtractor);
         this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService);
@@ -70,6 +70,10 @@ void processRowResults(RowResults rowResults) {
         }
     }
 
+    void cancel() {
+        isCancelled = true;
+    }
+
     private void addResultAndJoinIfEndOfBatch(RowResults rowResults) {
         currentResults.add(rowResults);
         if (currentResults.size() == RESULTS_BATCH_SIZE) {
@@ -87,7 +91,11 @@ private void joinCurrentResults() {
         }
         if (bulkRequest.numberOfActions() > 0) {
             resultsPersisterService.bulkIndexWithHeadersWithRetry(
-                dataExtractor.getHeaders(), bulkRequest, analyticsId, () -> true, errorMsg -> {});
+                dataExtractor.getHeaders(),
+                bulkRequest,
+                analyticsId,
+                () -> isCancelled == false,
+                errorMsg -> {});
         }
         currentResults = new LinkedList<>();
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
index 5ae76f9491861..3adab97b7f5f8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
@@ -9,6 +9,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -33,6 +34,8 @@
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -42,6 +45,22 @@
 import java.util.stream.Collectors;
 
 public class ResultsPersisterService {
+    /**
+     * List of rest statuses that we consider irrecoverable
+     */
+    public static final Set<RestStatus> IRRECOVERABLE_REST_STATUSES = Collections.unmodifiableSet(new HashSet<>(
+        Arrays.asList(
+            RestStatus.GONE,
+            RestStatus.NOT_IMPLEMENTED,
+            RestStatus.NOT_FOUND,
+            RestStatus.BAD_REQUEST,
+            RestStatus.UNAUTHORIZED,
+            RestStatus.FORBIDDEN,
+            RestStatus.METHOD_NOT_ALLOWED,
+            RestStatus.NOT_ACCEPTABLE
+        )
+    ));
+
     private static final Logger LOGGER = LogManager.getLogger(ResultsPersisterService.class);
 
     public static final Setting<Integer> PERSIST_RESULTS_MAX_RETRIES = Setting.intSetting(
@@ -124,9 +143,23 @@ private BulkResponse bulkIndexWithRetry(BulkRequest bulkRequest,
             if (bulkResponse.hasFailures() == false) {
                 return bulkResponse;
             }
-
+            for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
+                if (itemResponse.isFailed()) {
+                    if (isIrrecoverable(itemResponse.getFailure().getCause())) {
+                        Throwable unwrappedParticular = ExceptionsHelper.unwrapCause(itemResponse.getFailure().getCause());
+                        LOGGER.warn(new ParameterizedMessage(
+                            "[{}] experienced failure that cannot be automatically retried. Bulk failure message [{}]",
+                                jobId,
+                                bulkResponse.buildFailureMessage()),
+                            unwrappedParticular);
+                        throw new ElasticsearchException(
+                            "{} experienced failure that cannot be automatically retried. See logs for bulk failures",
+                            unwrappedParticular,
+                            jobId);
+                    }
+                }
+            }
             retryContext.nextIteration("index", bulkResponse.buildFailureMessage());
-
             // We should only retry the docs that failed.
             bulkRequest = buildNewRequestFromFailures(bulkRequest, bulkResponse);
         }
@@ -148,12 +181,28 @@ public SearchResponse searchWithRetry(SearchRequest searchRequest,
             } catch (ElasticsearchException e) {
                 LOGGER.warn("[" + jobId + "] Exception while executing search action", e);
                 failureMessage = e.getDetailedMessage();
+                if (isIrrecoverable(e)) {
+                    LOGGER.warn(new ParameterizedMessage("[{}] experienced failure that cannot be automatically retried", jobId), e);
+                    throw new ElasticsearchException("{} experienced failure that cannot be automatically retried", e, jobId);
+                }
             }
 
             retryContext.nextIteration("search", failureMessage);
         }
     }
 
+    /**
+     * @param ex The exception to check
+     * @return true when the failure will persist no matter how many times we retry.
+     */
+    private static boolean isIrrecoverable(Exception ex) {
+        Throwable t = ExceptionsHelper.unwrapCause(ex);
+        if (t instanceof ElasticsearchException) {
+            return IRRECOVERABLE_REST_STATUSES.contains(((ElasticsearchException) t).status());
+        }
+        return false;
+    }
+
     /**
      * {@link RetryContext} object handles logic that is executed between consecutive retries of an action.
      *
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
index 8e5a71f2fb2eb..af47efeef7d2b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.utils.persistence;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BulkAction;
@@ -27,8 +28,10 @@
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -131,7 +134,8 @@ public void testSearchWithRetries_SuccessAfterRetry() {
     }
 
     public void testSearchWithRetries_SuccessAfterRetryDueToException() {
-        doThrow(new IndexNotFoundException("my-index")).doAnswer(withResponse(SEARCH_RESPONSE_SUCCESS))
+        doThrow(new IndexPrimaryShardNotAllocatedException(new Index("my-index", "UUID")))
+            .doAnswer(withResponse(SEARCH_RESPONSE_SUCCESS))
             .when(client).execute(eq(SearchAction.INSTANCE), eq(SEARCH_REQUEST), any());
 
         List<String> messages = new ArrayList<>();
@@ -206,6 +210,21 @@ public void testSearchWithRetries_Failure_ShouldNotRetryAfterRandomNumberOfRetri
         verify(client, times(maxRetries + 1)).execute(eq(SearchAction.INSTANCE), eq(SEARCH_REQUEST), any());
     }
 
+    public void testSearchWithRetries_FailureOnIrrecoverableError() {
+        resultsPersisterService.setMaxFailureRetries(5);
+
+        doAnswer(withFailure(new ElasticsearchStatusException("bad search request", RestStatus.BAD_REQUEST)))
+            .when(client).execute(eq(SearchAction.INSTANCE), eq(SEARCH_REQUEST), any());
+
+        ElasticsearchException e =
+            expectThrows(
+                ElasticsearchException.class,
+                () -> resultsPersisterService.searchWithRetry(SEARCH_REQUEST, JOB_ID, () -> true, (s) -> {}));
+        assertThat(e.getMessage(), containsString("experienced failure that cannot be automatically retried"));
+
+        verify(client).execute(eq(SearchAction.INSTANCE), eq(SEARCH_REQUEST), any());
+    }
+
     private static Supplier<Boolean> shouldRetryUntil(int maxRetries) {
         return new Supplier<>() {
             int retries = 0;
@@ -240,6 +259,29 @@ public void testBulkRequestChangeOnFailures() {
         assertThat(lastMessage.get(), containsString("failed to index after [1] attempts. Will attempt again in"));
     }
 
+    public void testBulkRequestChangeOnIrrecoverableFailures() {
+        int maxFailureRetries = 10;
+        resultsPersisterService.setMaxFailureRetries(maxFailureRetries);
+        BulkItemResponse irrecoverable = new BulkItemResponse(
+            2,
+            DocWriteRequest.OpType.INDEX,
+            new BulkItemResponse.Failure("my-index", "fail", new ElasticsearchStatusException("boom", RestStatus.BAD_REQUEST)));
+        doAnswerWithResponses(
+            new BulkResponse(new BulkItemResponse[]{irrecoverable, BULK_ITEM_RESPONSE_SUCCESS}, 0L),
+            new BulkResponse(new BulkItemResponse[0], 0L))
+            .when(client).execute(eq(BulkAction.INSTANCE), any(), any());
+
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(INDEX_REQUEST_FAILURE);
+        bulkRequest.add(INDEX_REQUEST_SUCCESS);
+
+        ElasticsearchException ex = expectThrows(ElasticsearchException.class,
+            () -> resultsPersisterService.bulkIndexWithRetry(bulkRequest, JOB_ID, () -> true, (s)->{}));
+
+        verify(client).execute(eq(BulkAction.INSTANCE), any(), any());
+        assertThat(ex.getMessage(), containsString("experienced failure that cannot be automatically retried."));
+    }
+
     public void testBulkRequestDoesNotRetryWhenSupplierIsFalse() {
         doAnswerWithResponses(
                 new BulkResponse(new BulkItemResponse[]{BULK_ITEM_RESPONSE_FAILURE, BULK_ITEM_RESPONSE_SUCCESS}, 0L),
@@ -315,6 +357,15 @@ private static <Response> Answer<Response> withResponse(Response response) {
         };
     }
 
+    @SuppressWarnings("unchecked")
+    private static <Response> Answer<Response> withFailure(Exception failure) {
+        return invocationOnMock -> {
+            ActionListener<Response> listener = (ActionListener<Response>) invocationOnMock.getArguments()[2];
+            listener.onFailure(failure);
+            return null;
+        };
+    }
+
     private static ResultsPersisterService buildResultsPersisterService(OriginSettingClient client) {
         CheckedConsumer<Integer, InterruptedException> sleeper = millis -> {};
         ThreadPool tp = mock(ThreadPool.class);

From 0edfbd71799207360858110ad8447392c520e32b Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Thu, 19 Mar 2020 09:40:29 -0700
Subject: [PATCH 143/285] [DOCS] Adds example links to transform tutorial
 (#53640)

---
 docs/reference/transform/ecommerce-tutorial.asciidoc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/reference/transform/ecommerce-tutorial.asciidoc b/docs/reference/transform/ecommerce-tutorial.asciidoc
index 0e22002a5e7a5..16b45fde1ee09 100644
--- a/docs/reference/transform/ecommerce-tutorial.asciidoc
+++ b/docs/reference/transform/ecommerce-tutorial.asciidoc
@@ -258,3 +258,7 @@ TIP: If you do not want to keep the {transform}, you can delete it in
 <<delete-transform,delete {transform} API>>. When
 you delete a {transform}, its destination index and {kib} index
 patterns remain.
+
+Now that you've created a simple {transform} for {kib} sample data, consider
+possible use cases for your own data. For more ideas, see
+<<transform-usage>> and <<transform-examples>>.
\ No newline at end of file

From f2929fe2cdc28e3b1fa16bb3d7c52cded47432be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Thu, 19 Mar 2020 17:49:19 +0100
Subject: [PATCH 144/285] [DOCS] Adds performance considerations section to
 transforms overview (#53791)

Co-Authored-By: Lisa Cawley <lcawley@elastic.co>
---
 docs/reference/transform/overview.asciidoc | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/reference/transform/overview.asciidoc b/docs/reference/transform/overview.asciidoc
index 5edc8a9650eec..18a1db0f106ff 100644
--- a/docs/reference/transform/overview.asciidoc
+++ b/docs/reference/transform/overview.asciidoc
@@ -62,3 +62,19 @@ image::images/pivot-preview.jpg["Example of a {transform} pivot in {kib}"]
 IMPORTANT: The {transform} leaves your source index intact. It
 creates a new index that is dedicated to the transformed data.
 
+
+[[transform-performance]]
+==== Performance considerations
+
+{transforms-cap} perform search aggregations on the source 
+indices then index the results into the destination index. Therefore, a 
+{transform} never takes less time than the cumulated duration of the 
+aggregation that it performs and the indexing process.
+
+For better performance, make sure that your search aggregations and queries are 
+optimized, so they don't process unnecessary data.
+
+NOTE: When you use <<search-aggregations-bucket-datehistogram-aggregation>>, the 
+queries are not considered optimal as they run through a significant amount of 
+data. For this reason, {transforms} performing date histogram aggregations take 
+longer to run.

From 8279f82dea4d80e138841de1e9f8da724af4b313 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Thu, 19 Mar 2020 17:55:26 +0100
Subject: [PATCH 145/285] [DOCS] Fixes typo in start datafeed API docs.
 (#53811)

---
 .../reference/ml/anomaly-detection/apis/start-datafeed.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
index 66a946e27fc3e..452944ef1e828 100644
--- a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
@@ -49,7 +49,7 @@ following formats: +
 
 - ISO 8601 format with milliseconds, for example `2017-01-22T06:00:00.000Z`
 - ISO 8601 format without milliseconds, for example `2017-01-22T06:00:00+00:00`
-- Milliseconds from the Epoch, for example `1485061200000`
+- Milliseconds since the epoch, for example `1485061200000`
 
 Date-time arguments using either of the ISO 8601 formats must have a time zone
 designator, where Z is accepted as an abbreviation for UTC time.

From 2ec1f6b4c4342e90a7d4e8bbf9029b51c03c7d18 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova <mayya.sharipova@elastic.co>
Date: Thu, 19 Mar 2020 13:09:50 -0400
Subject: [PATCH 146/285] Fix testIndexhasDuplicateData tests (#49786)

testIndexHasDuplicateData tests were failing ocassionally,
due to approximate calculation of BKDReader.estimatePointCount,
where if the node is Leaf, the number of points in it
was (maxPointsInLeafNode + 1) / 2.
As DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024, for small indexes
used in tests, the estimation could be really off.

This rewrites tests, to make the  max points in leaf node to
be a small value to control the tests.

Closes #49703
---
 .../search/query/QueryPhase.java              | 39 +++++-----
 .../search/query/QueryPhaseTests.java         | 76 +++++++++++++------
 2 files changed, 74 insertions(+), 41 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 46cd4c93f0d1d..104dfdbee21a6 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -579,7 +579,7 @@ private static boolean canEarlyTerminate(IndexReader reader, SortAndFormats sort
      * Returns true if more than 50% of data in the index have the same value
      * The evaluation is approximation based on finding the median value and estimating its count
      */
-    static boolean indexFieldHasDuplicateData(IndexReader reader, String field) throws IOException {
+    private static boolean indexFieldHasDuplicateData(IndexReader reader, String field) throws IOException {
         long docsNoDupl = 0; // number of docs in segments with NO duplicate data that would benefit optimization
         long docsDupl = 0; // number of docs in segments with duplicate data that would NOT benefit optimization
         for (LeafReaderContext lrc : reader.leaves()) {
@@ -590,24 +590,8 @@ static boolean indexFieldHasDuplicateData(IndexReader reader, String field) thro
                 continue;
             }
             assert(pointValues.size() == docCount); // TODO: modify the code to handle multiple values
-
             int duplDocCount = docCount/2; // expected doc count of duplicate data
-            long minValue = LongPoint.decodeDimension(pointValues.getMinPackedValue(), 0);
-            long maxValue = LongPoint.decodeDimension(pointValues.getMaxPackedValue(), 0);
-            boolean hasDuplicateData = true;
-            while ((minValue < maxValue) && hasDuplicateData) {
-                long midValue = Math.floorDiv(minValue, 2) + Math.floorDiv(maxValue, 2); // to avoid overflow first divide each value by 2
-                long countLeft = estimatePointCount(pointValues, minValue, midValue);
-                long countRight = estimatePointCount(pointValues, midValue + 1, maxValue);
-                if ((countLeft >= countRight) && (countLeft > duplDocCount) ) {
-                    maxValue = midValue;
-                } else if ((countRight > countLeft) && (countRight > duplDocCount)) {
-                    minValue = midValue + 1;
-                } else {
-                    hasDuplicateData = false;
-                }
-            }
-            if (hasDuplicateData) {
+            if (pointsHaveDuplicateData(pointValues, duplDocCount)) {
                 docsDupl += docCount;
             } else {
                 docsNoDupl += docCount;
@@ -616,6 +600,25 @@ static boolean indexFieldHasDuplicateData(IndexReader reader, String field) thro
         return (docsDupl > docsNoDupl);
     }
 
+    static boolean pointsHaveDuplicateData(PointValues pointValues, int duplDocCount) throws IOException {
+        long minValue = LongPoint.decodeDimension(pointValues.getMinPackedValue(), 0);
+        long maxValue = LongPoint.decodeDimension(pointValues.getMaxPackedValue(), 0);
+        boolean hasDuplicateData = true;
+        while ((minValue < maxValue) && hasDuplicateData) {
+            long midValue = Math.floorDiv(minValue, 2) + Math.floorDiv(maxValue, 2); // to avoid overflow first divide each value by 2
+            long countLeft = estimatePointCount(pointValues, minValue, midValue);
+            long countRight = estimatePointCount(pointValues, midValue + 1, maxValue);
+            if ((countLeft >= countRight) && (countLeft > duplDocCount) ) {
+                maxValue = midValue;
+            } else if ((countRight > countLeft) && (countRight > duplDocCount)) {
+                minValue = midValue + 1;
+            } else {
+                hasDuplicateData = false;
+            }
+        }
+        return hasDuplicateData;
+    }
+
 
     private static long estimatePointCount(PointValues pointValues, long minValue, long maxValue) {
         final byte[] minValueAsBytes = new byte[Long.BYTES];
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 023328ede093a..489285748efcf 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -69,8 +69,13 @@
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.bkd.BKDReader;
+import org.apache.lucene.util.bkd.BKDWriter;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -94,7 +99,7 @@
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.search.query.QueryPhase.indexFieldHasDuplicateData;
+import static org.elasticsearch.search.query.QueryPhase.pointsHaveDuplicateData;
 import static org.elasticsearch.search.query.TopDocsCollectorContext.hasInfMaxScore;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -701,31 +706,56 @@ public void testNumericLongOrDateSortOptimization() throws Exception {
         dir.close();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49703")
     public void testIndexHasDuplicateData() throws IOException {
-        int docsCount = 7000;
-        int duplIndex = docsCount * 7 / 10;
-        int duplIndex2 = docsCount * 3 / 10;
+        int docsCount = 5000;
+        int maxPointsInLeafNode = 40;
+        float duplicateRatio = 0.7f;
         long duplicateValue = randomLongBetween(-10000000L, 10000000L);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(null));
-        for (int docId = 0; docId < docsCount; docId++) {
-            Document doc = new Document();
-            long rndValue = randomLongBetween(-10000000L, 10000000L);
-            long value = (docId < duplIndex) ? duplicateValue : rndValue;
-            long value2 = (docId < duplIndex2) ? duplicateValue : rndValue;
-            doc.add(new LongPoint("duplicateField", value));
-            doc.add(new LongPoint("notDuplicateField", value2));
-            writer.addDocument(doc);
+
+        try (Directory dir = newDirectory()) {
+            BKDWriter w = new BKDWriter(docsCount, dir, "tmp", 1, 1, 8, maxPointsInLeafNode, 1, docsCount);
+            byte[] longBytes = new byte[8];
+            for (int docId = 0; docId < docsCount; docId++) {
+                long value = randomFloat() < duplicateRatio ? duplicateValue : randomLongBetween(-10000000L, 10000000L);
+                LongPoint.encodeDimension(value, longBytes, 0);
+                w.add(longBytes, docId);
+            }
+            long indexFP;
+            try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
+                indexFP = w.finish(out);
+            }
+            try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
+                in.seek(indexFP);
+                BKDReader r = new BKDReader(in);
+                assertTrue(pointsHaveDuplicateData(r, r.getDocCount()/2));
+            }
+        }
+    }
+
+    public void testIndexHasNoDuplicateData() throws IOException {
+        int docsCount = 5000;
+        int maxPointsInLeafNode = 40;
+        float duplicateRatio = 0.3f;
+        long duplicateValue = randomLongBetween(-10000000L, 10000000L);
+
+        try (Directory dir = newDirectory()) {
+            BKDWriter w = new BKDWriter(docsCount, dir, "tmp", 1, 1, 8, maxPointsInLeafNode, 1, docsCount);
+            byte[] longBytes = new byte[8];
+            for (int docId = 0; docId < docsCount; docId++) {
+                long value = randomFloat() < duplicateRatio ? duplicateValue : randomLongBetween(-10000000L, 10000000L);
+                LongPoint.encodeDimension(value, longBytes, 0);
+                w.add(longBytes, docId);
+            }
+            long indexFP;
+            try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
+                indexFP = w.finish(out);
+            }
+            try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
+                in.seek(indexFP);
+                BKDReader r = new BKDReader(in);
+                assertFalse(pointsHaveDuplicateData(r, r.getDocCount()/2));
+            }
         }
-        writer.close();
-        final IndexReader reader = DirectoryReader.open(dir);
-        boolean hasDuplicateData = indexFieldHasDuplicateData(reader, "duplicateField");
-        boolean hasDuplicateData2 = indexFieldHasDuplicateData(reader, "notDuplicateField");
-        reader.close();
-        dir.close();
-        assertTrue(hasDuplicateData);
-        assertFalse(hasDuplicateData2);
     }
 
     public void testMaxScoreQueryVisitor() {

From 2e617c3b1026bb3fc4063feeb314c1925e83300d Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 19 Mar 2020 11:23:15 -0700
Subject: [PATCH 147/285] Improve IntelliJ IDE integration (#53747)

This commit makes a number of improvements when importing the
Elasticsearch project into IntelliJ IDEA. Specifically:

 - Contributing documentation has been updated to reflect that the
   'idea' task should no long be used and Gradle project import is
   instead the officially supported way of setting up the project.
 - Attempts to run the 'idea' task will result in a failure with a
   message directing folks to our CONTRIBUTING.md document.
 - The project JDK is explicit set rather that using whatever JAVA_HOME
   is.
 - Gradle build operation delegation is disabled, and test execution is
   configured to 'choose per test'.
 - Gradle is configured to inherit the project JDK.
 - Some code style conventions are automatically configured.
 - File encoding is explicitly set to UTF-8.
 - Parallel module compilation is enabled and deprecated feature
   warnings are disabled.
 - A remote debug run configuration using listen mode is created.
 - JUnit runner is configured with required system properties.
 - License headers are configured such that Apache 2 is the default
   notice added to all source files with exception of source in /x-pack
   which will use the Elastic license.
---
 .gitignore                                   |   6 +
 .idea/inspectionProfiles/Project_Default.xml |   9 ++
 .idea/scopes/x_pack.xml                      |   3 +
 CONTRIBUTING.md                              |  88 +++--------
 TESTING.asciidoc                             |  10 +-
 build.gradle                                 |  32 +---
 gradle/ide.gradle                            | 147 +++++++++++++++++++
 server/build.gradle                          |   6 +-
 8 files changed, 198 insertions(+), 103 deletions(-)
 create mode 100644 .idea/inspectionProfiles/Project_Default.xml
 create mode 100644 .idea/scopes/x_pack.xml
 create mode 100644 gradle/ide.gradle

diff --git a/.gitignore b/.gitignore
index ec394ab09063a..309342f3b611d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,12 @@
 *.ipr
 *.iws
 build-idea/
+out/
+
+# include shared intellij config
+!.idea/scopes/x_pack.xml
+!.idea/inspectionProfiles/Project_Default.xml
+
 # These files are generated in the main tree by IntelliJ
 benchmarks/src/main/generated/*
 
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000000000..5cf789707c58c
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,9 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="GroovyPointlessBoolean" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="PointlessBooleanExpression" enabled="false" level="WARNING" enabled_by_default="false">
+      <option name="m_ignoreExpressionsContainingConstants" value="true" />
+    </inspection_tool>
+  </profile>
+</component>
\ No newline at end of file
diff --git a/.idea/scopes/x_pack.xml b/.idea/scopes/x_pack.xml
new file mode 100644
index 0000000000000..1ccf16614d596
--- /dev/null
+++ b/.idea/scopes/x_pack.xml
@@ -0,0 +1,3 @@
+<component name="DependencyValidationManager">
+  <scope name="x-pack" pattern="file[*.x-pack]:*/||file[*.x-pack*]:*//*" />
+</component>
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8598b6001a667..c072b7b3649ad 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -111,7 +111,7 @@ using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
 script on Windows in the root of the repository. The examples below show the
 usage on Unix.
 
-We support development in IntelliJ versions [IntelliJ 2017.2][intellij] and
+We support development in IntelliJ versions IntelliJ 2019.2 and
 onwards. We would like to support Eclipse, but few of us use it and has fallen
 into [disrepair][eclipse].
 
@@ -123,40 +123,18 @@ You can access Elasticsearch with:
 
     curl -u elastic:password localhost:9200
 
-### Configuring IDEs And Running Tests
-
-IntelliJ users can automatically configure their IDE: `./gradlew idea`
-then `File->New Project From Existing Sources`. Point to the root of
-the source directory, select
-`Import project from external model->Gradle`, enable
-`Use auto-import`. In order to run tests directly from
-IDEA 2017.2 and above, it is required to disable the IDEA run launcher in order to avoid
-`idea_rt.jar` causing "jar hell". This can be achieved by adding the
-`-Didea.no.launcher=true` [JVM
-option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties).
-Alternatively, `idea.no.launcher=true` can be set in the
-[`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
-file which can be accessed under Help > Edit Custom Properties (this will require a
-restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
-`Run->Edit Configurations->...->Defaults->JUnit` and verify that the `Shorten command line` setting is set to
-`user-local default: none`. You may also need to [remove `ant-javafx.jar` from your
-classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
-reported as a source of jar hell.
-
-To run an instance of elasticsearch from the source code run `./gradlew run`
-
-The Elasticsearch codebase makes heavy use of Java `assert`s and the
-test runner requires that assertions be enabled within the JVM. This
-can be accomplished by passing the flag `-ea` to the JVM on startup.
-
-You can enable these in IntelliJ by going to
-`Run->Edit Configurations...->Defaults->JUnit->VM options` and input
-`-ea`.
-
-Some tests related to locale testing also require the flag
-`-Djava.locale.providers` to be set. Set the VM options/VM arguments for
-IntelliJ like describe above to use
-`-Djava.locale.providers=SPI,COMPAT`.
+### Importing the project into IntelliJ IDEA
+
+Elasticsearch builds using Java 13. Before importing into IntelliJ you will need
+to define an appropriate SDK. The convention is that **this SDK should be named
+"13"** so that the project import will detect it automatically. For more details
+on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
+
+You can import the Elasticsearch project into IntelliJ IDEA via:
+
+ - Select **File > Open**
+ - In the subsequent dialog navigate to the root `build.gradle` file
+ - In the subsequent dialog select **Open as Project**
 
 ### REST Endpoint Conventions
 
@@ -204,11 +182,7 @@ Please follow these formatting guidelines:
   part of a file. Please format such sections sympathetically with the rest
   of the code, while keeping lines to maximum length of 76 characters.
 * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause
-  the build to fail. This can be done automatically by your IDE:
-   * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`.
-     There are two configuration options: `Class count to use import with
-     '*'` and `Names count to use static import with '*'`. Set their values
-     to 99999 or some other absurdly high value.
+  the build to fail.
 * If *absolutely* necessary, you can disable formatting for regions of code
   with the `// tag::NAME` and `// end::NAME` directives, but note that
   these are intended for use in documentation, so please make it clear what
@@ -379,26 +353,9 @@ It is important that the only code covered by the Elastic licence is contained
 within the top-level `x-pack` directory. The build will fail its pre-commit
 checks if contributed code does not have the appropriate license headers.
 
-You may find it helpful to configure your IDE to automatically insert the
-appropriate license header depending on the part of the project to which you are
-contributing.
-
-#### IntelliJ: Copyright & Scope Profiles
-
-To have IntelliJ insert the correct license, it is necessary to create to copyright profiles.
-These may potentially be called `apache2` and `commercial`. These can be created in
-`Preferences/Settings->Editor->Copyright->Copyright Profiles`. To associate these profiles to
-their respective directories, two "Scopes" will need to be created. These can be created in
-`Preferences/Settings->Appearances & Behavior->Scopes`. When creating scopes, be sure to choose
-the `shared` scope type. Create a scope, `apache2`, with
-the associated pattern of `!file[group:x-pack]:*/`. This pattern will exclude all the files contained in
-the `x-pack` directory. The other scope, `commercial`, will have the inverse pattern of `file[group:x-pack]:*/`.
-The two scopes, together, should account for all the files in the project. To associate the scopes
-with their copyright-profiles, go into `Preferences/Settings->Editor>Copyright` and use the `+` to add
-the associations `apache2/apache2` and `commercial/commercial`.
-
-Configuring these options in IntelliJ can be quite buggy, so do not be alarmed if you have to open/close
-the settings window and/or restart IntelliJ to see your changes take effect.
+> **NOTE:** If you have imported the project into IntelliJ IDEA the project will
+> be automatically configured to add the correct license header to new source
+> files based on the source location.
 
 ### Creating A Distribution
 
@@ -411,7 +368,7 @@ cd elasticsearch/
 To build a darwin-tar distribution, run this command:
 
 ```sh
-./gradlew -p distribution/archives/darwin-tar assemble --parallel
+./gradlew -p distribution/archives/darwin-tar assemble
 ```
 
 You will find the distribution under:
@@ -421,9 +378,12 @@ To create all build artifacts (e.g., plugins and Javadocs) as well as
 distributions in all formats, run this command:
 
 ```sh
-./gradlew assemble --parallel
+./gradlew assemble
 ```
 
+> **NOTE:** Running the task above will fail if you don't have a available
+> Docker installation.
+
 The package distributions (Debian and RPM) can be found under:
 `./distribution/packages/(deb|rpm|oss-deb|oss-rpm)/build/distributions/`
 
@@ -556,10 +516,6 @@ known as "transitive" dependencies".</dd>
 should not be shipped with the project because it is "provided" by the runtime
 somehow. Elasticsearch plugins use this configuration to include dependencies
 that are bundled with Elasticsearch's server.</dd>
-<dt>`bundle`</dt><dd>Only available in projects with the shadow plugin,
-dependencies with this configuration are bundled into the jar produced by the
-build. Since IDEs do not understand this configuration we rig them to treat
-dependencies in this configuration as `compile` dependencies.</dd>
 <dt>`testCompile`</dt><dd>Code that is on the classpath for compiling tests
 that are part of this project but not production code. The canonical example
 of this is `junit`.</dd>
diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index 6c3ca01c7ae72..fe04cc49637ba 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -20,9 +20,9 @@ To create a platform-specific build including the x-pack modules, use the
 following depending on your operating system:
 
 -----------------------------
-./gradlew :distribution:archives:linux-tar:assemble --parallel
-./gradlew :distribution:archives:darwin-tar:assemble --parallel
-./gradlew :distribution:archives:windows-zip:assemble --parallel
+./gradlew :distribution:archives:linux-tar:assemble
+./gradlew :distribution:archives:darwin-tar:assemble
+./gradlew :distribution:archives:windows-zip:assemble
 -----------------------------
 
 === Running Elasticsearch from a checkout
@@ -51,6 +51,10 @@ recommended to configure the IDE to initiate multiple listening attempts. In cas
 is called "Auto restart" and needs to be checked. In case of Eclipse, "Connection limit" setting
 needs to be configured with a greater value (ie 10 or more).
 
+NOTE: If you have imported the project into IntelliJ according to the instructions in
+link:/CONTRIBUTING.md#importing-the-project-into-intellij-idea[CONTRIBUTING.md] then a debug run configuration
+named "Debug Elasticsearch" will be created for you and configured appropriately.
+
 ==== Distribution
 
 By default a node is started with the zip distribution.
diff --git a/build.gradle b/build.gradle
index 2dbf94ae7dabb..fa1ebcf6706f8 100644
--- a/build.gradle
+++ b/build.gradle
@@ -43,6 +43,7 @@ apply plugin: 'nebula.info-scm'
 apply from: 'gradle/build-scan.gradle'
 apply from: 'gradle/build-complete.gradle'
 apply from: 'gradle/runtime-jdk-provision.gradle'
+apply from: 'gradle/ide.gradle'
 
 // common maven publishing configuration
 allprojects {
@@ -180,7 +181,6 @@ allprojects {
       System.getProperty("eclipse.application") != null ||    // Detects gradle launched from the Eclipse compiler server
       gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
       gradle.startParameter.taskNames.contains('cleanEclipse')
-    isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
 
     // for BWC testing
     bwcVersions = versions
@@ -356,36 +356,6 @@ gradle.projectsEvaluated {
   }
 }
 
-// intellij configuration
-allprojects {
-  apply plugin: 'idea'
-
-  if (isIdea) {
-    project.buildDir = file('build-idea')
-  }
-  idea {
-    module {
-      inheritOutputDirs = false
-      outputDir = file('build-idea/classes/main')
-      testOutputDir = file('build-idea/classes/test')
-
-      // also ignore other possible build dirs
-      excludeDirs += file('build')
-      excludeDirs += file('build-eclipse')
-    }
-  }
-
-  tasks.named('cleanIdea') {
-    delete 'build-idea'
-  }
-}
-
-idea {
-  project {
-    vcs = 'Git'
-  }
-}
-
 // eclipse configuration
 allprojects {
   apply plugin: 'eclipse'
diff --git a/gradle/ide.gradle b/gradle/ide.gradle
new file mode 100644
index 0000000000000..3580f4a28f28e
--- /dev/null
+++ b/gradle/ide.gradle
@@ -0,0 +1,147 @@
+import org.jetbrains.gradle.ext.Remote
+import org.jetbrains.gradle.ext.JUnit
+
+buildscript {
+  repositories {
+    maven {
+      url "https://plugins.gradle.org/m2/"
+    }
+  }
+  dependencies {
+    classpath "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:0.7"
+  }
+}
+
+apply plugin: org.jetbrains.gradle.ext.IdeaExtPlugin
+
+allprojects {
+  apply plugin: 'idea'
+
+  tasks.named('idea') {
+    doFirst { throw new GradleException("Use of the 'idea' task has been deprecated. For details on importing into IntelliJ see CONTRIBUTING.md.") }
+  }
+}
+
+tasks.register('configureIdeaGradleJvm') {
+  group = 'ide'
+  description = 'Configures the appropriate JVM for Gradle'
+
+  doLast {
+    modifyXml('.idea/gradle.xml') { xml ->
+      def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings
+      // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle
+      gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) }
+    }
+  }
+}
+
+tasks.register('configureIdeaRunConfigs') {
+  group = 'ide'
+  description = 'Configures default run configuration settings'
+
+  doLast {
+    modifyXml('.idea/workspace.xml') { xml ->
+      def runManager = xml.component.find { it.'@name' == 'RunManager' }
+      if (runManager == null) {
+        throw new GradleException("IntelliJ 'RunManager' configuration is missing from workspace.xml. You may need to refresh your Gradle project.")
+      }
+
+      def debugConfig = runManager.configuration.find { it.'@name' == 'Debug Elasticsearch' }
+      // Enable "auto restart" on remote debug run configuration
+      if (debugConfig.option.any { it.'@name' == 'AUTO_RESTART' && it.'@value' == 'true'} == false) {
+        def restart = new NodeBuilder().option(name: 'AUTO_RESTART', value: 'true')
+        debugConfig.append(restart)
+      }
+    }
+  }
+}
+
+idea {
+  project {
+    vcs = 'Git'
+    jdkName = '13'
+
+    settings {
+      delegateActions {
+        delegateBuildRunToGradle = false
+        testRunner = 'choose_per_test'
+      }
+      taskTriggers {
+        afterSync tasks.named('configureIdeaGradleJvm'), tasks.named('configureIdeaRunConfigs')
+      }
+      codeStyle {
+        java {
+          classCountToUseImportOnDemand = 999
+        }
+      }
+      encodings {
+        encoding = 'UTF-8'
+      }
+      compiler {
+        parallelCompilation = true
+        javac {
+          generateDeprecationWarnings = false
+        }
+      }
+      runConfigurations {
+        'Debug Elasticsearch'(Remote) {
+          mode = 'listen'
+          host = 'localhost'
+          port = 5005
+        }
+        defaults(JUnit) {
+          vmParameters = '-Djava.locale.providers=SPI,COMPAT'
+        }
+      }
+      copyright {
+        useDefault = 'Apache'
+        scopes = ['x-pack': 'Elastic']
+        profiles {
+          Apache {
+            keyword = 'Licensed to Elasticsearch under one or more contributor'
+            notice = '''\
+               Licensed to Elasticsearch under one or more contributor
+               license agreements. See the NOTICE file distributed with
+               this work for additional information regarding copyright
+               ownership. Elasticsearch licenses this file to you under
+               the Apache License, Version 2.0 (the "License"); you may
+               not use this file except in compliance with the License.
+               You may obtain a copy of the License at
+
+                       http://www.apache.org/licenses/LICENSE-2.0
+
+               Unless required by applicable law or agreed to in writing,
+               software distributed under the License is distributed on an
+               "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+               KIND, either express or implied.  See the License for the
+               specific language governing permissions and limitations
+               under the License.'''.stripIndent()
+          }
+          Elastic {
+            keyword = 'Licensed under the Elastic License'
+            notice = '''\
+               Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+               or more contributor license agreements. Licensed under the Elastic License;
+               you may not use this file except in compliance with the Elastic License.'''.stripIndent()
+          }
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
+ *
+ * @param path Path to existing XML file
+ * @param action Action to perform on parsed XML document
+ */
+void modifyXml(Object path, Action<? super Node> action) {
+  File xmlFile = project.file(path)
+  Node xml = new XmlParser().parse(xmlFile)
+  action.execute(xml)
+
+  xmlFile.withPrintWriter { writer ->
+    new XmlNodePrinter(writer).print(xml)
+  }
+}
diff --git a/server/build.gradle b/server/build.gradle
index fe0b2c3fee933..59c41c7679d20 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -35,7 +35,7 @@ publishing {
 archivesBaseName = 'elasticsearch'
 
 // we want to keep the JDKs in our IDEs set to JDK 11 until minimum JDK is bumped to 17 so we do not include this source set in our IDEs
-if (!isEclipse && !isIdea) {
+if (!isEclipse) {
   sourceSets {
     java12 {
       java {
@@ -82,7 +82,7 @@ dependencies {
   compile project(":libs:elasticsearch-geo")
 
   compileOnly project(':libs:elasticsearch-plugin-classloader')
-  testRuntime project(':libs:elasticsearch-plugin-classloader')
+  testRuntimeOnly project(':libs:elasticsearch-plugin-classloader')
 
   // lucene
   compile "org.apache.lucene:lucene-core:${versions.lucene}"
@@ -123,7 +123,7 @@ dependencies {
   // repackaged jna with native bits linked against all elastic supported platforms
   compile "org.elasticsearch:jna:${versions.jna}"
 
-  if (!isEclipse && !isIdea) {
+  if (!isEclipse) {
     java12Compile sourceSets.main.output
   }
 

From dea6b7d67fab8d6db0a4cd4bd6502e56fe73ee27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Thu, 19 Mar 2020 20:13:40 +0100
Subject: [PATCH 148/285] Add async_search.submit API to HLRC (#53592)

This commit adds a new AsyncSearchClient to the High Level Rest Client which
initially supporst the submitAsyncSearch in its blocking and non-blocking
flavour. Also adding client side request and response objects and parsing code
to parse the xContent output of the client side AsyncSearchResponse together
with parsing roundtrip tests and a simple roundtrip integration test.

Relates to #49091
---
 .../client/AsyncSearchClient.java             |  66 ++++
 .../client/AsyncSearchRequestConverters.java  |  74 +++++
 .../client/RequestConverters.java             |  32 +-
 .../client/RestHighLevelClient.java           |  13 +-
 .../asyncsearch/AsyncSearchResponse.java      | 214 +++++++++++++
 .../asyncsearch/SubmitAsyncSearchRequest.java | 284 ++++++++++++++++++
 .../AsyncSearchRequestConvertersTests.java    | 115 +++++++
 .../client/RequestConvertersTests.java        |  41 ++-
 .../client/RestHighLevelClientTests.java      |   1 +
 .../client/asyncsearch/AsyncSearchIT.java     |  57 ++++
 .../asyncsearch/AsyncSearchResponseTests.java |  85 ++++++
 .../SubmitAsyncSearchRequestTests.java        |  57 ++++
 .../action/search/SearchResponse.java         |   2 +-
 .../search/action/AsyncSearchResponse.java    |   2 +
 14 files changed, 1020 insertions(+), 23 deletions(-)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequestTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
new file mode 100644
index 0000000000000..ab47b1c5e792a
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.asyncsearch.AsyncSearchResponse;
+import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptySet;
+
+public class AsyncSearchClient {
+    private final RestHighLevelClient restHighLevelClient;
+
+    AsyncSearchClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Submit a new async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::submitAsyncSearch, options,
+                AsyncSearchResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously submit a new async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-ilm-ilm-get-lifecycle-policy.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable submitAsyncSearchAsync(SubmitAsyncSearchRequest request, RequestOptions options,
+                                               ActionListener<AsyncSearchResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::submitAsyncSearch, options,
+                AsyncSearchResponse::fromXContent, listener, emptySet());
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
new file mode 100644
index 0000000000000..2d91bd926ef4d
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.client.RequestConverters.Params;
+import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE;
+
+final class AsyncSearchRequestConverters {
+
+    static Request submitAsyncSearch(SubmitAsyncSearchRequest asyncSearchRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder().addCommaSeparatedPathParts(
+                asyncSearchRequest.getIndices())
+                .addPathPartAsIs("_async_search").build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        Params params = new RequestConverters.Params();
+        // add all typical search params and search request source as body
+        addSearchRequestParams(params, asyncSearchRequest);
+        if (asyncSearchRequest.getSearchSource() != null) {
+            request.setEntity(RequestConverters.createEntity(asyncSearchRequest.getSearchSource(), REQUEST_BODY_CONTENT_TYPE));
+        }
+        // set async search submit specific parameters
+        if (asyncSearchRequest.isCleanOnCompletion() != null) {
+            params.putParam("clean_on_completion", asyncSearchRequest.isCleanOnCompletion().toString());
+        }
+        if (asyncSearchRequest.getKeepAlive() != null) {
+            params.putParam("keep_alive", asyncSearchRequest.getKeepAlive().getStringRep());
+        }
+        if (asyncSearchRequest.getWaitForCompletion() != null) {
+            params.putParam("wait_for_completion", asyncSearchRequest.getWaitForCompletion().getStringRep());
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static void addSearchRequestParams(Params params, SubmitAsyncSearchRequest request) {
+        params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
+        params.withRouting(request.getRouting());
+        params.withPreference(request.getPreference());
+        params.withIndicesOptions(request.getIndicesOptions());
+        params.withSearchType(request.getSearchType().name().toLowerCase(Locale.ROOT));
+        params.withMaxConcurrentShardRequests(request.getMaxConcurrentShardRequests());
+        if (request.getRequestCache() != null) {
+            params.withRequestCache(request.getRequestCache());
+        }
+        if (request.getAllowPartialSearchResults() != null) {
+            params.withAllowPartialResults(request.getAllowPartialSearchResults());
+        }
+        params.withBatchedReduceSize(request.getBatchedReduceSize());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index a8532f6d16339..0bb0eca83a389 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -403,22 +403,22 @@ static Request search(SearchRequest searchRequest, String searchEndpoint) throws
         return request;
     }
 
-    private static void addSearchRequestParams(Params params, SearchRequest searchRequest) {
+    static void addSearchRequestParams(Params params, SearchRequest searchRequest) {
         params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
         params.withRouting(searchRequest.routing());
         params.withPreference(searchRequest.preference());
         params.withIndicesOptions(searchRequest.indicesOptions());
-        params.putParam("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT));
+        params.withSearchType(searchRequest.searchType().name().toLowerCase(Locale.ROOT));
         params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips()));
         params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
-        params.putParam("max_concurrent_shard_requests", Integer.toString(searchRequest.getMaxConcurrentShardRequests()));
+        params.withMaxConcurrentShardRequests(searchRequest.getMaxConcurrentShardRequests());
         if (searchRequest.requestCache() != null) {
-            params.putParam("request_cache", Boolean.toString(searchRequest.requestCache()));
+            params.withRequestCache(searchRequest.requestCache());
         }
         if (searchRequest.allowPartialSearchResults() != null) {
-            params.putParam("allow_partial_search_results", Boolean.toString(searchRequest.allowPartialSearchResults()));
+            params.withAllowPartialResults(searchRequest.allowPartialSearchResults());
         }
-        params.putParam("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize()));
+        params.withBatchedReduceSize(searchRequest.getBatchedReduceSize());
         if (searchRequest.scroll() != null) {
             params.putParam("scroll", searchRequest.scroll().keepAlive());
         }
@@ -860,6 +860,26 @@ Params withPreference(String preference) {
             return putParam("preference", preference);
         }
 
+        Params withSearchType(String searchType) {
+            return putParam("search_type", searchType);
+        }
+
+        Params withMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
+            return putParam("max_concurrent_shard_requests", Integer.toString(maxConcurrentShardRequests));
+        }
+
+        Params withBatchedReduceSize(int batchedReduceSize) {
+            return putParam("batched_reduce_size", Integer.toString(batchedReduceSize));
+        }
+
+        Params withRequestCache(boolean requestCache) {
+            return putParam("request_cache", Boolean.toString(requestCache));
+        }
+
+        Params withAllowPartialResults(boolean allowPartialSearchResults) {
+            return putParam("allow_partial_search_results", Boolean.toString(allowPartialSearchResults));
+        }
+
         Params withRealtime(boolean realtime) {
             if (realtime == false) {
                 return putParam("realtime", Boolean.FALSE.toString());
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index e1ca36f552819..8b5262d6aada6 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -265,6 +265,7 @@ public class RestHighLevelClient implements Closeable {
     private final TransformClient transformClient = new TransformClient(this);
     private final EnrichClient enrichClient = new EnrichClient(this);
     private final EqlClient eqlClient = new EqlClient(this);
+    private final AsyncSearchClient asyncSearchClient = new AsyncSearchClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -428,13 +429,23 @@ public final XPackClient xpack() {
      * A wrapper for the {@link RestHighLevelClient} that provides methods for
      * accessing the Elastic Index Lifecycle APIs.
      * <p>
-     * See the <a href="http://FILL-ME-IN-WE-HAVE-NO-DOCS-YET.com"> X-Pack APIs
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management-api.html"> X-Pack APIs
      * on elastic.co</a> for more information.
      */
     public IndexLifecycleClient indexLifecycle() {
         return ilmClient;
     }
 
+    /**
+     * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Elastic Index Async Search APIs.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> X-Pack APIs on elastic.co</a>
+     * for more information.
+     */
+    public AsyncSearchClient asyncSearch() {
+        return asyncSearchClient;
+    }
+
     /**
      * Provides methods for accessing the Elastic Licensed Migration APIs that
      * are shipped with the default distribution of Elasticsearch. All of
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java
new file mode 100644
index 0000000000000..47dd444ea5463
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * A response of an async search request.
+ */
+public class AsyncSearchResponse implements ToXContentObject  {
+    @Nullable
+    private final String id;
+    private final int version;
+    @Nullable
+    private final SearchResponse searchResponse;
+    @Nullable
+    private final ElasticsearchException error;
+    private final boolean isRunning;
+    private final boolean isPartial;
+
+    private final long startTimeMillis;
+    private final long expirationTimeMillis;
+
+    /**
+     * Creates an {@link AsyncSearchResponse} with the arguments that are always present in the server response
+     */
+    AsyncSearchResponse(int version,
+                               boolean isPartial,
+                               boolean isRunning,
+                               long startTimeMillis,
+                               long expirationTimeMillis,
+                               @Nullable String id,
+                               @Nullable SearchResponse searchResponse,
+                               @Nullable ElasticsearchException error) {
+        this.version = version;
+        this.isPartial = isPartial;
+        this.isRunning = isRunning;
+        this.startTimeMillis = startTimeMillis;
+        this.expirationTimeMillis = expirationTimeMillis;
+        this.id = id;
+        this.searchResponse = searchResponse;
+        this.error = error;
+    }
+
+    /**
+     * Returns the id of the async search request or null if the response is not stored in the cluster.
+     */
+    @Nullable
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Returns the version of this response.
+     */
+    public int getVersion() {
+        return version;
+    }
+
+    /**
+     * Returns the current {@link SearchResponse} or <code>null</code> if not available.
+     *
+     * See {@link #isPartial()} to determine whether the response contains partial or complete
+     * results.
+     */
+    public SearchResponse getSearchResponse() {
+        return searchResponse;
+    }
+
+    /**
+     * Returns the failure reason or null if the query is running or has completed normally.
+     */
+    public ElasticsearchException getFailure() {
+        return error;
+    }
+
+    /**
+     * Returns <code>true</code> if the {@link SearchResponse} contains partial
+     * results computed from a subset of the total shards.
+     */
+    public boolean isPartial() {
+        return isPartial;
+    }
+
+    /**
+     * Whether the search is still running in the cluster.
+     *
+     * A value of <code>false</code> indicates that the response is final
+     * even if {@link #isPartial()} returns <code>true</code>. In such case,
+     * the partial response represents the status of the search before a
+     * non-recoverable failure.
+     */
+    public boolean isRunning() {
+        return isRunning;
+    }
+
+    /**
+     * When this response was created as a timestamp in milliseconds since epoch.
+     */
+    public long getStartTime() {
+        return startTimeMillis;
+    }
+
+    /**
+     * When this response will expired as a timestamp in milliseconds since epoch.
+     */
+    public long getExpirationTime() {
+        return expirationTimeMillis;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (id != null) {
+            builder.field("id", id);
+        }
+        builder.field("version", version);
+        builder.field("is_partial", isPartial);
+        builder.field("is_running", isRunning);
+        builder.field("start_time_in_millis", startTimeMillis);
+        builder.field("expiration_time_in_millis", expirationTimeMillis);
+
+        if (searchResponse != null) {
+            builder.field("response");
+            searchResponse.toXContent(builder, params);
+        }
+        if (error != null) {
+            builder.startObject("error");
+            error.toXContent(builder, params);
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public static final ParseField ID_FIELD = new ParseField("id");
+    public static final ParseField VERSION_FIELD = new ParseField("version");
+    public static final ParseField IS_PARTIAL_FIELD = new ParseField("is_partial");
+    public static final ParseField IS_RUNNING_FIELD = new ParseField("is_running");
+    public static final ParseField START_TIME_FIELD = new ParseField("start_time_in_millis");
+    public static final ParseField EXPIRATION_FIELD = new ParseField("expiration_time_in_millis");
+    public static final ParseField RESPONSE_FIELD = new ParseField("response");
+    public static final ParseField ERROR_FIELD = new ParseField("error");
+
+    public static final ConstructingObjectParser<AsyncSearchResponse, Void> PARSER = new ConstructingObjectParser<>(
+            "submit_async_search_response", true,
+            args -> new AsyncSearchResponse(
+                    (int) args[0],
+                    (boolean) args[1],
+                    (boolean) args[2],
+                    (long) args[3],
+                    (long) args[4],
+                    (String) args[5],
+                    (SearchResponse) args[6],
+                    (ElasticsearchException) args[7]));
+    static {
+        PARSER.declareInt(constructorArg(), VERSION_FIELD);
+        PARSER.declareBoolean(constructorArg(), IS_PARTIAL_FIELD);
+        PARSER.declareBoolean(constructorArg(), IS_RUNNING_FIELD);
+        PARSER.declareLong(constructorArg(), START_TIME_FIELD);
+        PARSER.declareLong(constructorArg(), EXPIRATION_FIELD);
+        PARSER.declareString(optionalConstructorArg(), ID_FIELD);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> AsyncSearchResponse.parseSearchResponse(p),
+                RESPONSE_FIELD);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), ERROR_FIELD);
+    }
+
+    private static SearchResponse parseSearchResponse(XContentParser p) throws IOException {
+        // we should be before the opening START_OBJECT of the response
+        ensureExpectedToken(Token.START_OBJECT, p.currentToken(), p::getTokenLocation);
+        p.nextToken();
+        return SearchResponse.innerFromXContent(p);
+    }
+
+    public static AsyncSearchResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
new file mode 100644
index 0000000000000..1b0a07c4dea41
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A request to track asynchronously the progress of a search against one or more indices.
+ */
+public class SubmitAsyncSearchRequest implements Validatable {
+
+    public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 1;
+    public static final int DEFAULT_BATCHED_REDUCE_SIZE = 5;
+    private static final boolean DEFAULT_CCS_MINIMIZE_ROUNDTRIPS = false;
+    private static final boolean DEFAULT_REQUEST_CACHE_VALUE = true;
+
+    public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
+
+    private TimeValue waitForCompletion;
+    private Boolean cleanOnCompletion;
+    private TimeValue keepAlive;
+    private final SearchRequest searchRequest;
+
+    /**
+     * Creates a new request
+     */
+    public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) {
+        this.searchRequest = new SearchRequest(indices, source);
+        searchRequest.setCcsMinimizeRoundtrips(DEFAULT_CCS_MINIMIZE_ROUNDTRIPS);
+        searchRequest.setPreFilterShardSize(DEFAULT_PRE_FILTER_SHARD_SIZE);
+        searchRequest.setBatchedReduceSize(DEFAULT_BATCHED_REDUCE_SIZE);
+        searchRequest.requestCache(DEFAULT_REQUEST_CACHE_VALUE);
+    }
+
+    /**
+     * Get the target indices
+     */
+    public String[] getIndices() {
+        return this.searchRequest.indices();
+    }
+
+
+    /**
+     * Get the minimum time that the request should wait before returning a partial result (defaults to 1 second).
+     */
+    public TimeValue getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    /**
+     * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
+     */
+    public void setWaitForCompletion(TimeValue waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+    }
+
+    /**
+     * Returns whether the resource resource should be removed on completion or failure (defaults to true).
+     */
+    public Boolean isCleanOnCompletion() {
+        return cleanOnCompletion;
+    }
+
+    /**
+     * Determines if the resource should be removed on completion or failure (defaults to true).
+     */
+    public void setCleanOnCompletion(boolean cleanOnCompletion) {
+        this.cleanOnCompletion = cleanOnCompletion;
+    }
+
+    /**
+     * Get the amount of time after which the result will expire (defaults to 5 days).
+     */
+    public TimeValue getKeepAlive() {
+        return keepAlive;
+    }
+
+    /**
+     * Sets the amount of time after which the result will expire (defaults to 5 days).
+     */
+    public void setKeepAlive(TimeValue keepAlive) {
+        this.keepAlive = keepAlive;
+    }
+
+    // setters for request parameters of the wrapped SearchRequest
+    /**
+     * Set the routing value to control the shards that the search will be executed on.
+     * A comma separated list of routing values to control the shards the search will be executed on.
+     */
+    public void setRouting(String routing) {
+        this.searchRequest.routing(routing);
+    }
+
+    /**
+     * Set the routing values to control the shards that the search will be executed on.
+     */
+    public void setRoutings(String... routings) {
+        this.searchRequest.routing(routings);
+    }
+
+    /**
+     * Get the routing value to control the shards that the search will be executed on.
+     */
+    public String getRouting() {
+        return this.searchRequest.routing();
+    }
+
+    /**
+     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
+     * {@code _local} to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
+     */
+    public void setPreference(String preference) {
+        this.searchRequest.preference(preference);
+    }
+
+    /**
+     * Get the preference to execute the search.
+     */
+    public String getPreference() {
+        return this.searchRequest.preference();
+    }
+
+    /**
+     * Specifies what type of requested indices to ignore and how to deal with indices wildcard expressions.
+     */
+    public void setIndicesOptions(IndicesOptions indicesOptions) {
+        this.searchRequest.indicesOptions(indicesOptions);
+    }
+
+    /**
+     * Get the indices Options.
+     */
+    public IndicesOptions getIndicesOptions() {
+        return this.searchRequest.indicesOptions();
+    }
+
+    /**
+     * The search type to execute, defaults to {@link SearchType#DEFAULT}.
+     */
+    public void setSearchType(SearchType searchType) {
+        this.searchRequest.searchType(searchType);
+    }
+
+    /**
+     * Get the search type to execute, defaults to {@link SearchType#DEFAULT}.
+     */
+    public SearchType getSearchType() {
+        return this.searchRequest.searchType();
+    }
+
+    /**
+     * Sets if this request should allow partial results. (If method is not called,
+     * will default to the cluster level setting).
+     */
+    public void setAllowPartialSearchResults(boolean allowPartialSearchResults) {
+        this.searchRequest.allowPartialSearchResults(allowPartialSearchResults);
+    }
+
+    /**
+     * Gets if this request should allow partial results.
+     */
+    public Boolean getAllowPartialSearchResults() {
+        return this.searchRequest.allowPartialSearchResults();
+    }
+
+    /**
+     * Sets the number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection
+     * mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large.
+     */
+    public void setBatchedReduceSize(int batchedReduceSize) {
+        this.searchRequest.setBatchedReduceSize(batchedReduceSize);
+    }
+
+    /**
+     * Gets the number of shard results that should be reduced at once on the coordinating node.
+     * This defaults to 5 for {@link SubmitAsyncSearchRequest}.
+     */
+    public int getBatchedReduceSize() {
+        return this.searchRequest.getBatchedReduceSize();
+    }
+
+    /**
+     * Sets if this request should use the request cache or not, assuming that it can (for
+     * example, if "now" is used, it will never be cached). By default (not set, or null,
+     * will default to the index level setting if request cache is enabled or not).
+     */
+    public void setRequestCache(Boolean requestCache) {
+        this.searchRequest.requestCache(requestCache);
+    }
+
+    /**
+     * Gets if this request should use the request cache or not.
+     * Defaults to `true` for {@link SubmitAsyncSearchRequest}.
+     */
+    public Boolean getRequestCache() {
+        return this.searchRequest.requestCache();
+    }
+
+    /**
+     * Returns the number of shard requests that should be executed concurrently on a single node.
+     * The default is {@code 5}.
+     */
+    public int getMaxConcurrentShardRequests() {
+        return this.searchRequest.getMaxConcurrentShardRequests();
+    }
+
+    /**
+     * Sets the number of shard requests that should be executed concurrently on a single node.
+     * The default is {@code 5}.
+     */
+    public void setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
+        this.searchRequest.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
+    }
+
+    /**
+     * Gets if the source of the {@link SearchSourceBuilder} initially used on this request.
+     */
+    public SearchSourceBuilder getSearchSource() {
+        return this.searchRequest.source();
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        final ValidationException validationException = new ValidationException();
+        if (searchRequest.isSuggestOnly()) {
+            validationException.addValidationError("suggest-only queries are not supported");
+        }
+        if (keepAlive != null && keepAlive.getMillis() < MIN_KEEP_ALIVE) {
+            validationException.addValidationError("[keep_alive] must be greater than 1 minute, got: " + keepAlive.toString());
+        }
+        if (validationException.validationErrors().isEmpty()) {
+            return Optional.empty();
+        }
+        return Optional.of(validationException);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        SubmitAsyncSearchRequest request = (SubmitAsyncSearchRequest) o;
+        return Objects.equals(searchRequest, request.searchRequest)
+                && Objects.equals(getKeepAlive(), request.getKeepAlive())
+                && Objects.equals(getWaitForCompletion(), request.getWaitForCompletion())
+                && Objects.equals(isCleanOnCompletion(), request.isCleanOnCompletion());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(searchRequest, getKeepAlive(), getWaitForCompletion(), isCleanOnCompletion());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
new file mode 100644
index 0000000000000..df8fc65c751e6
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.StringJoiner;
+
+import static org.elasticsearch.client.RequestConvertersTests.createTestSearchSourceBuilder;
+import static org.elasticsearch.client.RequestConvertersTests.setRandomIndicesOptions;
+
+public class AsyncSearchRequestConvertersTests extends ESTestCase {
+
+    public void testSubmitAsyncSearch() throws Exception {
+        String[] indices = RequestConvertersTests.randomIndicesNames(0, 5);
+        Map<String, String> expectedParams = new HashMap<>();
+        SearchSourceBuilder searchSourceBuilder = createTestSearchSourceBuilder();
+        SubmitAsyncSearchRequest submitRequest = new SubmitAsyncSearchRequest(searchSourceBuilder, indices);
+
+        // the following parameters might be overwritten by random ones later,
+        // but we need to set these since they are the default we send over http
+        expectedParams.put("request_cache", "true");
+        expectedParams.put("batched_reduce_size", "5");
+        setRandomSearchParams(submitRequest, expectedParams);
+        setRandomIndicesOptions(submitRequest::setIndicesOptions, submitRequest::getIndicesOptions, expectedParams);
+
+        if (randomBoolean()) {
+            boolean cleanOnCompletion = randomBoolean();
+            submitRequest.setCleanOnCompletion(cleanOnCompletion);
+            expectedParams.put("clean_on_completion", Boolean.toString(cleanOnCompletion));
+        }
+        if (randomBoolean()) {
+            TimeValue keepAlive = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setKeepAlive(keepAlive);
+            expectedParams.put("keep_alive", keepAlive.getStringRep());
+        }
+        if (randomBoolean()) {
+            TimeValue waitForCompletion = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setWaitForCompletion(waitForCompletion);
+            expectedParams.put("wait_for_completion", waitForCompletion.getStringRep());
+        }
+
+        Request request = AsyncSearchRequestConverters.submitAsyncSearch(submitRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String index = String.join(",", indices);
+        if (Strings.hasLength(index)) {
+            endpoint.add(index);
+        }
+        endpoint.add("_async_search");
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        RequestConvertersTests.assertToXContentBody(searchSourceBuilder, request.getEntity());
+    }
+
+    private static void setRandomSearchParams(SubmitAsyncSearchRequest request, Map<String, String> expectedParams) {
+        expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true");
+        if (randomBoolean()) {
+            request.setRouting(randomAlphaOfLengthBetween(3, 10));
+            expectedParams.put("routing", request.getRouting());
+        }
+        if (randomBoolean()) {
+            request.setPreference(randomAlphaOfLengthBetween(3, 10));
+            expectedParams.put("preference", request.getPreference());
+        }
+        if (randomBoolean()) {
+            request.setSearchType(randomFrom(SearchType.CURRENTLY_SUPPORTED));
+        }
+        expectedParams.put("search_type", request.getSearchType().name().toLowerCase(Locale.ROOT));
+        if (randomBoolean()) {
+            request.setAllowPartialSearchResults(randomBoolean());
+            expectedParams.put("allow_partial_search_results", Boolean.toString(request.getAllowPartialSearchResults()));
+        }
+        if (randomBoolean()) {
+            request.setRequestCache(randomBoolean());
+            expectedParams.put("request_cache", Boolean.toString(request.getRequestCache()));
+        }
+        if (randomBoolean()) {
+            request.setBatchedReduceSize(randomIntBetween(2, Integer.MAX_VALUE));
+        }
+        expectedParams.put("batched_reduce_size", Integer.toString(request.getBatchedReduceSize()));
+        if (randomBoolean()) {
+            request.setMaxConcurrentShardRequests(randomIntBetween(1, Integer.MAX_VALUE));
+        }
+        expectedParams.put("max_concurrent_shard_requests", Integer.toString(request.getMaxConcurrentShardRequests()));
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 3e4ecda385388..a7291b3dbbcf3 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -1021,12 +1021,34 @@ public void testSearchNullSource() throws IOException {
     public void testSearch() throws Exception {
         String searchEndpoint = randomFrom("_" + randomAlphaOfLength(5));
         String[] indices = randomIndicesNames(0, 5);
+        Map<String, String> expectedParams = new HashMap<>();
+        SearchRequest searchRequest = createTestSearchRequest(indices, expectedParams);
+
+        Request request = RequestConverters.search(searchRequest, searchEndpoint);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String index = String.join(",", indices);
+        if (Strings.hasLength(index)) {
+            endpoint.add(index);
+        }
+        endpoint.add(searchEndpoint);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertToXContentBody(searchRequest.source(), request.getEntity());
+    }
+
+    public static SearchRequest createTestSearchRequest(String[] indices, Map<String, String> expectedParams) {
         SearchRequest searchRequest = new SearchRequest(indices);
 
-        Map<String, String> expectedParams = new HashMap<>();
         setRandomSearchParams(searchRequest, expectedParams);
         setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams);
 
+        SearchSourceBuilder searchSourceBuilder = createTestSearchSourceBuilder();
+        searchRequest.source(searchSourceBuilder);
+        return searchRequest;
+    }
+
+    public static SearchSourceBuilder createTestSearchSourceBuilder() {
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         // rarely skip setting the search source completely
         if (frequently()) {
@@ -1070,22 +1092,11 @@ public void testSearch() throws Exception {
                     searchSourceBuilder.collapse(new CollapseBuilder(randomAlphaOfLengthBetween(3, 10)));
                 }
             }
-            searchRequest.source(searchSourceBuilder);
-        }
-
-        Request request = RequestConverters.search(searchRequest, searchEndpoint);
-        StringJoiner endpoint = new StringJoiner("/", "/", "");
-        String index = String.join(",", indices);
-        if (Strings.hasLength(index)) {
-            endpoint.add(index);
         }
-        endpoint.add(searchEndpoint);
-        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
-        assertEquals(endpoint.toString(), request.getEndpoint());
-        assertEquals(expectedParams, request.getParameters());
-        assertToXContentBody(searchSourceBuilder, request.getEntity());
+        return searchSourceBuilder;
     }
 
+
     public void testSearchNullIndicesAndTypes() {
         expectThrows(NullPointerException.class, () -> new SearchRequest((String[]) null));
         expectThrows(NullPointerException.class, () -> new SearchRequest().indices((String[]) null));
@@ -1867,7 +1878,7 @@ private static void setRandomSearchParams(SearchRequest searchRequest,
         expectedParams.put("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
     }
 
-    static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<IndicesOptions> getter,
+    public static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<IndicesOptions> getter,
                                         Map<String, String> expectedParams) {
 
         if (randomBoolean()) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 870db8a63a970..468dbbd5d3ef7 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -890,6 +890,7 @@ public void testApiNamingConventions() throws Exception {
                                 apiName.startsWith("eql.") == false &&
                                 apiName.endsWith("freeze") == false &&
                                 apiName.endsWith("reload_analyzers") == false &&
+                                apiName.startsWith("async_search") == false &&
                                 // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we
                                 // can get rid of 7.0's deprecated "getTemplate"
                                 apiName.equals("indices.get_index_template") == false) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
new file mode 100644
index 0000000000000..a1c608b3c6ef4
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public class AsyncSearchIT extends ESRestHighLevelClientTestCase {
+
+    public void testSubmitAsyncSearchRequest() throws IOException {
+        String index = "test-index";
+        createIndex(index, Settings.EMPTY);
+
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
+        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(sourceBuilder, index);
+        // 15 sec should be enough to make sure we always complete right away
+        request.setWaitForCompletion(new TimeValue(15, TimeUnit.SECONDS));
+        AsyncSearchResponse response = highLevelClient().asyncSearch().submitAsyncSearch(request, RequestOptions.DEFAULT);
+        assertTrue(response.getVersion() >= 0);
+        assertFalse(response.isPartial());
+        assertTrue(response.getStartTime() > 0);
+        assertTrue(response.getExpirationTime() > 0);
+        assertNotNull(response.getSearchResponse());
+        if (response.isRunning() == false) {
+            assertNull(response.getId());
+            assertFalse(response.isPartial());
+        } else {
+            assertTrue(response.isPartial());
+            assertNotNull(response.getId());
+        }
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
new file mode 100644
index 0000000000000..6caf69db9ac6c
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponse.Clusters;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class AsyncSearchResponseTests
+        extends AbstractResponseTestCase<org.elasticsearch.xpack.core.search.action.AsyncSearchResponse, AsyncSearchResponse> {
+
+    @Override
+    protected org.elasticsearch.xpack.core.search.action.AsyncSearchResponse createServerTestInstance(XContentType xContentType) {
+        int version = randomIntBetween(0, Integer.MAX_VALUE);
+        boolean isPartial = randomBoolean();
+        boolean isRunning = randomBoolean();
+        long startTimeMillis = randomLongBetween(0, Long.MAX_VALUE);
+        long expirationTimeMillis = randomLongBetween(0, Long.MAX_VALUE);
+        String id = randomBoolean() ? null : randomAlphaOfLength(10);
+        ElasticsearchException error = randomBoolean() ? null : new ElasticsearchException(randomAlphaOfLength(10));
+        // add search response, minimal object is okay since the full randomization of parsing is tested in SearchResponseTests
+        SearchResponse searchResponse = randomBoolean() ? null
+                : new SearchResponse(InternalSearchResponse.empty(), randomAlphaOfLength(10), 1, 1, 0, randomIntBetween(0, 10000),
+                        ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY);
+        org.elasticsearch.xpack.core.search.action.AsyncSearchResponse testResponse =
+                new org.elasticsearch.xpack.core.search.action.AsyncSearchResponse(id, version, searchResponse, error, isPartial, isRunning,
+                        startTimeMillis, expirationTimeMillis);
+        return testResponse;
+    }
+
+    @Override
+    protected AsyncSearchResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return AsyncSearchResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(org.elasticsearch.xpack.core.search.action.AsyncSearchResponse expected, AsyncSearchResponse parsed) {
+        assertNotSame(parsed, expected);
+        assertEquals(expected.getId(), parsed.getId());
+        assertEquals(expected.getVersion(), parsed.getVersion());
+        assertEquals(expected.isRunning(), parsed.isRunning());
+        assertEquals(expected.isPartial(), parsed.isPartial());
+        assertEquals(expected.getStartTime(), parsed.getStartTime());
+        assertEquals(expected.getExpirationTime(), parsed.getExpirationTime());
+        // we cannot directly compare error since Exceptions are wrapped differently on parsing, but we can check original message
+        if (expected.getFailure() != null) {
+            assertThat(parsed.getFailure().getMessage(), containsString(expected.getFailure().getMessage()));
+        } else {
+            assertNull(parsed.getFailure());
+        }
+        // we don't need to check the complete parsed search response since this is done elsewhere
+        // only spot-check some randomized properties for equality here
+        if (expected.getSearchResponse() != null) {
+            assertEquals(expected.getSearchResponse().getTook(), parsed.getSearchResponse().getTook());
+            assertEquals(expected.getSearchResponse().getScrollId(), parsed.getSearchResponse().getScrollId());
+        } else {
+            assertNull(parsed.getSearchResponse());
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequestTests.java
new file mode 100644
index 0000000000000..f7075052cab2b
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequestTests.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+
+public class SubmitAsyncSearchRequestTests extends ESTestCase {
+
+    public void testValidation() {
+        {
+            SearchSourceBuilder source = new SearchSourceBuilder();
+            SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(source, "test");
+            Optional<ValidationException> validation = request.validate();
+            assertFalse(validation.isPresent());
+        }
+        {
+            SearchSourceBuilder source = new SearchSourceBuilder().suggest(new SuggestBuilder());
+            SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(source, "test");
+            Optional<ValidationException> validation = request.validate();
+            assertTrue(validation.isPresent());
+            assertEquals(1, validation.get().validationErrors().size());
+            assertEquals("suggest-only queries are not supported", validation.get().validationErrors().get(0));
+        }
+        {
+            SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new SearchSourceBuilder(), "test");
+            request.setKeepAlive(new TimeValue(1));
+            Optional<ValidationException> validation = request.validate();
+            assertTrue(validation.isPresent());
+            assertEquals(1, validation.get().validationErrors().size());
+            assertEquals("[keep_alive] must be greater than 1 minute, got: 1ms", validation.get().validationErrors().get(0));
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 81d61f2996ef4..9cabe14e9eed0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -260,7 +260,7 @@ public static SearchResponse fromXContent(XContentParser parser) throws IOExcept
         return innerFromXContent(parser);
     }
 
-    static SearchResponse innerFromXContent(XContentParser parser) throws IOException {
+    public static SearchResponse innerFromXContent(XContentParser parser) throws IOException {
         ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
         String currentFieldName = parser.currentName();
         SearchHits hits = null;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
index 9828635ae64fe..80d1439f80abc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
@@ -26,7 +26,9 @@ public class AsyncSearchResponse extends ActionResponse implements StatusToXCont
     @Nullable
     private final String id;
     private final int version;
+    @Nullable
     private final SearchResponse searchResponse;
+    @Nullable
     private final ElasticsearchException error;
     private final boolean isRunning;
     private final boolean isPartial;

From df79943c5215f4d62763c106106fc236d6a16744 Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 19 Mar 2020 13:15:37 -0600
Subject: [PATCH 149/285] Add IndexTemplateV2 to MetaData (#53753)

* Add IndexTemplateV2 to MetaData

This adds the `IndexTemplateV2` and `IndexTemplateV2Metadata` class to be used for the new
implementation of index templates. The new metadata is stored as a `MetaData.Custom` implementation.

Relates to #53101

* Add ITV2Metadata unit tests

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../elasticsearch/cluster/ClusterModule.java  |   5 +
 .../cluster/metadata/IndexTemplateV2.java     | 201 ++++++++++++++++++
 .../metadata/IndexTemplateV2Metadata.java     | 171 +++++++++++++++
 .../cluster/metadata/MetaData.java            |  34 +++
 .../IndexTemplateV2MetadataTests.java         |  58 +++++
 .../metadata/IndexTemplateV2Tests.java        | 188 ++++++++++++++++
 .../cluster/metadata/MetaDataTests.java       |   1 +
 .../metadata/ToAndFromJsonMetaDataTests.java  |  22 ++
 8 files changed, 680 insertions(+)
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2MetadataTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java

diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 70138ecc981bc..f52b32f3ccf67 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexTemplateV2Metadata;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
@@ -130,6 +131,8 @@ public static List<Entry> getNamedWriteables() {
             PersistentTasksCustomMetaData::readDiffFrom);
         registerMetaDataCustom(entries, ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new,
             ComponentTemplateMetadata::readDiffFrom);
+        registerMetaDataCustom(entries, IndexTemplateV2Metadata.TYPE, IndexTemplateV2Metadata::new,
+            IndexTemplateV2Metadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
@@ -150,6 +153,8 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
             PersistentTasksCustomMetaData::fromXContent));
         entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(ComponentTemplateMetadata.TYPE),
             ComponentTemplateMetadata::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexTemplateV2Metadata.TYPE),
+            IndexTemplateV2Metadata::fromXContent));
         return entries;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java
new file mode 100644
index 0000000000000..e707b84488171
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * An index template is comprised of a set of index patterns, an optional template, and a list of
+ * ids corresponding to component templates that should be composed in order when creating a new
+ * index.
+ */
+public class IndexTemplateV2 extends AbstractDiffable<IndexTemplateV2> implements ToXContentObject {
+    private static final ParseField INDEX_PATTERNS = new ParseField("index_patterns");
+    private static final ParseField TEMPLATE = new ParseField("template");
+    private static final ParseField PRIORITY = new ParseField("priority");
+    private static final ParseField COMPOSED_OF = new ParseField("composed_of");
+    private static final ParseField VERSION = new ParseField("version");
+    private static final ParseField METADATA = new ParseField("_meta");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<IndexTemplateV2, Void> PARSER = new ConstructingObjectParser<>("index_template", false,
+        a -> new IndexTemplateV2((List<String>) a[0],
+            (Template) a[1],
+            (List<String>) a[2],
+            (Long) a[3],
+            (Long) a[4],
+            (Map<String, Object>) a[5]));
+
+    static {
+        PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX_PATTERNS);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Template.PARSER, TEMPLATE);
+        PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), COMPOSED_OF);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PRIORITY);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
+    }
+
+    private final List<String> indexPatterns;
+    @Nullable
+    private final Template template;
+    @Nullable
+    private final List<String> componentTemplates;
+    @Nullable
+    private final Long priority;
+    @Nullable
+    private final Long version;
+    @Nullable
+    private final Map<String, Object> metadata;
+
+    static Diff<IndexTemplateV2> readITV2DiffFrom(StreamInput in) throws IOException {
+        return AbstractDiffable.readDiffFrom(IndexTemplateV2::new, in);
+    }
+
+    public static IndexTemplateV2 parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public IndexTemplateV2(List<String> indexPatterns, @Nullable Template template, @Nullable List<String> componentTemplates,
+                           @Nullable Long priority, @Nullable Long version, @Nullable Map<String, Object> metadata) {
+        this.indexPatterns = indexPatterns;
+        this.template = template;
+        this.componentTemplates = componentTemplates;
+        this.priority = priority;
+        this.version = version;
+        this.metadata = metadata;
+    }
+
+    public IndexTemplateV2(StreamInput in) throws IOException {
+        this.indexPatterns = in.readStringList();
+        if (in.readBoolean()) {
+            this.template = new Template(in);
+        } else {
+            this.template = null;
+        }
+        this.componentTemplates = in.readOptionalStringList();
+        this.priority = in.readOptionalVLong();
+        this.version = in.readOptionalVLong();
+        this.metadata = in.readMap();
+    }
+
+    public List<String> indexPatterns() {
+        return indexPatterns;
+    }
+
+    public Template template() {
+        return template;
+    }
+
+    public List<String> composedOf() {
+        return componentTemplates;
+    }
+
+    public Long priority() {
+        return priority;
+    }
+
+    public Long version() {
+        return version;
+    }
+
+    public Map<String, Object> metadata() {
+        return metadata;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeStringCollection(this.indexPatterns);
+        if (this.template == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            this.template.writeTo(out);
+        }
+        out.writeOptionalStringCollection(this.componentTemplates);
+        out.writeOptionalVLong(this.priority);
+        out.writeOptionalVLong(this.version);
+        out.writeMap(this.metadata);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INDEX_PATTERNS.getPreferredName(), this.indexPatterns);
+        if (this.template != null) {
+            builder.field(TEMPLATE.getPreferredName(), this.template);
+        }
+        if (this.componentTemplates != null) {
+            builder.field(COMPOSED_OF.getPreferredName(), this.componentTemplates);
+        }
+        if (this.priority != null) {
+            builder.field(PRIORITY.getPreferredName(), priority);
+        }
+        if (this.version != null) {
+            builder.field(VERSION.getPreferredName(), version);
+        }
+        if (this.metadata != null) {
+            builder.field(METADATA.getPreferredName(), metadata);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.indexPatterns, this.template, this.componentTemplates, this.priority, this.version, this.metadata);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        IndexTemplateV2 other = (IndexTemplateV2) obj;
+        return Objects.equals(this.indexPatterns, other.indexPatterns) &&
+            Objects.equals(this.template, other.template) &&
+            Objects.equals(this.componentTemplates, other.componentTemplates) &&
+            Objects.equals(this.priority, other.priority) &&
+            Objects.equals(this.version, other.version) &&
+            Objects.equals(this.metadata, other.metadata);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java
new file mode 100644
index 0000000000000..7205ec73d9b2d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * The {@link IndexTemplateV2Metadata} class is a custom {@link MetaData.Custom} implementation that
+ * stores a map of ids to {@link IndexTemplateV2} templates.
+ */
+public class IndexTemplateV2Metadata implements MetaData.Custom {
+    public static final String TYPE = "index_template";
+    private static final ParseField INDEX_TEMPLATE = new ParseField("index_template");
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<IndexTemplateV2Metadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
+        a -> new IndexTemplateV2Metadata((Map<String, IndexTemplateV2>) a[0]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
+            Map<String, IndexTemplateV2> templates = new HashMap<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                String name = p.currentName();
+                templates.put(name, IndexTemplateV2.parse(p));
+            }
+            return templates;
+        }, INDEX_TEMPLATE);
+    }
+
+    private final Map<String, IndexTemplateV2> indexTemplates;
+
+    public IndexTemplateV2Metadata(Map<String, IndexTemplateV2> templates) {
+        this.indexTemplates = templates;
+    }
+
+    public IndexTemplateV2Metadata(StreamInput in) throws IOException {
+        this.indexTemplates = in.readMap(StreamInput::readString, IndexTemplateV2::new);
+    }
+
+    public static IndexTemplateV2Metadata fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public Map<String, IndexTemplateV2> indexTemplates() {
+        return indexTemplates;
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new IndexTemplateV2MetadataDiff((IndexTemplateV2Metadata) before, this);
+    }
+
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new IndexTemplateV2MetadataDiff(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        // TODO: update this once backported
+        return Version.V_8_0_0;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(this.indexTemplates, StreamOutput::writeString, (outstream, val) -> val.writeTo(outstream));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(INDEX_TEMPLATE.getPreferredName());
+        for (Map.Entry<String, IndexTemplateV2> template : indexTemplates.entrySet()) {
+            builder.field(template.getKey(), template.getValue());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.indexTemplates);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        IndexTemplateV2Metadata other = (IndexTemplateV2Metadata) obj;
+        return Objects.equals(this.indexTemplates, other.indexTemplates);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    static class IndexTemplateV2MetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, IndexTemplateV2>> indexTemplateDiff;
+
+        IndexTemplateV2MetadataDiff(IndexTemplateV2Metadata before, IndexTemplateV2Metadata after) {
+            this.indexTemplateDiff = DiffableUtils.diff(before.indexTemplates, after.indexTemplates,
+                DiffableUtils.getStringKeySerializer());
+        }
+
+        IndexTemplateV2MetadataDiff(StreamInput in) throws IOException {
+            this.indexTemplateDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                IndexTemplateV2::new, IndexTemplateV2::readITV2DiffFrom);
+        }
+
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            return new IndexTemplateV2Metadata(indexTemplateDiff.apply(((IndexTemplateV2Metadata) part).indexTemplates));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            indexTemplateDiff.writeTo(out);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 6154e6ecd675f..8471364b07dea 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -668,6 +668,12 @@ public Map<String, ComponentTemplate> componentTemplates() {
             .orElse(Collections.emptyMap());
     }
 
+    public Map<String, IndexTemplateV2> templatesV2() {
+        return Optional.ofNullable((IndexTemplateV2Metadata) this.custom(IndexTemplateV2Metadata.TYPE))
+            .map(IndexTemplateV2Metadata::indexTemplates)
+            .orElse(Collections.emptyMap());
+    }
+
     public ImmutableOpenMap<String, Custom> customs() {
         return this.customs;
     }
@@ -1087,6 +1093,34 @@ public Builder componentTemplates(Map<String, ComponentTemplate> componentTempla
             return this;
         }
 
+        public Builder indexTemplates(Map<String, IndexTemplateV2> indexTemplates) {
+            this.customs.put(IndexTemplateV2Metadata.TYPE, new IndexTemplateV2Metadata(indexTemplates));
+            return this;
+        }
+
+        public Builder put(String name, IndexTemplateV2 indexTemplate) {
+            Objects.requireNonNull(indexTemplate, "it is invalid to add a null index template: " + name);
+            // ಠ_ಠ at ImmutableOpenMap
+            Map<String, IndexTemplateV2> existingTemplates =
+                Optional.ofNullable((IndexTemplateV2Metadata) this.customs.get(IndexTemplateV2Metadata.TYPE))
+                    .map(itmd -> new HashMap<>(itmd.indexTemplates()))
+                    .orElse(new HashMap<>());
+            existingTemplates.put(name, indexTemplate);
+            this.customs.put(IndexTemplateV2Metadata.TYPE, new IndexTemplateV2Metadata(existingTemplates));
+            return this;
+        }
+
+        public Builder removeIndexTemplate(String name) {
+            // ಠ_ಠ at ImmutableOpenMap
+            Map<String, IndexTemplateV2> existingTemplates =
+                Optional.ofNullable((IndexTemplateV2Metadata) this.customs.get(IndexTemplateV2Metadata.TYPE))
+                    .map(itmd -> new HashMap<>(itmd.indexTemplates()))
+                    .orElse(new HashMap<>());
+            existingTemplates.remove(name);
+            this.customs.put(IndexTemplateV2Metadata.TYPE, new IndexTemplateV2Metadata(existingTemplates));
+            return this;
+        }
+
         public Custom getCustom(String type) {
             return customs.get(type);
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2MetadataTests.java
new file mode 100644
index 0000000000000..b479ef96cb5ff
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2MetadataTests.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class IndexTemplateV2MetadataTests extends AbstractNamedWriteableTestCase<IndexTemplateV2Metadata> {
+    @Override
+    protected IndexTemplateV2Metadata createTestInstance() {
+        if (randomBoolean()) {
+            return new IndexTemplateV2Metadata(Collections.emptyMap());
+        }
+        Map<String, IndexTemplateV2> templates = new HashMap<>();
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            templates.put(randomAlphaOfLength(5), IndexTemplateV2Tests.randomInstance());
+        }
+        return new IndexTemplateV2Metadata(templates);
+    }
+
+    @Override
+    protected IndexTemplateV2Metadata mutateInstance(IndexTemplateV2Metadata instance) throws IOException {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.singletonList(new NamedWriteableRegistry.Entry(IndexTemplateV2Metadata.class,
+            IndexTemplateV2Metadata.TYPE, IndexTemplateV2Metadata::new)));
+    }
+
+    @Override
+    protected Class<IndexTemplateV2Metadata> categoryClass() {
+        return IndexTemplateV2Metadata.class;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java
new file mode 100644
index 0000000000000..bafb28a0c295c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase<IndexTemplateV2> {
+    @Override
+    protected IndexTemplateV2 makeTestChanges(IndexTemplateV2 testInstance) {
+        try {
+            return mutateInstance(testInstance);
+        } catch (IOException e) {
+            logger.error(e);
+            fail("mutating should not throw an exception, but got: " + e);
+            return null;
+        }
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<IndexTemplateV2>> diffReader() {
+        return IndexTemplateV2::readITV2DiffFrom;
+    }
+
+    @Override
+    protected IndexTemplateV2 doParseInstance(XContentParser parser) throws IOException {
+        return IndexTemplateV2.parse(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<IndexTemplateV2> instanceReader() {
+        return IndexTemplateV2::new;
+    }
+
+    @Override
+    protected IndexTemplateV2 createTestInstance() {
+        return randomInstance();
+    }
+
+    public static IndexTemplateV2 randomInstance() {
+        Settings settings = null;
+        CompressedXContent mappings = null;
+        Map<String, AliasMetaData> aliases = null;
+        Template template = null;
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                settings = randomSettings();
+            }
+            if (randomBoolean()) {
+                mappings = randomMappings();
+            }
+            if (randomBoolean()) {
+                aliases = randomAliases();
+            }
+            template = new Template(settings, mappings, aliases);
+        }
+
+        Map<String, Object> meta = null;
+        if (randomBoolean()) {
+            meta = randomMeta();
+        }
+
+        List<String> indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4));
+        List<String> componentTemplates = randomList(0, 10, () -> randomAlphaOfLength(5));
+        return new IndexTemplateV2(indexPatterns,
+            template,
+            componentTemplates,
+            randomBoolean() ? null : randomNonNegativeLong(),
+            randomBoolean() ? null : randomNonNegativeLong(),
+            meta);
+    }
+
+    private static Map<String, AliasMetaData> randomAliases() {
+        String aliasName = randomAlphaOfLength(5);
+        AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
+            .filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
+            .routing(randomBoolean() ? null : randomAlphaOfLength(3))
+            .isHidden(randomBoolean() ? null : randomBoolean())
+            .writeIndex(randomBoolean() ? null : randomBoolean())
+            .build();
+        return Collections.singletonMap(aliasName, aliasMeta);
+    }
+
+    private static CompressedXContent randomMappings() {
+        try {
+            return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}");
+        } catch (IOException e) {
+            fail("got an IO exception creating fake mappings: " + e);
+            return null;
+        }
+    }
+
+    private static Settings randomSettings() {
+        return Settings.builder()
+            .put(randomAlphaOfLength(4), randomAlphaOfLength(10))
+            .build();
+    }
+
+    private static Map<String, Object> randomMeta() {
+        if (randomBoolean()) {
+            return Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4));
+        } else {
+            return Collections.singletonMap(randomAlphaOfLength(5),
+                Collections.singletonMap(randomAlphaOfLength(4), randomAlphaOfLength(4)));
+        }
+    }
+
+    @Override
+    protected IndexTemplateV2 mutateInstance(IndexTemplateV2 orig) throws IOException {
+        return mutateTemplate(orig);
+    }
+
+    public static IndexTemplateV2 mutateTemplate(IndexTemplateV2 orig) {
+        switch (randomIntBetween(0, 5)) {
+            case 0:
+                List<String> newIndexPatterns = randomValueOtherThan(orig.indexPatterns(),
+                    () -> randomList(1, 4, () -> randomAlphaOfLength(4)));
+                return new IndexTemplateV2(newIndexPatterns, orig.template(), orig.composedOf(),
+                    orig.priority(), orig.version(), orig.metadata());
+            case 1:
+                return new IndexTemplateV2(orig.indexPatterns(),
+                    randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), randomMappings(), randomAliases())),
+                    orig.composedOf(),
+                    orig.priority(),
+                    orig.version(),
+                    orig.metadata());
+            case 2:
+                List<String> newComposedOf = randomValueOtherThan(orig.composedOf(),
+                    () -> randomList(0, 10, () -> randomAlphaOfLength(5)));
+                return new IndexTemplateV2(orig.indexPatterns(),
+                    orig.template(),
+                    newComposedOf,
+                    orig.priority(),
+                    orig.version(),
+                    orig.metadata());
+            case 3:
+                return new IndexTemplateV2(orig.indexPatterns(),
+                    orig.template(),
+                    orig.composedOf(),
+                    randomValueOtherThan(orig.priority(), ESTestCase::randomNonNegativeLong),
+                    orig.version(),
+                    orig.metadata());
+            case 4:
+                return new IndexTemplateV2(orig.indexPatterns(),
+                    orig.template(),
+                    orig.composedOf(),
+                    orig.priority(),
+                    randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong),
+                    orig.metadata());
+            case 5:
+                return new IndexTemplateV2(orig.indexPatterns(),
+                    orig.template(),
+                    orig.composedOf(),
+                    orig.priority(),
+                    orig.version(),
+                    randomValueOtherThan(orig.metadata(), IndexTemplateV2Tests::randomMeta));
+            default:
+                throw new IllegalStateException("illegal randomization branch");
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 1d0bd3e36c57b..bd2e48a20c2e4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -938,6 +938,7 @@ public static MetaData randomMetaData() {
             .indexGraveyard(IndexGraveyardTests.createRandom())
             .version(randomNonNegativeLong())
             .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
+            .put("index_template_v2_" + randomAlphaOfLength(3), IndexTemplateV2Tests.randomInstance())
             .build();
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index 7257a21b15900..d5e76c97e1ad4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 
 import static org.elasticsearch.cluster.metadata.AliasMetaData.newAliasMetaDataBuilder;
@@ -122,6 +123,14 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         new CompressedXContent("{\"baz\":\"eggplant\"}"),
                         Collections.singletonMap("alias", AliasMetaData.builder("alias").build())),
                     5L, Collections.singletonMap("my_meta", Collections.singletonMap("foo", "bar"))))
+                .put("index_templatev2", new IndexTemplateV2(Arrays.asList("foo", "bar*"),
+                    new Template(Settings.builder().put("setting", "value").build(),
+                        new CompressedXContent("{\"baz\":\"eggplant\"}"),
+                        Collections.singletonMap("alias", AliasMetaData.builder("alias").build())),
+                    Collections.singletonList("component_template"),
+                    5L,
+                    4L,
+                    Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken"))))
                 .put(IndexMetaData.builder("test12")
                         .settings(settings(Version.CURRENT)
                                 .put("setting1", "value1")
@@ -301,6 +310,19 @@ public void testSimpleJsonFromAndTo() throws IOException {
             equalTo(new Template(Settings.builder().put("setting", "value").build(),
                 new CompressedXContent("{\"baz\":\"eggplant\"}"),
                 Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
+
+        // index template v2
+        assertNotNull(parsedMetaData.templatesV2().get("index_templatev2"));
+        assertThat(parsedMetaData.templatesV2().get("index_templatev2").priority(), is(5L));
+        assertThat(parsedMetaData.templatesV2().get("index_templatev2").version(), is(4L));
+        assertThat(parsedMetaData.templatesV2().get("index_templatev2").indexPatterns(), is(Arrays.asList("foo", "bar*")));
+        assertThat(parsedMetaData.templatesV2().get("index_templatev2").composedOf(), is(Collections.singletonList("component_template")));
+        assertThat(parsedMetaData.templatesV2().get("index_templatev2").metadata(),
+            equalTo(Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken"))));
+        assertThat(parsedMetaData.templatesV2().get("index_templatev2").template(),
+            equalTo(new Template(Settings.builder().put("setting", "value").build(),
+                new CompressedXContent("{\"baz\":\"eggplant\"}"),
+                Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
     }
 
     private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";

From 7d77c2a64dbf5dcf96031f92028bd3dbe3820363 Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 19 Mar 2020 15:03:53 -0600
Subject: [PATCH 150/285] Disable BWC tests for IndexTemplateV2Metadata
 backport (#53830)

Relates to #53829
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index fa1ebcf6706f8..295717bac44b4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53829" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 7485b72f7b73496facd2c58c96cdf333259b56dd Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 19 Mar 2020 15:04:21 -0600
Subject: [PATCH 151/285] Update minimally supported version for
 IndexTemplateV2Metadata (#53829)

To be merged after #53827 and after bwc tests have been disabled
---
 .../cluster/metadata/IndexTemplateV2Metadata.java              | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java
index 7205ec73d9b2d..42052d735b29e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Metadata.java
@@ -98,8 +98,7 @@ public String getWriteableName() {
 
     @Override
     public Version getMinimalSupportedVersion() {
-        // TODO: update this once backported
-        return Version.V_8_0_0;
+        return Version.V_7_7_0;
     }
 
     @Override

From c593c8b9a2ec78d0037d8e3b4153fc8b1ed3ff8c Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Thu, 19 Mar 2020 14:39:50 -0700
Subject: [PATCH 152/285] Use boolean methods for allowed realm types in
 license state (#53456)

In xpack the license state contains methods to determine whether a
particular feature is allowed to be used. The one exception is
allowsRealmTypes() which returns an enum of the types of realms allowed.
This change converts the enum values to boolean methods. There are 2
notable changes: NONE is removed as we always fall back to basic license
behavior, and NATIVE is not needed because it would always return true
since we should always have a basic license.
---
 .../license/XPackLicenseState.java            | 37 ++--------
 .../license/XPackLicenseStateTests.java       | 17 ++---
 .../xpack/security/authc/Realms.java          | 72 +++++++------------
 .../RestDelegatePkiAuthenticationAction.java  |  3 +-
 .../oidc/OpenIdConnectBaseRestHandler.java    |  2 +-
 .../rest/action/saml/SamlBaseRestHandler.java |  2 +-
 .../authc/AuthenticationServiceTests.java     |  3 +-
 .../xpack/security/authc/RealmsTests.java     | 53 +++++++-------
 8 files changed, 66 insertions(+), 123 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index 4e58023eb260b..3b80e605a2931 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -402,8 +402,7 @@ public boolean isActive() {
     }
 
     /**
-     * @return true if authentication and authorization should be enabled. this does not indicate what realms are available
-     * @see #allowedRealmType() for the enabled realms
+     * @return true if authentication and authorization should be enabled.
      */
     public boolean isAuthAllowed() {
         return isAllowedBySecurityAndLicense(OperationMode.BASIC, false, true);
@@ -438,38 +437,12 @@ public boolean isDocumentAndFieldLevelSecurityAllowed() {
         return isAllowedBySecurityAndLicense(OperationMode.PLATINUM, false, true);
     }
 
-    /** Classes of realms that may be available based on the license type. */
-    public enum AllowedRealmType {
-        NONE,
-        NATIVE,
-        DEFAULT,
-        ALL
+    public boolean areAllRealmsAllowed() {
+        return isAllowedBySecurityAndLicense(OperationMode.PLATINUM, false, true);
     }
 
-    /**
-     * @return the type of realms that are enabled based on the license {@link OperationMode}
-     */
-    public AllowedRealmType allowedRealmType() {
-        return executeAgainstStatus(status -> {
-            final boolean isSecurityCurrentlyEnabled = isSecurityEnabled(status.mode, isSecurityExplicitlyEnabled, isSecurityEnabled);
-            if (isSecurityCurrentlyEnabled) {
-                switch (status.mode) {
-                    case PLATINUM:
-                    case ENTERPRISE:
-                    case TRIAL:
-                        return AllowedRealmType.ALL;
-                    case GOLD:
-                        return AllowedRealmType.DEFAULT;
-                    case BASIC:
-                    case STANDARD:
-                        return AllowedRealmType.NATIVE;
-                    default:
-                        return AllowedRealmType.NONE;
-                }
-            } else {
-                return AllowedRealmType.NONE;
-            }
-        });
+    public boolean areStandardRealmsAllowed() {
+        return isAllowedBySecurityAndLicense(OperationMode.GOLD, false, true);
     }
 
     public boolean isCustomRoleProvidersAllowed() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
index ce1345d4941a8..b08f9e45c8c52 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
@@ -82,7 +82,7 @@ public void testSecurityDefaults() {
         assertThat(licenseState.isAuditingAllowed(), is(true));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
+        assertThat(licenseState.areAllRealmsAllowed(), is(true));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(true));
 
         licenseState = new XPackLicenseState(Settings.EMPTY);
@@ -105,7 +105,6 @@ public void testSecurityBasicWithoutExplicitSecurityEnabled() {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NONE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(false));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(false));
@@ -124,7 +123,6 @@ public void testSecurityBasicWithExplicitSecurityEnabled() {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NATIVE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(false));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
@@ -142,7 +140,6 @@ public void testSecurityDefaultBasicExpired() {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(false));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NONE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(false));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(false));
@@ -158,7 +155,6 @@ public void testSecurityEnabledBasicExpired() {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(false));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NATIVE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(false));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
@@ -174,7 +170,6 @@ public void testSecurityStandard() {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NATIVE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
     }
 
@@ -188,7 +183,6 @@ public void testSecurityStandardExpired() {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(false));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NATIVE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
     }
 
@@ -202,7 +196,7 @@ public void testSecurityGold() {
         assertThat(licenseState.isAuditingAllowed(), is(true));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.DEFAULT));
+        assertThat(licenseState.areStandardRealmsAllowed(), is(true));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(true));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
@@ -218,7 +212,7 @@ public void testSecurityGoldExpired() {
         assertThat(licenseState.isAuditingAllowed(), is(true));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(false));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.DEFAULT));
+        assertThat(licenseState.areStandardRealmsAllowed(), is(true));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(true));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
@@ -234,7 +228,7 @@ public void testSecurityPlatinum() {
         assertThat(licenseState.isAuditingAllowed(), is(true));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
+        assertThat(licenseState.areAllRealmsAllowed(), is(true));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(true));
         assertThat(licenseState.isTokenServiceAllowed(), is(true));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
@@ -250,7 +244,7 @@ public void testSecurityPlatinumExpired() {
         assertThat(licenseState.isAuditingAllowed(), is(true));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(false));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
+        assertThat(licenseState.areAllRealmsAllowed(), is(true));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
         assertThat(licenseState.isTokenServiceAllowed(), is(true));
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
@@ -270,7 +264,6 @@ private void assertSecurityNotAllowed(XPackLicenseState licenseState) {
         assertThat(licenseState.isAuditingAllowed(), is(false));
         assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
-        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NONE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
index 482775cf5b613..4e00eb3b70473 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.license.XPackLicenseState.AllowedRealmType;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
@@ -102,35 +101,21 @@ public Realms(Settings settings, Environment env, Map<String, Realm.Factory> fac
 
     @Override
     public Iterator<Realm> iterator() {
-        if (licenseState.isAuthAllowed() == false) {
-            return Collections.emptyIterator();
-        }
-
-        AllowedRealmType allowedRealmType = licenseState.allowedRealmType();
-        switch (allowedRealmType) {
-            case ALL:
-                return realms.iterator();
-            case DEFAULT:
-                return standardRealmsOnly.iterator();
-            case NATIVE:
-                return nativeRealmsOnly.iterator();
-            default:
-                throw new IllegalStateException("authentication should not be enabled");
-        }
+        return asList().iterator();
     }
 
     /**
      * Returns a list of realms that are configured, but are not permitted under the current license.
      */
     public List<Realm> getUnlicensedRealms() {
+        final XPackLicenseState licenseStateSnapshot = licenseState.copyCurrentLicenseState();
         // If auth is not allowed, then everything is unlicensed
-        if (licenseState.isAuthAllowed() == false) {
+        if (licenseStateSnapshot.isAuthAllowed() == false) {
             return Collections.unmodifiableList(realms);
         }
 
-        AllowedRealmType allowedRealmType = licenseState.allowedRealmType();
         // If all realms are allowed, then nothing is unlicensed
-        if (allowedRealmType == AllowedRealmType.ALL) {
+        if (licenseStateSnapshot.areAllRealmsAllowed()) {
             return Collections.emptyList();
         }
 
@@ -150,20 +135,17 @@ public Stream<Realm> stream() {
     }
 
     public List<Realm> asList() {
-        if (licenseState.isAuthAllowed() == false) {
+        final XPackLicenseState licenseStateSnapshot = licenseState.copyCurrentLicenseState();
+        if (licenseStateSnapshot.isAuthAllowed() == false) {
             return Collections.emptyList();
         }
-
-        AllowedRealmType allowedRealmType = licenseState.allowedRealmType();
-        switch (allowedRealmType) {
-            case ALL:
-                return Collections.unmodifiableList(realms);
-            case DEFAULT:
-                return Collections.unmodifiableList(standardRealmsOnly);
-            case NATIVE:
-                return Collections.unmodifiableList(nativeRealmsOnly);
-            default:
-                throw new IllegalStateException("authentication should not be enabled");
+        if (licenseStateSnapshot.areAllRealmsAllowed()) {
+            return realms;
+        } else if (licenseStateSnapshot.areStandardRealmsAllowed()) {
+            return standardRealmsOnly;
+        } else {
+            // native realms are basic licensed, and always allowed, even for an expired license
+            return nativeRealmsOnly;
         }
     }
 
@@ -241,19 +223,20 @@ protected List<Realm> initRealms() throws Exception {
         if (Strings.hasText(duplicateRealms)) {
             throw new IllegalArgumentException("Found multiple realms configured with the same name: " + duplicateRealms + "");
         }
-        return realms;
+        return Collections.unmodifiableList(realms);
     }
 
     public void usageStats(ActionListener<Map<String, Object>> listener) {
+        final XPackLicenseState licenseStateSnapshot = licenseState.copyCurrentLicenseState();
         Map<String, Object> realmMap = new HashMap<>();
         final AtomicBoolean failed = new AtomicBoolean(false);
         final List<Realm> realmList = asList().stream()
             .filter(r -> ReservedRealm.TYPE.equals(r.type()) == false)
             .collect(Collectors.toList());
+        final Set<String> realmTypes = realmList.stream().map(Realm::type).collect(Collectors.toSet());
         final CountDown countDown = new CountDown(realmList.size());
         final Runnable doCountDown = () -> {
             if ((realmList.isEmpty() || countDown.countDown()) && failed.get() == false) {
-                final AllowedRealmType allowedRealmType = licenseState.allowedRealmType();
                 // iterate over the factories so we can add enabled & available info
                 for (String type : factories.keySet()) {
                     assert ReservedRealm.TYPE.equals(type) == false;
@@ -261,15 +244,13 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                         if (value == null) {
                             return MapBuilder.<String, Object>newMapBuilder()
                                 .put("enabled", false)
-                                .put("available", isRealmTypeAvailable(allowedRealmType, type))
+                                .put("available", isRealmTypeAvailable(licenseStateSnapshot, type))
                                 .map();
                         }
 
                         assert value instanceof Map;
                         Map<String, Object> realmTypeUsage = (Map<String, Object>) value;
                         realmTypeUsage.put("enabled", true);
-                        // the realms iterator returned this type so it must be enabled
-                        assert isRealmTypeAvailable(allowedRealmType, type);
                         realmTypeUsage.put("available", true);
                         return value;
                     });
@@ -363,18 +344,13 @@ private static Map<String, Object> convertToMapOfLists(Map<String, Object> map)
         return converted;
     }
 
-    public static boolean isRealmTypeAvailable(AllowedRealmType enabledRealmType, String type) {
-        switch (enabledRealmType) {
-            case ALL:
-                return true;
-            case NONE:
-                return false;
-            case NATIVE:
-                return FileRealmSettings.TYPE.equals(type) || NativeRealmSettings.TYPE.equals(type);
-            case DEFAULT:
-                return InternalRealms.isStandardRealm(type) || ReservedRealm.TYPE.equals(type);
-            default:
-                throw new IllegalStateException("unknown enabled realm type [" + enabledRealmType + "]");
+    public static boolean isRealmTypeAvailable(XPackLicenseState licenseState, String type) {
+        if (licenseState.areAllRealmsAllowed()) {
+            return true;
+        } else if (licenseState.areStandardRealmsAllowed()) {
+            return InternalRealms.isStandardRealm(type) || ReservedRealm.TYPE.equals(type);
+        } else {
+            return FileRealmSettings.TYPE.equals(type) || NativeRealmSettings.TYPE.equals(type);
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestDelegatePkiAuthenticationAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestDelegatePkiAuthenticationAction.java
index 64a4fcc11208d..870f6e469d54e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestDelegatePkiAuthenticationAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestDelegatePkiAuthenticationAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationResponse;
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
 import org.elasticsearch.xpack.security.action.TransportDelegatePkiAuthenticationAction;
-import org.elasticsearch.xpack.security.authc.Realms;
 
 import java.io.IOException;
 import java.util.List;
@@ -54,7 +53,7 @@ protected Exception checkFeatureAvailable(RestRequest request) {
         Exception failedFeature = super.checkFeatureAvailable(request);
         if (failedFeature != null) {
             return failedFeature;
-        } else if (Realms.isRealmTypeAvailable(licenseState.allowedRealmType(), PkiRealmSettings.TYPE)) {
+        } else if (licenseState.areStandardRealmsAllowed()) {
             return null;
         } else {
             logger.info("The '{}' realm is not available under the current license", PkiRealmSettings.TYPE);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oidc/OpenIdConnectBaseRestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oidc/OpenIdConnectBaseRestHandler.java
index 3fe31b97f59b9..c0eec495ead40 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oidc/OpenIdConnectBaseRestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oidc/OpenIdConnectBaseRestHandler.java
@@ -33,7 +33,7 @@ protected Exception checkFeatureAvailable(RestRequest request) {
         Exception failedFeature = super.checkFeatureAvailable(request);
         if (failedFeature != null) {
             return failedFeature;
-        } else if (Realms.isRealmTypeAvailable(licenseState.allowedRealmType(), OIDC_REALM_TYPE)) {
+        } else if (Realms.isRealmTypeAvailable(licenseState, OIDC_REALM_TYPE)) {
             return null;
         } else {
             logger.info("The '{}' realm is not available under the current license", OIDC_REALM_TYPE);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandler.java
index e30d844bb5644..4ab8b3604ada9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandler.java
@@ -32,7 +32,7 @@ protected Exception checkFeatureAvailable(RestRequest request) {
         Exception failedFeature = super.checkFeatureAvailable(request);
         if (failedFeature != null) {
             return failedFeature;
-        } else if (Realms.isRealmTypeAvailable(licenseState.allowedRealmType(), SAML_REALM_TYPE)) {
+        } else if (Realms.isRealmTypeAvailable(licenseState, SAML_REALM_TYPE)) {
             return null;
         } else {
             logger.info("The '{}' realm is not available under the current license", SAML_REALM_TYPE);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index e8bb282d6de4a..e616b07c3f54f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -188,10 +188,11 @@ public void init() throws Exception {
             .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)
             .build();
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
-        when(licenseState.allowedRealmType()).thenReturn(XPackLicenseState.AllowedRealmType.ALL);
+        when(licenseState.areAllRealmsAllowed()).thenReturn(true);
         when(licenseState.isAuthAllowed()).thenReturn(true);
         when(licenseState.isApiKeyServiceAllowed()).thenReturn(true);
         when(licenseState.isTokenServiceAllowed()).thenReturn(true);
+        when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
         when(licenseState.isAuditingAllowed()).thenReturn(true);
         ReservedRealm reservedRealm = mock(ReservedRealm.class);
         when(reservedRealm.type()).thenReturn("reserved");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
index 2a6891ab5dedc..ea29cef8577da 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.license.XPackLicenseState.AllowedRealmType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
@@ -71,14 +70,30 @@ public void init() throws Exception {
             factories.put(name, config -> new DummyRealm(name, config));
         }
         licenseState = mock(XPackLicenseState.class);
+        when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
         threadContext = new ThreadContext(Settings.EMPTY);
         reservedRealm = mock(ReservedRealm.class);
         when(licenseState.isAuthAllowed()).thenReturn(true);
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.ALL);
+        allowAllRealms();
         when(reservedRealm.type()).thenReturn(ReservedRealm.TYPE);
         when(reservedRealm.name()).thenReturn("reserved");
     }
 
+    private void allowAllRealms() {
+        when(licenseState.areAllRealmsAllowed()).thenReturn(true);
+        when(licenseState.areStandardRealmsAllowed()).thenReturn(true);
+    }
+
+    private void allowOnlyStandardRealms() {
+        when(licenseState.areAllRealmsAllowed()).thenReturn(false);
+        when(licenseState.areStandardRealmsAllowed()).thenReturn(true);
+    }
+
+    private void allowOnlyNativeRealms() {
+        when(licenseState.areAllRealmsAllowed()).thenReturn(false);
+        when(licenseState.areStandardRealmsAllowed()).thenReturn(false);
+    }
+
     public void testWithSettings() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
@@ -225,7 +240,7 @@ public void testUnlicensedWithOnlyCustomRealms() throws Exception {
         assertThat(realms.getUnlicensedRealms(), empty());
         assertThat(realms.getUnlicensedRealms(), sameInstance(realms.getUnlicensedRealms()));
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.DEFAULT);
+        allowOnlyNativeRealms();
 
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
@@ -253,7 +268,7 @@ public void testUnlicensedWithOnlyCustomRealms() throws Exception {
             i++;
         }
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NATIVE);
+        allowOnlyNativeRealms();
 
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
@@ -309,7 +324,7 @@ public void testUnlicensedWithInternalRealms() throws Exception {
         assertThat(realms.getUnlicensedRealms(), empty());
         assertThat(realms.getUnlicensedRealms(), sameInstance(realms.getUnlicensedRealms()));
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.DEFAULT);
+        allowOnlyStandardRealms();
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
         realm = iter.next();
@@ -327,7 +342,7 @@ public void testUnlicensedWithInternalRealms() throws Exception {
         assertThat(realm.type(), equalTo("type_0"));
         assertThat(realm.name(), equalTo("custom"));
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NATIVE);
+        allowOnlyNativeRealms();
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
         realm = iter.next();
@@ -374,7 +389,7 @@ public void testUnlicensedWithNativeRealmSettings() throws Exception {
         assertThat(iter.hasNext(), is(false));
         assertThat(realms.getUnlicensedRealms(), empty());
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NATIVE);
+        allowOnlyNativeRealms();
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
         realm = iter.next();
@@ -409,7 +424,7 @@ public void testUnlicensedWithNonStandardRealms() throws Exception {
         assertThat(iter.hasNext(), is(false));
         assertThat(realms.getUnlicensedRealms(), empty());
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.DEFAULT);
+        allowOnlyStandardRealms();
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
         realm = iter.next();
@@ -427,7 +442,7 @@ public void testUnlicensedWithNonStandardRealms() throws Exception {
         assertThat(realm.type(), equalTo(selectedRealmType));
         assertThat(realm.name(), equalTo("foo"));
 
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NATIVE);
+        allowOnlyNativeRealms();
         iter = realms.iterator();
         assertThat(iter.hasNext(), is(true));
         realm = iter.next();
@@ -497,7 +512,7 @@ public void testDisabledRealmsAreNotAdded() throws Exception {
         assertThat(realms.getUnlicensedRealms(), empty());
 
         // check that disabled realms are not included in unlicensed realms
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NATIVE);
+        allowOnlyNativeRealms();
         assertThat(realms.getUnlicensedRealms(), hasSize(orderToIndex.size()));
     }
 
@@ -550,23 +565,9 @@ public void testUsageStats() throws Exception {
             assertThat(typeMap.size(), is(2));
         }
 
-        // disable ALL using license
-        when(licenseState.isAuthAllowed()).thenReturn(false);
-        when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NONE);
-        future = new PlainActionFuture<>();
-        realms.usageStats(future);
-        usageStats = future.get();
-        assertThat(usageStats.size(), is(factories.size()));
-        for (Entry<String, Object> entry : usageStats.entrySet()) {
-            Map<String, Object> typeMap = (Map<String, Object>) entry.getValue();
-            assertThat(typeMap, hasEntry("enabled", false));
-            assertThat(typeMap, hasEntry("available", false));
-            assertThat(typeMap.size(), is(2));
-        }
-
-        // check native or internal realms enabled only
+        // check standard realms include native
         when(licenseState.isAuthAllowed()).thenReturn(true);
-        when(licenseState.allowedRealmType()).thenReturn(randomFrom(AllowedRealmType.NATIVE, AllowedRealmType.DEFAULT));
+        allowOnlyStandardRealms();
         future = new PlainActionFuture<>();
         realms.usageStats(future);
         usageStats = future.get();

From b873d8eaa3a4469b3c179eacfcd37224d7e81e05 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Thu, 19 Mar 2020 15:03:47 -0700
Subject: [PATCH 153/285] Make lucene snapshot repository exclusive to lucene
 packages (#53353)

When depending on lucene snapshots we point maven at our own s3 backed
repository. However, in this case lucene packages should only be
retrieved from this location, and no other packages should ever be found
in that repo. This commit makes the maven repository exclusive to lucene
packages.
---
 .../groovy/org/elasticsearch/gradle/BuildPlugin.groovy   | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 8b8c088c86244..be8d69da60c4b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -46,6 +46,7 @@ import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.ModuleDependency
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.artifacts.repositories.ExclusiveContentRepository
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository
@@ -327,10 +328,16 @@ class BuildPlugin implements Plugin<Project> {
             // extract the revision number from the version with a regex matcher
             List<String> matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List<String>
             String revision = matches.get(1)
-            repos.maven { MavenArtifactRepository repo ->
+            MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo ->
                 repo.name = 'lucene-snapshots'
                 repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}"
             }
+            repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo ->
+                exclusiveRepo.filter {
+                    it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}")
+                }
+                exclusiveRepo.forRepositories(luceneRepo)
+            }
         }
     }
 

From c2ae78dbba263876c5d5b4521401ab6439b1b998 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Thu, 19 Mar 2020 15:05:44 -0700
Subject: [PATCH 154/285] Fix Joda compatibility in stream protocol (#53823)

The JodaCompatibleZonedDateTime is a compatibility object that unions
Joda's DateTime and Java's ZonedDateTime, meant for use in scripts. When
it was added, we serialized the JCZDT as a Joda DateTime so that when
sending to older nodes they could still read the object. However, on
newer nodes, we continued also reading this as a Joda DateTime. This
commit changes the read side to form a JCZDT.

closes #53586
---
 .../elasticsearch/common/io/stream/StreamInput.java  | 12 ++++++++----
 .../common/io/stream/BytesStreamsTests.java          | 10 ++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 300b42153dc47..d30b226dfb2dc 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -38,9 +38,10 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.joda.time.DateTime;
+import org.elasticsearch.script.JodaCompatibleZonedDateTime;
 import org.joda.time.DateTimeZone;
 
 import java.io.ByteArrayInputStream;
@@ -768,9 +769,12 @@ private List<Object> readArrayList() throws IOException {
         return list;
     }
 
-    private DateTime readDateTime() throws IOException {
-        final String timeZoneId = readString();
-        return new DateTime(readLong(), DateTimeZone.forID(timeZoneId));
+    private JodaCompatibleZonedDateTime readDateTime() throws IOException {
+        // we reuse DateTime to communicate with older nodes that don't know about the joda compat layer, but
+        // here we are on a new node so we always want a compat datetime
+        final ZoneId zoneId = DateUtils.dateTimeZoneToZoneId(DateTimeZone.forID(readString()));
+        long millis = readLong();
+        return new JodaCompatibleZonedDateTime(Instant.ofEpochMilli(millis), zoneId);
     }
 
     private ZonedDateTime readZonedDateTime() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index aad01d56f2496..94e2801df8cee 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -27,11 +27,14 @@
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.script.JodaCompatibleZonedDateTime;
 import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
 import java.io.EOFException;
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -49,6 +52,7 @@
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -303,6 +307,7 @@ public void testSimpleStreams() throws Exception {
         out.writeTimeZone(DateTimeZone.forID("CET"));
         out.writeOptionalTimeZone(DateTimeZone.getDefault());
         out.writeOptionalTimeZone(null);
+        out.writeGenericValue(new DateTime(123456, DateTimeZone.forID("America/Los_Angeles")));
         final byte[] bytes = BytesReference.toBytes(out.bytes());
         StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
         assertEquals(in.available(), bytes.length);
@@ -335,6 +340,11 @@ public void testSimpleStreams() throws Exception {
         assertEquals(DateTimeZone.forID("CET"), in.readTimeZone());
         assertEquals(DateTimeZone.getDefault(), in.readOptionalTimeZone());
         assertNull(in.readOptionalTimeZone());
+        Object dt = in.readGenericValue();
+        assertThat(dt, instanceOf(JodaCompatibleZonedDateTime.class));
+        JodaCompatibleZonedDateTime jdt = (JodaCompatibleZonedDateTime) dt;
+        assertThat(jdt.getZonedDateTime().toInstant().toEpochMilli(), equalTo(123456L));
+        assertThat(jdt.getZonedDateTime().getZone(), equalTo(ZoneId.of("America/Los_Angeles")));
         assertEquals(0, in.available());
         in.close();
         out.close();

From 0e6d139b12da3385943cbfbe6b97cfef716e5883 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 19 Mar 2020 15:09:02 -0700
Subject: [PATCH 155/285] Enable assertions for IntelliJ junit test runner

---
 gradle/ide.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/ide.gradle b/gradle/ide.gradle
index 3580f4a28f28e..b3d866d8342b8 100644
--- a/gradle/ide.gradle
+++ b/gradle/ide.gradle
@@ -90,7 +90,7 @@ idea {
           port = 5005
         }
         defaults(JUnit) {
-          vmParameters = '-Djava.locale.providers=SPI,COMPAT'
+          vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT'
         }
       }
       copyright {

From 64d228ceaab3d13e33e57e94c8c7999f013df2fc Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Thu, 19 Mar 2020 16:55:24 -0600
Subject: [PATCH 156/285] Re-enable BWC tests (#53833)

This is to re-enable after #53830
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 295717bac44b4..fa1ebcf6706f8 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53829" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 07eefa9807daafd11d1b847e641fda469e35b14b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 19 Mar 2020 18:56:04 -0400
Subject: [PATCH 157/285] Clean up eclipse build (#53831)

* Clean up eclipse build

Fixes up the "forbidden" warnings that you get when you import
Elasticsearch using "import gradle projects".

With this, and the manual step of switching circular project definitions
to warnings this gets most thing *compiling*.

* Move
---
 build.gradle                              | 18 +++++++++++++++++-
 x-pack/plugin/sql/sql-client/build.gradle | 13 -------------
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/build.gradle b/build.gradle
index fa1ebcf6706f8..932afc36f001f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -17,15 +17,16 @@
  * under the License.
  */
 
-
 import com.avast.gradle.dockercompose.tasks.ComposePull
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.BwcVersions
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
+import org.gradle.plugins.ide.eclipse.model.AccessRule
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
@@ -378,6 +379,21 @@ allprojects {
       }
     }
   }
+  /*
+   * Allow accessing com/sun/net/httpserver in projects that have
+   * configured forbidden apis to allow it.
+   */
+  plugins.withType(ForbiddenApisPlugin) {
+    eclipse.classpath.file.whenMerged { classpath ->
+      if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
+        classpath.entries
+          .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
+          .each {
+            it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
+          }
+      }
+    }
+  }
 
   File licenseHeaderFile
   String prefix = ':x-pack'
diff --git a/x-pack/plugin/sql/sql-client/build.gradle b/x-pack/plugin/sql/sql-client/build.gradle
index 071ac6eab84b1..0679a3fe5097b 100644
--- a/x-pack/plugin/sql/sql-client/build.gradle
+++ b/x-pack/plugin/sql/sql-client/build.gradle
@@ -34,16 +34,3 @@ forbiddenApisTest {
 forbiddenPatterns {
   exclude '**/*.keystore'
 }
-
-// Allow for com.sun.net.httpserver.* usage for testing
-eclipse {
-  classpath.file {
-    whenMerged { cp ->
-      def con = entries.find { e ->
-        e.kind == "con" && e.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER")
-      }
-      con.accessRules.add(new org.gradle.plugins.ide.eclipse.model.AccessRule(
-        "accessible", "com/sun/net/httpserver/*"))
-    }
-  }
-}

From 04aee0abfd173acd01dec21a56379f0f6da5ef33 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Fri, 20 Mar 2020 10:36:20 +0200
Subject: [PATCH 158/285] [ML] Data frame analytics analysis stats (#53788)

Adds parsing and indexing of analysis instrumentation stats.
The latest one is also returned from the get-stats API.

Note that we chose to duplicate objects even where they are currently
similar. There are already ideas on how these will diverge in the future
and while the duplication looks ugly at the moment, it is the option
that offers the highest flexibility.
---
 .../ml/dataframe/DataFrameAnalyticsStats.java |  32 +-
 .../ml/dataframe/stats/AnalysisStats.java     |  29 ++
 .../AnalysisStatsNamedXContentProvider.java   |  52 ++++
 .../classification/ClassificationStats.java   | 135 ++++++++
 .../stats/classification/Hyperparameters.java | 293 ++++++++++++++++++
 .../stats/classification/TimingStats.java     |  87 ++++++
 .../stats/classification/ValidationLoss.java  |  87 ++++++
 .../ml/dataframe/stats/common/FoldValues.java |  87 ++++++
 .../{ => stats/common}/MemoryUsage.java       |  10 +-
 .../OutlierDetectionStats.java                | 105 +++++++
 .../stats/outlierdetection/Parameters.java    | 146 +++++++++
 .../stats/outlierdetection/TimingStats.java   |  74 +++++
 .../stats/regression/Hyperparameters.java     | 278 +++++++++++++++++
 .../stats/regression/RegressionStats.java     | 135 ++++++++
 .../stats/regression/TimingStats.java         |  87 ++++++
 .../stats/regression/ValidationLoss.java      |  87 ++++++
 ...icsearch.plugins.spi.NamedXContentProvider |   1 +
 .../client/MLRequestConvertersTests.java      |   2 +
 .../client/RestHighLevelClientTests.java      |  11 +-
 .../DataFrameAnalyticsStatsTests.java         |  29 +-
 .../ClassificationStatsTests.java             |  53 ++++
 .../classification/HyperparametersTests.java  |  62 ++++
 .../classification/TimingStatsTests.java      |  50 +++
 .../classification/ValidationLossTests.java   |  50 +++
 .../stats/common/FoldValuesTests.java         |  51 +++
 .../{ => stats/common}/MemoryUsageTests.java  |   2 +-
 .../OutlierDetectionStatsTests.java           |  51 +++
 .../outlierdetection/ParametersTests.java     |  53 ++++
 .../outlierdetection/TimingStatsTests.java    |  48 +++
 .../regression/HyperparametersTests.java      |  62 ++++
 .../regression/RegressionStatsTests.java      |  54 ++++
 .../stats/regression/TimingStatsTests.java    |  50 +++
 .../stats/regression/ValidationLossTests.java |  50 +++
 .../GetDataFrameAnalyticsStatsAction.java     |  24 +-
 .../ml/dataframe/stats/AnalysisStats.java     |  15 +
 .../AnalysisStatsNamedWriteablesProvider.java |  25 ++
 .../xpack/core/ml/dataframe/stats/Fields.java |  20 ++
 .../core/ml/dataframe/stats/MemoryUsage.java  |  19 +-
 .../classification/ClassificationStats.java   | 148 +++++++++
 .../stats/classification/Hyperparameters.java | 237 ++++++++++++++
 .../stats/classification/TimingStats.java     |  80 +++++
 .../stats/classification/ValidationLoss.java  |  82 +++++
 .../ml/dataframe/stats/common/FoldValues.java |  84 +++++
 .../OutlierDetectionStats.java                | 122 ++++++++
 .../stats/outlierdetection/Parameters.java    | 125 ++++++++
 .../stats/outlierdetection/TimingStats.java   |  73 +++++
 .../stats/regression/Hyperparameters.java     | 226 ++++++++++++++
 .../stats/regression/RegressionStats.java     | 148 +++++++++
 .../stats/regression/TimingStats.java         |  79 +++++
 .../stats/regression/ValidationLoss.java      |  82 +++++
 .../xpack/core/ml/stats_index_mappings.json   | 110 ++++++-
 ...rameAnalyticsStatsActionResponseTests.java |  22 +-
 .../ClassificationStatsTests.java             |  69 +++++
 .../classification/HyperparametersTests.java  |  69 +++++
 .../classification/TimingStatsTests.java      |  54 ++++
 .../classification/ValidationLossTests.java   |  52 ++++
 .../stats/common/FoldValuesTests.java         |  58 ++++
 .../OutlierDetectionStatsTests.java           |  68 ++++
 .../outlierdetection/ParametersTests.java     |  61 ++++
 .../outlierdetection/TimingStatsTests.java    |  54 ++++
 .../regression/HyperparametersTests.java      |  68 ++++
 .../regression/RegressionStatsTests.java      |  69 +++++
 .../stats/regression/TimingStatsTests.java    |  54 ++++
 .../stats/regression/ValidationLossTests.java |  52 ++++
 .../xpack/ml/MachineLearning.java             |   2 +
 ...sportGetDataFrameAnalyticsStatsAction.java |  35 ++-
 .../process/AnalyticsResultProcessor.java     |  18 ++
 .../process/results/AnalyticsResult.java      |  59 +++-
 .../xpack/ml/dataframe/stats/StatsHolder.java |  11 +
 .../process/AnalyticsProcessManagerTests.java |   2 +-
 .../AnalyticsResultProcessorTests.java        |  16 +-
 .../process/results/AnalyticsResultTests.java |  26 +-
 72 files changed, 4972 insertions(+), 49 deletions(-)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStatsNamedXContentProvider.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/Hyperparameters.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLoss.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValues.java
 rename client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/{ => stats/common}/MemoryUsage.java (94%)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/Parameters.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/Hyperparameters.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLoss.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/HyperparametersTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLossTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValuesTests.java
 rename client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/{ => stats/common}/MemoryUsageTests.java (96%)
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/ParametersTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/HyperparametersTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLossTests.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStatsNamedWriteablesProvider.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/Fields.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/Hyperparameters.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLoss.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValues.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/Parameters.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/Hyperparameters.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStats.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLoss.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
index 53e3adf2b8433..70169ce09b278 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
@@ -20,12 +20,15 @@
 package org.elasticsearch.client.ml.dataframe;
 
 import org.elasticsearch.client.ml.NodeAttributes;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsage;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.inject.internal.ToStringBuilder;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,6 +48,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
     static final ParseField FAILURE_REASON = new ParseField("failure_reason");
     static final ParseField PROGRESS = new ParseField("progress");
     static final ParseField MEMORY_USAGE = new ParseField("memory_usage");
+    static final ParseField ANALYSIS_STATS = new ParseField("analysis_stats");
     static final ParseField NODE = new ParseField("node");
     static final ParseField ASSIGNMENT_EXPLANATION = new ParseField("assignment_explanation");
 
@@ -57,8 +61,9 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
                 (String) args[2],
                 (List<PhaseProgress>) args[3],
                 (MemoryUsage) args[4],
-                (NodeAttributes) args[5],
-                (String) args[6]));
+                (AnalysisStats) args[5],
+                (NodeAttributes) args[6],
+                (String) args[7]));
 
     static {
         PARSER.declareString(constructorArg(), ID);
@@ -71,26 +76,38 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
         PARSER.declareString(optionalConstructorArg(), FAILURE_REASON);
         PARSER.declareObjectArray(optionalConstructorArg(), PhaseProgress.PARSER, PROGRESS);
         PARSER.declareObject(optionalConstructorArg(), MemoryUsage.PARSER, MEMORY_USAGE);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAnalysisStats(p), ANALYSIS_STATS);
         PARSER.declareObject(optionalConstructorArg(), NodeAttributes.PARSER, NODE);
         PARSER.declareString(optionalConstructorArg(), ASSIGNMENT_EXPLANATION);
     }
 
+    private static AnalysisStats parseAnalysisStats(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        AnalysisStats analysisStats = parser.namedObject(AnalysisStats.class, parser.currentName(), true);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return analysisStats;
+    }
+
     private final String id;
     private final DataFrameAnalyticsState state;
     private final String failureReason;
     private final List<PhaseProgress> progress;
     private final MemoryUsage memoryUsage;
+    private final AnalysisStats analysisStats;
     private final NodeAttributes node;
     private final String assignmentExplanation;
 
     public DataFrameAnalyticsStats(String id, DataFrameAnalyticsState state, @Nullable String failureReason,
                                    @Nullable List<PhaseProgress> progress, @Nullable MemoryUsage memoryUsage,
-                                   @Nullable NodeAttributes node, @Nullable String assignmentExplanation) {
+                                   @Nullable AnalysisStats analysisStats, @Nullable NodeAttributes node,
+                                   @Nullable String assignmentExplanation) {
         this.id = id;
         this.state = state;
         this.failureReason = failureReason;
         this.progress = progress;
         this.memoryUsage = memoryUsage;
+        this.analysisStats = analysisStats;
         this.node = node;
         this.assignmentExplanation = assignmentExplanation;
     }
@@ -116,6 +133,11 @@ public MemoryUsage getMemoryUsage() {
         return memoryUsage;
     }
 
+    @Nullable
+    public AnalysisStats getAnalysisStats() {
+        return analysisStats;
+    }
+
     public NodeAttributes getNode() {
         return node;
     }
@@ -135,13 +157,14 @@ public boolean equals(Object o) {
             && Objects.equals(failureReason, other.failureReason)
             && Objects.equals(progress, other.progress)
             && Objects.equals(memoryUsage, other.memoryUsage)
+            && Objects.equals(analysisStats, other.analysisStats)
             && Objects.equals(node, other.node)
             && Objects.equals(assignmentExplanation, other.assignmentExplanation);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, state, failureReason, progress, memoryUsage, node, assignmentExplanation);
+        return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
     }
 
     @Override
@@ -152,6 +175,7 @@ public String toString() {
             .add("failureReason", failureReason)
             .add("progress", progress)
             .add("memoryUsage", memoryUsage)
+            .add("analysisStats", analysisStats)
             .add("node", node)
             .add("assignmentExplanation", assignmentExplanation)
             .toString();
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStats.java
new file mode 100644
index 0000000000000..c1a823682a762
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStats.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats;
+
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+/**
+ * Statistics for the data frame analysis
+ */
+public interface AnalysisStats extends ToXContentObject {
+
+    String getName();
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStatsNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStatsNamedXContentProvider.java
new file mode 100644
index 0000000000000..8c9bc615e8653
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStatsNamedXContentProvider.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats;
+
+import org.elasticsearch.client.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.client.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.client.ml.dataframe.stats.regression.RegressionStats;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class AnalysisStatsNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+            new NamedXContentRegistry.Entry(
+                AnalysisStats.class,
+                ClassificationStats.NAME,
+                (p, c) -> ClassificationStats.PARSER.apply(p, null)
+            ),
+            new NamedXContentRegistry.Entry(
+                AnalysisStats.class,
+                OutlierDetectionStats.NAME,
+                (p, c) -> OutlierDetectionStats.PARSER.apply(p, null)
+            ),
+            new NamedXContentRegistry.Entry(
+                AnalysisStats.class,
+                RegressionStats.NAME,
+                (p, c) -> RegressionStats.PARSER.apply(p, null)
+            )
+        );
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStats.java
new file mode 100644
index 0000000000000..101f74f2fe239
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStats.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.client.common.TimeUtil;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class ClassificationStats implements AnalysisStats {
+
+    public static final ParseField NAME = new ParseField("classification_stats");
+
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField ITERATION = new ParseField("iteration");
+    public static final ParseField HYPERPARAMETERS = new ParseField("hyperparameters");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
+    public static final ParseField VALIDATION_LOSS = new ParseField("validation_loss");
+
+    public static final ConstructingObjectParser<ClassificationStats, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        true,
+        a -> new ClassificationStats(
+            (Instant) a[0],
+            (Integer) a[1],
+            (Hyperparameters) a[2],
+            (TimingStats) a[3],
+            (ValidationLoss) a[4]
+        )
+    );
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtil.parseTimeFieldToInstant(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), ITERATION);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Hyperparameters.PARSER, HYPERPARAMETERS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimingStats.PARSER, TIMING_STATS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), ValidationLoss.PARSER, VALIDATION_LOSS);
+    }
+
+    private final Instant timestamp;
+    private final Integer iteration;
+    private final Hyperparameters hyperparameters;
+    private final TimingStats timingStats;
+    private final ValidationLoss validationLoss;
+
+    public ClassificationStats(Instant timestamp, Integer iteration, Hyperparameters hyperparameters, TimingStats timingStats,
+                               ValidationLoss validationLoss) {
+        this.timestamp = Instant.ofEpochMilli(Objects.requireNonNull(timestamp).toEpochMilli());
+        this.iteration = iteration;
+        this.hyperparameters = Objects.requireNonNull(hyperparameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+        this.validationLoss = Objects.requireNonNull(validationLoss);
+    }
+
+    public Instant getTimestamp() {
+        return timestamp;
+    }
+
+    public Integer getIteration() {
+        return iteration;
+    }
+
+    public Hyperparameters getHyperparameters() {
+        return hyperparameters;
+    }
+
+    public TimingStats getTimingStats() {
+        return timingStats;
+    }
+
+    public ValidationLoss getValidationLoss() {
+        return validationLoss;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        if (iteration != null) {
+            builder.field(ITERATION.getPreferredName(), iteration);
+        }
+        builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.field(VALIDATION_LOSS.getPreferredName(), validationLoss);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ClassificationStats that = (ClassificationStats) o;
+        return Objects.equals(timestamp, that.timestamp)
+            && Objects.equals(iteration, that.iteration)
+            && Objects.equals(hyperparameters, that.hyperparameters)
+            && Objects.equals(timingStats, that.timingStats)
+            && Objects.equals(validationLoss, that.validationLoss);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(timestamp, iteration, hyperparameters, timingStats, validationLoss);
+    }
+
+    @Override
+    public String getName() {
+        return NAME.getPreferredName();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/Hyperparameters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/Hyperparameters.java
new file mode 100644
index 0000000000000..c8d581b1d9c41
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/Hyperparameters.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class Hyperparameters implements ToXContentObject {
+
+    public static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
+    public static final ParseField DOWNSAMPLE_FACTOR = new ParseField("downsample_factor");
+    public static final ParseField ETA = new ParseField("eta");
+    public static final ParseField ETA_GROWTH_RATE_PER_TREE = new ParseField("eta_growth_rate_per_tree");
+    public static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
+    public static final ParseField MAX_ATTEMPTS_TO_ADD_TREE = new ParseField("max_attempts_to_add_tree");
+    public static final ParseField MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER = new ParseField(
+        "max_optimization_rounds_per_hyperparameter");
+    public static final ParseField MAX_TREES = new ParseField("max_trees");
+    public static final ParseField NUM_FOLDS = new ParseField("num_folds");
+    public static final ParseField NUM_SPLITS_PER_FEATURE = new ParseField("num_splits_per_feature");
+    public static final ParseField REGULARIZATION_DEPTH_PENALTY_MULTIPLIER = new ParseField("regularization_depth_penalty_multiplier");
+    public static final ParseField REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER
+        = new ParseField("regularization_leaf_weight_penalty_multiplier");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_LIMIT = new ParseField("regularization_soft_tree_depth_limit");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE = new ParseField("regularization_soft_tree_depth_tolerance");
+    public static final ParseField REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER =
+        new ParseField("regularization_tree_size_penalty_multiplier");
+
+    public static ConstructingObjectParser<Hyperparameters, Void> PARSER = new ConstructingObjectParser<>("classification_hyperparameters",
+        true,
+        a -> new Hyperparameters(
+            (String) a[0],
+            (Double) a[1],
+            (Double) a[2],
+            (Double) a[3],
+            (Double) a[4],
+            (Integer) a[5],
+            (Integer) a[6],
+            (Integer) a[7],
+            (Integer) a[8],
+            (Integer) a[9],
+            (Double) a[10],
+            (Double) a[11],
+            (Double) a[12],
+            (Double) a[13],
+            (Double) a[14]
+        ));
+
+    static {
+        PARSER.declareString(optionalConstructorArg(), CLASS_ASSIGNMENT_OBJECTIVE);
+        PARSER.declareDouble(optionalConstructorArg(), DOWNSAMPLE_FACTOR);
+        PARSER.declareDouble(optionalConstructorArg(), ETA);
+        PARSER.declareDouble(optionalConstructorArg(), ETA_GROWTH_RATE_PER_TREE);
+        PARSER.declareDouble(optionalConstructorArg(), FEATURE_BAG_FRACTION);
+        PARSER.declareInt(optionalConstructorArg(), MAX_ATTEMPTS_TO_ADD_TREE);
+        PARSER.declareInt(optionalConstructorArg(), MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER);
+        PARSER.declareInt(optionalConstructorArg(), MAX_TREES);
+        PARSER.declareInt(optionalConstructorArg(), NUM_FOLDS);
+        PARSER.declareInt(optionalConstructorArg(), NUM_SPLITS_PER_FEATURE);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_DEPTH_PENALTY_MULTIPLIER);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_LIMIT);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER);
+    }
+
+    private final String classAssignmentObjective;
+    private final Double downsampleFactor;
+    private final Double eta;
+    private final Double etaGrowthRatePerTree;
+    private final Double featureBagFraction;
+    private final Integer maxAttemptsToAddTree;
+    private final Integer maxOptimizationRoundsPerHyperparameter;
+    private final Integer maxTrees;
+    private final Integer numFolds;
+    private final Integer numSplitsPerFeature;
+    private final Double regularizationDepthPenaltyMultiplier;
+    private final Double regularizationLeafWeightPenaltyMultiplier;
+    private final Double regularizationSoftTreeDepthLimit;
+    private final Double regularizationSoftTreeDepthTolerance;
+    private final Double regularizationTreeSizePenaltyMultiplier;
+
+    public Hyperparameters(String classAssignmentObjective,
+                           Double downsampleFactor,
+                           Double eta,
+                           Double etaGrowthRatePerTree,
+                           Double featureBagFraction,
+                           Integer maxAttemptsToAddTree,
+                           Integer maxOptimizationRoundsPerHyperparameter,
+                           Integer maxTrees,
+                           Integer numFolds,
+                           Integer numSplitsPerFeature,
+                           Double regularizationDepthPenaltyMultiplier,
+                           Double regularizationLeafWeightPenaltyMultiplier,
+                           Double regularizationSoftTreeDepthLimit,
+                           Double regularizationSoftTreeDepthTolerance,
+                           Double regularizationTreeSizePenaltyMultiplier) {
+        this.classAssignmentObjective = classAssignmentObjective;
+        this.downsampleFactor = downsampleFactor;
+        this.eta = eta;
+        this.etaGrowthRatePerTree = etaGrowthRatePerTree;
+        this.featureBagFraction = featureBagFraction;
+        this.maxAttemptsToAddTree = maxAttemptsToAddTree;
+        this.maxOptimizationRoundsPerHyperparameter = maxOptimizationRoundsPerHyperparameter;
+        this.maxTrees = maxTrees;
+        this.numFolds = numFolds;
+        this.numSplitsPerFeature = numSplitsPerFeature;
+        this.regularizationDepthPenaltyMultiplier = regularizationDepthPenaltyMultiplier;
+        this.regularizationLeafWeightPenaltyMultiplier = regularizationLeafWeightPenaltyMultiplier;
+        this.regularizationSoftTreeDepthLimit = regularizationSoftTreeDepthLimit;
+        this.regularizationSoftTreeDepthTolerance = regularizationSoftTreeDepthTolerance;
+        this.regularizationTreeSizePenaltyMultiplier = regularizationTreeSizePenaltyMultiplier;
+    }
+
+    public String getClassAssignmentObjective() {
+        return classAssignmentObjective;
+    }
+
+    public Double getDownsampleFactor() {
+        return downsampleFactor;
+    }
+
+    public Double getEta() {
+        return eta;
+    }
+
+    public Double getEtaGrowthRatePerTree() {
+        return etaGrowthRatePerTree;
+    }
+
+    public Double getFeatureBagFraction() {
+        return featureBagFraction;
+    }
+
+    public Integer getMaxAttemptsToAddTree() {
+        return maxAttemptsToAddTree;
+    }
+
+    public Integer getMaxOptimizationRoundsPerHyperparameter() {
+        return maxOptimizationRoundsPerHyperparameter;
+    }
+
+    public Integer getMaxTrees() {
+        return maxTrees;
+    }
+
+    public Integer getNumFolds() {
+        return numFolds;
+    }
+
+    public Integer getNumSplitsPerFeature() {
+        return numSplitsPerFeature;
+    }
+
+    public Double getRegularizationDepthPenaltyMultiplier() {
+        return regularizationDepthPenaltyMultiplier;
+    }
+
+    public Double getRegularizationLeafWeightPenaltyMultiplier() {
+        return regularizationLeafWeightPenaltyMultiplier;
+    }
+
+    public Double getRegularizationSoftTreeDepthLimit() {
+        return regularizationSoftTreeDepthLimit;
+    }
+
+    public Double getRegularizationSoftTreeDepthTolerance() {
+        return regularizationSoftTreeDepthTolerance;
+    }
+
+    public Double getRegularizationTreeSizePenaltyMultiplier() {
+        return regularizationTreeSizePenaltyMultiplier;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (classAssignmentObjective != null) {
+            builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
+        }
+        if (downsampleFactor != null) {
+            builder.field(DOWNSAMPLE_FACTOR.getPreferredName(), downsampleFactor);
+        }
+        if (eta != null) {
+            builder.field(ETA.getPreferredName(), eta);
+        }
+        if (etaGrowthRatePerTree != null) {
+            builder.field(ETA_GROWTH_RATE_PER_TREE.getPreferredName(), etaGrowthRatePerTree);
+        }
+        if (featureBagFraction != null) {
+            builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
+        }
+        if (maxAttemptsToAddTree != null) {
+            builder.field(MAX_ATTEMPTS_TO_ADD_TREE.getPreferredName(), maxAttemptsToAddTree);
+        }
+        if (maxOptimizationRoundsPerHyperparameter != null) {
+            builder.field(MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER.getPreferredName(), maxOptimizationRoundsPerHyperparameter);
+        }
+        if (maxTrees != null) {
+            builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        }
+        if (numFolds != null) {
+            builder.field(NUM_FOLDS.getPreferredName(), numFolds);
+        }
+        if (numSplitsPerFeature != null) {
+            builder.field(NUM_SPLITS_PER_FEATURE.getPreferredName(), numSplitsPerFeature);
+        }
+        if (regularizationDepthPenaltyMultiplier != null) {
+            builder.field(REGULARIZATION_DEPTH_PENALTY_MULTIPLIER.getPreferredName(), regularizationDepthPenaltyMultiplier);
+        }
+        if (regularizationLeafWeightPenaltyMultiplier != null) {
+            builder.field(REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER.getPreferredName(), regularizationLeafWeightPenaltyMultiplier);
+        }
+        if (regularizationSoftTreeDepthLimit != null) {
+            builder.field(REGULARIZATION_SOFT_TREE_DEPTH_LIMIT.getPreferredName(), regularizationSoftTreeDepthLimit);
+        }
+        if (regularizationSoftTreeDepthTolerance != null) {
+            builder.field(REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE.getPreferredName(), regularizationSoftTreeDepthTolerance);
+        }
+        if (regularizationTreeSizePenaltyMultiplier != null) {
+            builder.field(REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER.getPreferredName(), regularizationTreeSizePenaltyMultiplier);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Hyperparameters that = (Hyperparameters) o;
+        return Objects.equals(classAssignmentObjective, that.classAssignmentObjective)
+            && Objects.equals(downsampleFactor, that.downsampleFactor)
+            && Objects.equals(eta, that.eta)
+            && Objects.equals(etaGrowthRatePerTree, that.etaGrowthRatePerTree)
+            && Objects.equals(featureBagFraction, that.featureBagFraction)
+            && Objects.equals(maxAttemptsToAddTree, that.maxAttemptsToAddTree)
+            && Objects.equals(maxOptimizationRoundsPerHyperparameter, that.maxOptimizationRoundsPerHyperparameter)
+            && Objects.equals(maxTrees, that.maxTrees)
+            && Objects.equals(numFolds, that.numFolds)
+            && Objects.equals(numSplitsPerFeature, that.numSplitsPerFeature)
+            && Objects.equals(regularizationDepthPenaltyMultiplier, that.regularizationDepthPenaltyMultiplier)
+            && Objects.equals(regularizationLeafWeightPenaltyMultiplier, that.regularizationLeafWeightPenaltyMultiplier)
+            && Objects.equals(regularizationSoftTreeDepthLimit, that.regularizationSoftTreeDepthLimit)
+            && Objects.equals(regularizationSoftTreeDepthTolerance, that.regularizationSoftTreeDepthTolerance)
+            && Objects.equals(regularizationTreeSizePenaltyMultiplier, that.regularizationTreeSizePenaltyMultiplier);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            classAssignmentObjective,
+            downsampleFactor,
+            eta,
+            etaGrowthRatePerTree,
+            featureBagFraction,
+            maxAttemptsToAddTree,
+            maxOptimizationRoundsPerHyperparameter,
+            maxTrees,
+            numFolds,
+            numSplitsPerFeature,
+            regularizationDepthPenaltyMultiplier,
+            regularizationLeafWeightPenaltyMultiplier,
+            regularizationSoftTreeDepthLimit,
+            regularizationSoftTreeDepthTolerance,
+            regularizationTreeSizePenaltyMultiplier
+        );
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStats.java
new file mode 100644
index 0000000000000..bad599298a780
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStats.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class TimingStats implements ToXContentObject {
+
+    public static final ParseField ELAPSED_TIME = new ParseField("elapsed_time");
+    public static final ParseField ITERATION_TIME = new ParseField("iteration_time");
+
+    public static ConstructingObjectParser<TimingStats, Void> PARSER = new ConstructingObjectParser<>("classification_timing_stats", true,
+        a -> new TimingStats(
+            a[0] == null ? null : TimeValue.timeValueMillis((long) a[0]),
+            a[1] == null ? null : TimeValue.timeValueMillis((long) a[1])
+        ));
+
+    static {
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ELAPSED_TIME);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ITERATION_TIME);
+    }
+
+    private final TimeValue elapsedTime;
+    private final TimeValue iterationTime;
+
+    public TimingStats(TimeValue elapsedTime, TimeValue iterationTime) {
+        this.elapsedTime = elapsedTime;
+        this.iterationTime = iterationTime;
+    }
+
+    public TimeValue getElapsedTime() {
+        return elapsedTime;
+    }
+
+    public TimeValue getIterationTime() {
+        return iterationTime;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (elapsedTime != null) {
+            builder.humanReadableField(ELAPSED_TIME.getPreferredName(), ELAPSED_TIME.getPreferredName() + "_string", elapsedTime);
+        }
+        if (iterationTime != null) {
+            builder.humanReadableField(ITERATION_TIME.getPreferredName(), ITERATION_TIME.getPreferredName() + "_string", iterationTime);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(elapsedTime, that.elapsedTime) && Objects.equals(iterationTime, that.iterationTime);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(elapsedTime, iterationTime);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLoss.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLoss.java
new file mode 100644
index 0000000000000..a552f5d85e124
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLoss.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.client.ml.dataframe.stats.common.FoldValues;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class ValidationLoss implements ToXContentObject {
+
+    public static final ParseField LOSS_TYPE = new ParseField("loss_type");
+    public static final ParseField FOLD_VALUES = new ParseField("fold_values");
+
+    @SuppressWarnings("unchecked")
+    public static ConstructingObjectParser<ValidationLoss, Void> PARSER = new ConstructingObjectParser<>("classification_validation_loss",
+        true,
+        a -> new ValidationLoss((String) a[0], (List<FoldValues>) a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LOSS_TYPE);
+        PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), FoldValues.PARSER, FOLD_VALUES);
+    }
+
+    private final String lossType;
+    private final List<FoldValues> foldValues;
+
+    public ValidationLoss(String lossType, List<FoldValues> values) {
+        this.lossType = lossType;
+        this.foldValues = values;
+    }
+
+    public String getLossType() {
+        return lossType;
+    }
+
+    public List<FoldValues> getFoldValues() {
+        return foldValues;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (lossType != null) {
+            builder.field(LOSS_TYPE.getPreferredName(), lossType);
+        }
+        if (foldValues != null) {
+            builder.field(FOLD_VALUES.getPreferredName(), foldValues);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ValidationLoss that = (ValidationLoss) o;
+        return Objects.equals(lossType, that.lossType) && Objects.equals(foldValues, that.foldValues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(lossType, foldValues);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValues.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValues.java
new file mode 100644
index 0000000000000..30490981d9651
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValues.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class FoldValues implements ToXContentObject {
+
+    public static final ParseField FOLD = new ParseField("fold");
+    public static final ParseField VALUES = new ParseField("values");
+
+    @SuppressWarnings("unchecked")
+    public static ConstructingObjectParser<FoldValues, Void> PARSER = new ConstructingObjectParser<>("fold_values", true,
+        a -> new FoldValues((int) a[0], (List<Double>) a[1]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), FOLD);
+        PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), VALUES);
+    }
+
+    private final int fold;
+    private final double[] values;
+
+    private FoldValues(int fold, List<Double> values) {
+        this(fold, values.stream().mapToDouble(Double::doubleValue).toArray());
+    }
+
+    public FoldValues(int fold, double[] values) {
+        this.fold = fold;
+        this.values = values;
+    }
+
+    public int getFold() {
+        return fold;
+    }
+
+    public double[] getValues() {
+        return values;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(FOLD.getPreferredName(), fold);
+        builder.array(VALUES.getPreferredName(), values);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FoldValues other = (FoldValues) o;
+        return fold == other.fold && Arrays.equals(values, other.values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(fold, Arrays.hashCode(values));
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MemoryUsage.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/MemoryUsage.java
similarity index 94%
rename from client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MemoryUsage.java
rename to client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/MemoryUsage.java
index 323ebb52a7aed..f492d26528e02 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MemoryUsage.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/MemoryUsage.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.client.ml.dataframe;
+package org.elasticsearch.client.ml.dataframe.stats.common;
 
 import org.elasticsearch.client.common.TimeUtil;
 import org.elasticsearch.common.ParseField;
@@ -54,6 +54,14 @@ public MemoryUsage(Instant timestamp, long peakUsageBytes) {
         this.peakUsageBytes = peakUsageBytes;
     }
 
+    public Instant getTimestamp() {
+        return timestamp;
+    }
+
+    public long getPeakUsageBytes() {
+        return peakUsageBytes;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
new file mode 100644
index 0000000000000..15098e3770f3b
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.client.common.TimeUtil;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class OutlierDetectionStats implements AnalysisStats {
+
+    public static final ParseField NAME = new ParseField("outlier_detection_stats");
+
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField PARAMETERS = new ParseField("parameters");
+    public static final ParseField TIMING_STATS = new ParseField("timings_stats");
+
+    public static final ConstructingObjectParser<OutlierDetectionStats, Void> PARSER = new ConstructingObjectParser<>(
+        NAME.getPreferredName(), true,
+        a -> new OutlierDetectionStats((Instant) a[0], (Parameters) a[1], (TimingStats) a[2]));
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtil.parseTimeFieldToInstant(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Parameters.PARSER, PARAMETERS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimingStats.PARSER, TIMING_STATS);
+    }
+
+    private final Instant timestamp;
+    private final Parameters parameters;
+    private final TimingStats timingStats;
+
+    public OutlierDetectionStats(Instant timestamp, Parameters parameters, TimingStats timingStats) {
+        this.timestamp = Instant.ofEpochMilli(Objects.requireNonNull(timestamp).toEpochMilli());
+        this.parameters = Objects.requireNonNull(parameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+    }
+
+    public Instant getTimestamp() {
+        return timestamp;
+    }
+
+    public Parameters getParameters() {
+        return parameters;
+    }
+
+    public TimingStats getTimingStats() {
+        return timingStats;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.field(PARAMETERS.getPreferredName(), parameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        OutlierDetectionStats that = (OutlierDetectionStats) o;
+        return Objects.equals(timestamp, that.timestamp)
+            && Objects.equals(parameters, that.parameters)
+            && Objects.equals(timingStats, that.timingStats);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(timestamp, parameters, timingStats);
+    }
+
+    @Override
+    public String getName() {
+        return NAME.getPreferredName();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/Parameters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/Parameters.java
new file mode 100644
index 0000000000000..deafb55081de0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/Parameters.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class Parameters implements ToXContentObject {
+
+    public static final ParseField N_NEIGHBORS = new ParseField("n_neighbors");
+    public static final ParseField METHOD = new ParseField("method");
+    public static final ParseField FEATURE_INFLUENCE_THRESHOLD = new ParseField("feature_influence_threshold");
+    public static final ParseField COMPUTE_FEATURE_INFLUENCE = new ParseField("compute_feature_influence");
+    public static final ParseField OUTLIER_FRACTION = new ParseField("outlier_fraction");
+    public static final ParseField STANDARDIZATION_ENABLED = new ParseField("standardization_enabled");
+
+    @SuppressWarnings("unchecked")
+    public static ConstructingObjectParser<Parameters, Void> PARSER = new ConstructingObjectParser<>("outlier_detection_parameters",
+        true,
+        a -> new Parameters(
+            (Integer) a[0],
+            (String) a[1],
+            (Boolean) a[2],
+            (Double) a[3],
+            (Double) a[4],
+            (Boolean) a[5]
+        ));
+
+    static {
+        PARSER.declareInt(optionalConstructorArg(), N_NEIGHBORS);
+        PARSER.declareString(optionalConstructorArg(), METHOD);
+        PARSER.declareBoolean(optionalConstructorArg(), COMPUTE_FEATURE_INFLUENCE);
+        PARSER.declareDouble(optionalConstructorArg(), FEATURE_INFLUENCE_THRESHOLD);
+        PARSER.declareDouble(optionalConstructorArg(), OUTLIER_FRACTION);
+        PARSER.declareBoolean(optionalConstructorArg(), STANDARDIZATION_ENABLED);
+    }
+
+    private final Integer nNeighbors;
+    private final String method;
+    private final Boolean computeFeatureInfluence;
+    private final Double featureInfluenceThreshold;
+    private final Double outlierFraction;
+    private final Boolean standardizationEnabled;
+
+    public Parameters(Integer nNeighbors, String method, Boolean computeFeatureInfluence, Double featureInfluenceThreshold,
+                      Double outlierFraction, Boolean standardizationEnabled) {
+        this.nNeighbors = nNeighbors;
+        this.method = method;
+        this.computeFeatureInfluence = computeFeatureInfluence;
+        this.featureInfluenceThreshold = featureInfluenceThreshold;
+        this.outlierFraction = outlierFraction;
+        this.standardizationEnabled = standardizationEnabled;
+    }
+
+    public Integer getnNeighbors() {
+        return nNeighbors;
+    }
+
+    public String getMethod() {
+        return method;
+    }
+
+    public Boolean getComputeFeatureInfluence() {
+        return computeFeatureInfluence;
+    }
+
+    public Double getFeatureInfluenceThreshold() {
+        return featureInfluenceThreshold;
+    }
+
+    public Double getOutlierFraction() {
+        return outlierFraction;
+    }
+
+    public Boolean getStandardizationEnabled() {
+        return standardizationEnabled;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (nNeighbors != null) {
+            builder.field(N_NEIGHBORS.getPreferredName(), nNeighbors);
+        }
+        if (method != null) {
+            builder.field(METHOD.getPreferredName(), method);
+        }
+        if (computeFeatureInfluence != null) {
+            builder.field(COMPUTE_FEATURE_INFLUENCE.getPreferredName(), computeFeatureInfluence);
+        }
+        if (featureInfluenceThreshold != null) {
+            builder.field(FEATURE_INFLUENCE_THRESHOLD.getPreferredName(), featureInfluenceThreshold);
+        }
+        if (outlierFraction != null) {
+            builder.field(OUTLIER_FRACTION.getPreferredName(), outlierFraction);
+        }
+        if (standardizationEnabled != null) {
+            builder.field(STANDARDIZATION_ENABLED.getPreferredName(), standardizationEnabled);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Parameters that = (Parameters) o;
+        return Objects.equals(nNeighbors, that.nNeighbors)
+            && Objects.equals(method, that.method)
+            && Objects.equals(computeFeatureInfluence, that.computeFeatureInfluence)
+            && Objects.equals(featureInfluenceThreshold, that.featureInfluenceThreshold)
+            && Objects.equals(outlierFraction, that.outlierFraction)
+            && Objects.equals(standardizationEnabled, that.standardizationEnabled);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nNeighbors, method, computeFeatureInfluence, featureInfluenceThreshold, outlierFraction,
+            standardizationEnabled);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStats.java
new file mode 100644
index 0000000000000..96f93a6651de7
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStats.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class TimingStats implements ToXContentObject {
+
+    public static final ParseField ELAPSED_TIME = new ParseField("elapsed_time");
+
+    public static ConstructingObjectParser<TimingStats, Void> PARSER = new ConstructingObjectParser<>("outlier_detection_timing_stats",
+        true,
+        a -> new TimingStats(a[0] == null ? null : TimeValue.timeValueMillis((long) a[0])));
+
+    static {
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ELAPSED_TIME);
+    }
+
+    private final TimeValue elapsedTime;
+
+    public TimingStats(TimeValue elapsedTime) {
+        this.elapsedTime = elapsedTime;
+    }
+
+    public TimeValue getElapsedTime() {
+        return elapsedTime;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (elapsedTime != null) {
+            builder.humanReadableField(ELAPSED_TIME.getPreferredName(), ELAPSED_TIME.getPreferredName() + "_string", elapsedTime);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(elapsedTime, that.elapsedTime);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(elapsedTime);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/Hyperparameters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/Hyperparameters.java
new file mode 100644
index 0000000000000..cb1a0b99ab58b
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/Hyperparameters.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class Hyperparameters implements ToXContentObject {
+
+    public static final ParseField DOWNSAMPLE_FACTOR = new ParseField("downsample_factor");
+    public static final ParseField ETA = new ParseField("eta");
+    public static final ParseField ETA_GROWTH_RATE_PER_TREE = new ParseField("eta_growth_rate_per_tree");
+    public static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
+    public static final ParseField MAX_ATTEMPTS_TO_ADD_TREE = new ParseField("max_attempts_to_add_tree");
+    public static final ParseField MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER = new ParseField(
+        "max_optimization_rounds_per_hyperparameter");
+    public static final ParseField MAX_TREES = new ParseField("max_trees");
+    public static final ParseField NUM_FOLDS = new ParseField("num_folds");
+    public static final ParseField NUM_SPLITS_PER_FEATURE = new ParseField("num_splits_per_feature");
+    public static final ParseField REGULARIZATION_DEPTH_PENALTY_MULTIPLIER = new ParseField("regularization_depth_penalty_multiplier");
+    public static final ParseField REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER
+        = new ParseField("regularization_leaf_weight_penalty_multiplier");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_LIMIT = new ParseField("regularization_soft_tree_depth_limit");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE = new ParseField("regularization_soft_tree_depth_tolerance");
+    public static final ParseField REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER =
+        new ParseField("regularization_tree_size_penalty_multiplier");
+
+    public static ConstructingObjectParser<Hyperparameters, Void> PARSER = new ConstructingObjectParser<>("regression_hyperparameters",
+        true,
+        a -> new Hyperparameters(
+            (Double) a[0],
+            (Double) a[1],
+            (Double) a[2],
+            (Double) a[3],
+            (Integer) a[4],
+            (Integer) a[5],
+            (Integer) a[6],
+            (Integer) a[7],
+            (Integer) a[8],
+            (Double) a[9],
+            (Double) a[10],
+            (Double) a[11],
+            (Double) a[12],
+            (Double) a[13]
+        ));
+
+    static {
+        PARSER.declareDouble(optionalConstructorArg(), DOWNSAMPLE_FACTOR);
+        PARSER.declareDouble(optionalConstructorArg(), ETA);
+        PARSER.declareDouble(optionalConstructorArg(), ETA_GROWTH_RATE_PER_TREE);
+        PARSER.declareDouble(optionalConstructorArg(), FEATURE_BAG_FRACTION);
+        PARSER.declareInt(optionalConstructorArg(), MAX_ATTEMPTS_TO_ADD_TREE);
+        PARSER.declareInt(optionalConstructorArg(), MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER);
+        PARSER.declareInt(optionalConstructorArg(), MAX_TREES);
+        PARSER.declareInt(optionalConstructorArg(), NUM_FOLDS);
+        PARSER.declareInt(optionalConstructorArg(), NUM_SPLITS_PER_FEATURE);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_DEPTH_PENALTY_MULTIPLIER);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_LIMIT);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE);
+        PARSER.declareDouble(optionalConstructorArg(), REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER);
+    }
+
+    private final Double downsampleFactor;
+    private final Double eta;
+    private final Double etaGrowthRatePerTree;
+    private final Double featureBagFraction;
+    private final Integer maxAttemptsToAddTree;
+    private final Integer maxOptimizationRoundsPerHyperparameter;
+    private final Integer maxTrees;
+    private final Integer numFolds;
+    private final Integer numSplitsPerFeature;
+    private final Double regularizationDepthPenaltyMultiplier;
+    private final Double regularizationLeafWeightPenaltyMultiplier;
+    private final Double regularizationSoftTreeDepthLimit;
+    private final Double regularizationSoftTreeDepthTolerance;
+    private final Double regularizationTreeSizePenaltyMultiplier;
+
+    public Hyperparameters(Double downsampleFactor,
+                           Double eta,
+                           Double etaGrowthRatePerTree,
+                           Double featureBagFraction,
+                           Integer maxAttemptsToAddTree,
+                           Integer maxOptimizationRoundsPerHyperparameter,
+                           Integer maxTrees,
+                           Integer numFolds,
+                           Integer numSplitsPerFeature,
+                           Double regularizationDepthPenaltyMultiplier,
+                           Double regularizationLeafWeightPenaltyMultiplier,
+                           Double regularizationSoftTreeDepthLimit,
+                           Double regularizationSoftTreeDepthTolerance,
+                           Double regularizationTreeSizePenaltyMultiplier) {
+        this.downsampleFactor = downsampleFactor;
+        this.eta = eta;
+        this.etaGrowthRatePerTree = etaGrowthRatePerTree;
+        this.featureBagFraction = featureBagFraction;
+        this.maxAttemptsToAddTree = maxAttemptsToAddTree;
+        this.maxOptimizationRoundsPerHyperparameter = maxOptimizationRoundsPerHyperparameter;
+        this.maxTrees = maxTrees;
+        this.numFolds = numFolds;
+        this.numSplitsPerFeature = numSplitsPerFeature;
+        this.regularizationDepthPenaltyMultiplier = regularizationDepthPenaltyMultiplier;
+        this.regularizationLeafWeightPenaltyMultiplier = regularizationLeafWeightPenaltyMultiplier;
+        this.regularizationSoftTreeDepthLimit = regularizationSoftTreeDepthLimit;
+        this.regularizationSoftTreeDepthTolerance = regularizationSoftTreeDepthTolerance;
+        this.regularizationTreeSizePenaltyMultiplier = regularizationTreeSizePenaltyMultiplier;
+    }
+
+    public Double getDownsampleFactor() {
+        return downsampleFactor;
+    }
+
+    public Double getEta() {
+        return eta;
+    }
+
+    public Double getEtaGrowthRatePerTree() {
+        return etaGrowthRatePerTree;
+    }
+
+    public Double getFeatureBagFraction() {
+        return featureBagFraction;
+    }
+
+    public Integer getMaxAttemptsToAddTree() {
+        return maxAttemptsToAddTree;
+    }
+
+    public Integer getMaxOptimizationRoundsPerHyperparameter() {
+        return maxOptimizationRoundsPerHyperparameter;
+    }
+
+    public Integer getMaxTrees() {
+        return maxTrees;
+    }
+
+    public Integer getNumFolds() {
+        return numFolds;
+    }
+
+    public Integer getNumSplitsPerFeature() {
+        return numSplitsPerFeature;
+    }
+
+    public Double getRegularizationDepthPenaltyMultiplier() {
+        return regularizationDepthPenaltyMultiplier;
+    }
+
+    public Double getRegularizationLeafWeightPenaltyMultiplier() {
+        return regularizationLeafWeightPenaltyMultiplier;
+    }
+
+    public Double getRegularizationSoftTreeDepthLimit() {
+        return regularizationSoftTreeDepthLimit;
+    }
+
+    public Double getRegularizationSoftTreeDepthTolerance() {
+        return regularizationSoftTreeDepthTolerance;
+    }
+
+    public Double getRegularizationTreeSizePenaltyMultiplier() {
+        return regularizationTreeSizePenaltyMultiplier;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (downsampleFactor != null) {
+            builder.field(DOWNSAMPLE_FACTOR.getPreferredName(), downsampleFactor);
+        }
+        if (eta != null) {
+            builder.field(ETA.getPreferredName(), eta);
+        }
+        if (etaGrowthRatePerTree != null) {
+            builder.field(ETA_GROWTH_RATE_PER_TREE.getPreferredName(), etaGrowthRatePerTree);
+        }
+        if (featureBagFraction != null) {
+            builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
+        }
+        if (maxAttemptsToAddTree != null) {
+            builder.field(MAX_ATTEMPTS_TO_ADD_TREE.getPreferredName(), maxAttemptsToAddTree);
+        }
+        if (maxOptimizationRoundsPerHyperparameter != null) {
+            builder.field(MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER.getPreferredName(), maxOptimizationRoundsPerHyperparameter);
+        }
+        if (maxTrees != null) {
+            builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        }
+        if (numFolds != null) {
+            builder.field(NUM_FOLDS.getPreferredName(), numFolds);
+        }
+        if (numSplitsPerFeature != null) {
+            builder.field(NUM_SPLITS_PER_FEATURE.getPreferredName(), numSplitsPerFeature);
+        }
+        if (regularizationDepthPenaltyMultiplier != null) {
+            builder.field(REGULARIZATION_DEPTH_PENALTY_MULTIPLIER.getPreferredName(), regularizationDepthPenaltyMultiplier);
+        }
+        if (regularizationLeafWeightPenaltyMultiplier != null) {
+            builder.field(REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER.getPreferredName(), regularizationLeafWeightPenaltyMultiplier);
+        }
+        if (regularizationSoftTreeDepthLimit != null) {
+            builder.field(REGULARIZATION_SOFT_TREE_DEPTH_LIMIT.getPreferredName(), regularizationSoftTreeDepthLimit);
+        }
+        if (regularizationSoftTreeDepthTolerance != null) {
+            builder.field(REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE.getPreferredName(), regularizationSoftTreeDepthTolerance);
+        }
+        if (regularizationTreeSizePenaltyMultiplier != null) {
+            builder.field(REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER.getPreferredName(), regularizationTreeSizePenaltyMultiplier);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Hyperparameters that = (Hyperparameters) o;
+        return Objects.equals(downsampleFactor, that.downsampleFactor)
+            && Objects.equals(eta, that.eta)
+            && Objects.equals(etaGrowthRatePerTree, that.etaGrowthRatePerTree)
+            && Objects.equals(featureBagFraction, that.featureBagFraction)
+            && Objects.equals(maxAttemptsToAddTree, that.maxAttemptsToAddTree)
+            && Objects.equals(maxOptimizationRoundsPerHyperparameter, that.maxOptimizationRoundsPerHyperparameter)
+            && Objects.equals(maxTrees, that.maxTrees)
+            && Objects.equals(numFolds, that.numFolds)
+            && Objects.equals(numSplitsPerFeature, that.numSplitsPerFeature)
+            && Objects.equals(regularizationDepthPenaltyMultiplier, that.regularizationDepthPenaltyMultiplier)
+            && Objects.equals(regularizationLeafWeightPenaltyMultiplier, that.regularizationLeafWeightPenaltyMultiplier)
+            && Objects.equals(regularizationSoftTreeDepthLimit, that.regularizationSoftTreeDepthLimit)
+            && Objects.equals(regularizationSoftTreeDepthTolerance, that.regularizationSoftTreeDepthTolerance)
+            && Objects.equals(regularizationTreeSizePenaltyMultiplier, that.regularizationTreeSizePenaltyMultiplier);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            downsampleFactor,
+            eta,
+            etaGrowthRatePerTree,
+            featureBagFraction,
+            maxAttemptsToAddTree,
+            maxOptimizationRoundsPerHyperparameter,
+            maxTrees,
+            numFolds,
+            numSplitsPerFeature,
+            regularizationDepthPenaltyMultiplier,
+            regularizationLeafWeightPenaltyMultiplier,
+            regularizationSoftTreeDepthLimit,
+            regularizationSoftTreeDepthTolerance,
+            regularizationTreeSizePenaltyMultiplier
+        );
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStats.java
new file mode 100644
index 0000000000000..7e890c3618f82
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStats.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.client.common.TimeUtil;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class RegressionStats implements AnalysisStats {
+
+    public static final ParseField NAME = new ParseField("regression_stats");
+
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField ITERATION = new ParseField("iteration");
+    public static final ParseField HYPERPARAMETERS = new ParseField("hyperparameters");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
+    public static final ParseField VALIDATION_LOSS = new ParseField("validation_loss");
+
+    public static final ConstructingObjectParser<RegressionStats, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        true,
+        a -> new RegressionStats(
+            (Instant) a[0],
+            (Integer) a[1],
+            (Hyperparameters) a[2],
+            (TimingStats) a[3],
+            (ValidationLoss) a[4]
+        )
+    );
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtil.parseTimeFieldToInstant(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), ITERATION);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Hyperparameters.PARSER, HYPERPARAMETERS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimingStats.PARSER, TIMING_STATS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), ValidationLoss.PARSER, VALIDATION_LOSS);
+    }
+
+    private final Instant timestamp;
+    private final Integer iteration;
+    private final Hyperparameters hyperparameters;
+    private final TimingStats timingStats;
+    private final ValidationLoss validationLoss;
+
+    public RegressionStats(Instant timestamp, Integer iteration, Hyperparameters hyperparameters, TimingStats timingStats,
+                           ValidationLoss validationLoss) {
+        this.timestamp = Instant.ofEpochMilli(Objects.requireNonNull(timestamp).toEpochMilli());
+        this.iteration = iteration;
+        this.hyperparameters = Objects.requireNonNull(hyperparameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+        this.validationLoss = Objects.requireNonNull(validationLoss);
+    }
+
+    public Instant getTimestamp() {
+        return timestamp;
+    }
+
+    public Integer getIteration() {
+        return iteration;
+    }
+
+    public Hyperparameters getHyperparameters() {
+        return hyperparameters;
+    }
+
+    public TimingStats getTimingStats() {
+        return timingStats;
+    }
+
+    public ValidationLoss getValidationLoss() {
+        return validationLoss;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        if (iteration != null) {
+            builder.field(ITERATION.getPreferredName(), iteration);
+        }
+        builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.field(VALIDATION_LOSS.getPreferredName(), validationLoss);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RegressionStats that = (RegressionStats) o;
+        return Objects.equals(timestamp, that.timestamp)
+            && Objects.equals(iteration, that.iteration)
+            && Objects.equals(hyperparameters, that.hyperparameters)
+            && Objects.equals(timingStats, that.timingStats)
+            && Objects.equals(validationLoss, that.validationLoss);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(timestamp, iteration, hyperparameters, timingStats, validationLoss);
+    }
+
+    @Override
+    public String getName() {
+        return NAME.getPreferredName();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStats.java
new file mode 100644
index 0000000000000..1a844a410f469
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStats.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class TimingStats implements ToXContentObject {
+
+    public static final ParseField ELAPSED_TIME = new ParseField("elapsed_time");
+    public static final ParseField ITERATION_TIME = new ParseField("iteration_time");
+
+    public static ConstructingObjectParser<TimingStats, Void> PARSER = new ConstructingObjectParser<>("regression_timing_stats", true,
+        a -> new TimingStats(
+            a[0] == null ? null : TimeValue.timeValueMillis((long) a[0]),
+            a[1] == null ? null : TimeValue.timeValueMillis((long) a[1])
+        ));
+
+    static {
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ELAPSED_TIME);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ITERATION_TIME);
+    }
+
+    private final TimeValue elapsedTime;
+    private final TimeValue iterationTime;
+
+    public TimingStats(TimeValue elapsedTime, TimeValue iterationTime) {
+        this.elapsedTime = elapsedTime;
+        this.iterationTime = iterationTime;
+    }
+
+    public TimeValue getElapsedTime() {
+        return elapsedTime;
+    }
+
+    public TimeValue getIterationTime() {
+        return iterationTime;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (elapsedTime != null) {
+            builder.humanReadableField(ELAPSED_TIME.getPreferredName(), ELAPSED_TIME.getPreferredName() + "_string", elapsedTime);
+        }
+        if (iterationTime != null) {
+            builder.humanReadableField(ITERATION_TIME.getPreferredName(), ITERATION_TIME.getPreferredName() + "_string", iterationTime);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(elapsedTime, that.elapsedTime) && Objects.equals(iterationTime, that.iterationTime);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(elapsedTime, iterationTime);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLoss.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLoss.java
new file mode 100644
index 0000000000000..ee2513b0f395f
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLoss.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.client.ml.dataframe.stats.common.FoldValues;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class ValidationLoss implements ToXContentObject {
+
+    public static final ParseField LOSS_TYPE = new ParseField("loss_type");
+    public static final ParseField FOLD_VALUES = new ParseField("fold_values");
+
+    @SuppressWarnings("unchecked")
+    public static ConstructingObjectParser<ValidationLoss, Void> PARSER = new ConstructingObjectParser<>("regression_validation_loss",
+        true,
+        a -> new ValidationLoss((String) a[0], (List<FoldValues>) a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LOSS_TYPE);
+        PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), FoldValues.PARSER, FOLD_VALUES);
+    }
+
+    private final String lossType;
+    private final List<FoldValues> foldValues;
+
+    public ValidationLoss(String lossType, List<FoldValues> values) {
+        this.lossType = lossType;
+        this.foldValues = values;
+    }
+
+    public String getLossType() {
+        return lossType;
+    }
+
+    public List<FoldValues> getFoldValues() {
+        return foldValues;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (lossType != null) {
+            builder.field(LOSS_TYPE.getPreferredName(), lossType);
+        }
+        if (foldValues != null) {
+            builder.field(FOLD_VALUES.getPreferredName(), foldValues);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ValidationLoss that = (ValidationLoss) o;
+        return Objects.equals(lossType, that.lossType) && Objects.equals(foldValues, that.foldValues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(lossType, foldValues);
+    }
+}
diff --git a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
index 145d06bd46b76..9426b3d1bdde7 100644
--- a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
+++ b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -1,5 +1,6 @@
 org.elasticsearch.client.ilm.IndexLifecycleNamedXContentProvider
 org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider
 org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider
+org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider
 org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider
 org.elasticsearch.client.transform.TransformNamedXContentProvider
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
index 6c280fba5ab01..7f8ed9e61bd1f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
@@ -91,6 +91,7 @@
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider;
 import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider;
 import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
 import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider;
 import org.elasticsearch.client.ml.inference.TrainedModelConfig;
@@ -1067,6 +1068,7 @@ protected NamedXContentRegistry xContentRegistry() {
         namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
         namedXContent.addAll(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
         namedXContent.addAll(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new AnalysisStatsNamedXContentProvider().getNamedXContentParsers());
         return new NamedXContentRegistry(namedXContent);
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 468dbbd5d3ef7..f7cacdfadccef 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -69,6 +68,9 @@
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
+import org.elasticsearch.client.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.client.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.client.ml.dataframe.stats.regression.RegressionStats;
 import org.elasticsearch.client.ml.inference.preprocessing.CustomWordEmbedding;
 import org.elasticsearch.client.ml.inference.preprocessing.FrequencyEncoding;
 import org.elasticsearch.client.ml.inference.preprocessing.OneHotEncoding;
@@ -697,7 +699,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(59, namedXContents.size());
+        assertEquals(62, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -707,7 +709,7 @@ public void testProvidedNamedXContents() {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals("Had: " + categories, 12, categories.size());
+        assertEquals("Had: " + categories, 13, categories.size());
         assertEquals(Integer.valueOf(3), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
@@ -737,6 +739,9 @@ public void testProvidedNamedXContents() {
         assertTrue(names.contains(OutlierDetection.NAME.getPreferredName()));
         assertTrue(names.contains(org.elasticsearch.client.ml.dataframe.Regression.NAME.getPreferredName()));
         assertTrue(names.contains(org.elasticsearch.client.ml.dataframe.Classification.NAME.getPreferredName()));
+        assertTrue(names.contains(OutlierDetectionStats.NAME.getPreferredName()));
+        assertTrue(names.contains(RegressionStats.NAME.getPreferredName()));
+        assertTrue(names.contains(ClassificationStats.NAME.getPreferredName()));
         assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class));
         assertTrue(names.contains(TimeSyncConfig.NAME));
         assertEquals(Integer.valueOf(3), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.Evaluation.class));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
index 48ebf71e36023..25345181982af 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
@@ -20,6 +20,13 @@
 package org.elasticsearch.client.ml.dataframe;
 
 import org.elasticsearch.client.ml.NodeAttributesTests;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider;
+import org.elasticsearch.client.ml.dataframe.stats.classification.ClassificationStatsTests;
+import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsageTests;
+import org.elasticsearch.client.ml.dataframe.stats.outlierdetection.OutlierDetectionStatsTests;
+import org.elasticsearch.client.ml.dataframe.stats.regression.RegressionStatsTests;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -31,23 +38,38 @@
 
 public class DataFrameAnalyticsStatsTests extends ESTestCase {
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new AnalysisStatsNamedXContentProvider().getNamedXContentParsers());
+        return new NamedXContentRegistry(namedXContent);
+    }
+
     public void testFromXContent() throws IOException {
         xContentTester(this::createParser,
             DataFrameAnalyticsStatsTests::randomDataFrameAnalyticsStats,
             DataFrameAnalyticsStatsTests::toXContent,
             DataFrameAnalyticsStats::fromXContent)
             .supportsUnknownFields(true)
-            .randomFieldsExcludeFilter(field -> field.startsWith("node.attributes"))
+            .randomFieldsExcludeFilter(field -> field.startsWith("node.attributes") || field.startsWith("analysis_stats"))
             .test();
     }
 
     public static DataFrameAnalyticsStats randomDataFrameAnalyticsStats() {
+        AnalysisStats analysisStats = randomBoolean() ? null :
+            randomFrom(
+                ClassificationStatsTests.createRandom(),
+                OutlierDetectionStatsTests.createRandom(),
+                RegressionStatsTests.createRandom()
+            );
+
         return new DataFrameAnalyticsStats(
             randomAlphaOfLengthBetween(1, 10),
             randomFrom(DataFrameAnalyticsState.values()),
             randomBoolean() ? null : randomAlphaOfLength(10),
             randomBoolean() ? null : createRandomProgress(),
             randomBoolean() ? null : MemoryUsageTests.createRandom(),
+            analysisStats,
             randomBoolean() ? null : NodeAttributesTests.createRandom(),
             randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20));
     }
@@ -74,6 +96,11 @@ public static void toXContent(DataFrameAnalyticsStats stats, XContentBuilder bui
         if (stats.getMemoryUsage() != null) {
             builder.field(DataFrameAnalyticsStats.MEMORY_USAGE.getPreferredName(), stats.getMemoryUsage());
         }
+        if (stats.getAnalysisStats() != null) {
+            builder.startObject("analysis_stats");
+            builder.field(stats.getAnalysisStats().getName(), stats.getAnalysisStats());
+            builder.endObject();
+        }
         if (stats.getNode() != null) {
             builder.field(DataFrameAnalyticsStats.NODE.getPreferredName(), stats.getNode());
         }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStatsTests.java
new file mode 100644
index 0000000000000..d23633c01d28a
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStatsTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.time.Instant;
+
+public class ClassificationStatsTests extends AbstractXContentTestCase<ClassificationStats> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected ClassificationStats doParseInstance(XContentParser parser) throws IOException {
+        return ClassificationStats.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ClassificationStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static ClassificationStats createRandom() {
+        return new ClassificationStats(
+            Instant.now(),
+            randomBoolean() ? null : randomIntBetween(1, Integer.MAX_VALUE),
+            HyperparametersTests.createRandom(),
+            TimingStatsTests.createRandom(),
+            ValidationLossTests.createRandom()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/HyperparametersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/HyperparametersTests.java
new file mode 100644
index 0000000000000..aa1ab12c542ea
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/HyperparametersTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class HyperparametersTests extends AbstractXContentTestCase<Hyperparameters> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Hyperparameters doParseInstance(XContentParser parser) throws IOException {
+        return Hyperparameters.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected Hyperparameters createTestInstance() {
+        return createRandom();
+    }
+
+    public static Hyperparameters createRandom() {
+        return new Hyperparameters(
+            randomBoolean() ? null : randomAlphaOfLength(10),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStatsTests.java
new file mode 100644
index 0000000000000..5e2c4c842e18d
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/TimingStatsTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class TimingStatsTests extends AbstractXContentTestCase<TimingStats> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) throws IOException {
+        return TimingStats.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected TimingStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static TimingStats createRandom() {
+        return new TimingStats(
+            randomBoolean() ? null : TimeValue.timeValueMillis(randomNonNegativeLong()),
+            randomBoolean() ? null : TimeValue.timeValueMillis(randomNonNegativeLong())
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLossTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLossTests.java
new file mode 100644
index 0000000000000..c841af43d4393
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/classification/ValidationLossTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.client.ml.dataframe.stats.common.FoldValuesTests;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class ValidationLossTests extends AbstractXContentTestCase<ValidationLoss> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected ValidationLoss doParseInstance(XContentParser parser) throws IOException {
+        return ValidationLoss.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ValidationLoss createTestInstance() {
+        return createRandom();
+    }
+
+    public static ValidationLoss createRandom() {
+        return new ValidationLoss(
+            randomBoolean() ? null : randomAlphaOfLength(10),
+            randomBoolean() ? null : randomList(5, () -> FoldValuesTests.createRandom())
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValuesTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValuesTests.java
new file mode 100644
index 0000000000000..90d9219327648
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/FoldValuesTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class FoldValuesTests extends AbstractXContentTestCase<FoldValues> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected FoldValues doParseInstance(XContentParser parser) throws IOException {
+        return FoldValues.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected FoldValues createTestInstance() {
+        return createRandom();
+    }
+
+    public static FoldValues createRandom() {
+        int valuesSize = randomIntBetween(0, 10);
+        double[] values = new double[valuesSize];
+        for (int i = 0; i < valuesSize; i++) {
+            values[i] = randomDouble();
+        }
+        return new FoldValues(randomIntBetween(0, Integer.MAX_VALUE), values);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/MemoryUsageTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/MemoryUsageTests.java
similarity index 96%
rename from client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/MemoryUsageTests.java
rename to client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/MemoryUsageTests.java
index 8e06db6f2b37f..0e27295752190 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/MemoryUsageTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/MemoryUsageTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.client.ml.dataframe;
+package org.elasticsearch.client.ml.dataframe.stats.common;
 
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
new file mode 100644
index 0000000000000..f40de67a62cd2
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.time.Instant;
+
+public class OutlierDetectionStatsTests extends AbstractXContentTestCase<OutlierDetectionStats> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected OutlierDetectionStats doParseInstance(XContentParser parser) throws IOException {
+        return OutlierDetectionStats.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected OutlierDetectionStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static OutlierDetectionStats createRandom() {
+        return new OutlierDetectionStats(
+            Instant.now(),
+            ParametersTests.createRandom(),
+            TimingStatsTests.createRandom()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/ParametersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/ParametersTests.java
new file mode 100644
index 0000000000000..4f566562683de
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/ParametersTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class ParametersTests extends AbstractXContentTestCase<Parameters> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Parameters doParseInstance(XContentParser parser) throws IOException {
+        return Parameters.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected Parameters createTestInstance() {
+        return createRandom();
+    }
+
+    public static Parameters createRandom() {
+        return new Parameters(
+            randomBoolean() ? null : randomIntBetween(1, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomAlphaOfLength(5),
+            randomBoolean() ? null : randomBoolean(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomBoolean()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
new file mode 100644
index 0000000000000..5483782e1d1cf
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class TimingStatsTests extends AbstractXContentTestCase<TimingStats> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) throws IOException {
+        return TimingStats.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected TimingStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static TimingStats createRandom() {
+        return new TimingStats(randomBoolean() ? null : TimeValue.timeValueMillis(randomNonNegativeLong()));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/HyperparametersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/HyperparametersTests.java
new file mode 100644
index 0000000000000..43d0571bb206f
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/HyperparametersTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class HyperparametersTests extends AbstractXContentTestCase<Hyperparameters> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Hyperparameters doParseInstance(XContentParser parser) throws IOException {
+        return Hyperparameters.PARSER.apply(parser, null);
+    }
+
+
+    @Override
+    protected Hyperparameters createTestInstance() {
+        return createRandom();
+    }
+
+    public static Hyperparameters createRandom() {
+        return new Hyperparameters(
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomIntBetween(0, Integer.MAX_VALUE),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStatsTests.java
new file mode 100644
index 0000000000000..d4e784bb335cc
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/RegressionStatsTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.time.Instant;
+
+public class RegressionStatsTests extends AbstractXContentTestCase<RegressionStats> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected RegressionStats doParseInstance(XContentParser parser) throws IOException {
+        return RegressionStats.PARSER.apply(parser, null);
+    }
+
+
+    @Override
+    protected RegressionStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static RegressionStats createRandom() {
+        return new RegressionStats(
+            Instant.now(),
+            randomBoolean() ? null : randomIntBetween(1, Integer.MAX_VALUE),
+            HyperparametersTests.createRandom(),
+            TimingStatsTests.createRandom(),
+            ValidationLossTests.createRandom()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStatsTests.java
new file mode 100644
index 0000000000000..95fe6531f3b83
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/TimingStatsTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class TimingStatsTests extends AbstractXContentTestCase<TimingStats> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) throws IOException {
+        return TimingStats.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected TimingStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static TimingStats createRandom() {
+        return new TimingStats(
+            randomBoolean() ? null : TimeValue.timeValueMillis(randomNonNegativeLong()),
+            randomBoolean() ? null : TimeValue.timeValueMillis(randomNonNegativeLong())
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLossTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLossTests.java
new file mode 100644
index 0000000000000..d2a9f960bbbb2
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/regression/ValidationLossTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.regression;
+
+import org.elasticsearch.client.ml.dataframe.stats.common.FoldValuesTests;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class ValidationLossTests extends AbstractXContentTestCase<ValidationLoss> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected ValidationLoss doParseInstance(XContentParser parser) throws IOException {
+        return ValidationLoss.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ValidationLoss createTestInstance() {
+        return createRandom();
+    }
+
+    public static ValidationLoss createRandom() {
+        return new ValidationLoss(
+            randomBoolean() ? null : randomAlphaOfLength(10),
+            randomBoolean() ? null : randomList(5, () -> FoldValuesTests.createRandom())
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
index c0590797ead4d..370ac21020059 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
@@ -167,18 +168,23 @@ public static class Stats implements ToXContentObject, Writeable {
             @Nullable
             private final MemoryUsage memoryUsage;
 
+            @Nullable
+            private final AnalysisStats analysisStats;
+
             @Nullable
             private final DiscoveryNode node;
             @Nullable
             private final String assignmentExplanation;
 
             public Stats(String id, DataFrameAnalyticsState state, @Nullable String failureReason, List<PhaseProgress> progress,
-                         @Nullable MemoryUsage memoryUsage, @Nullable DiscoveryNode node, @Nullable String assignmentExplanation) {
+                         @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats, @Nullable DiscoveryNode node,
+                         @Nullable String assignmentExplanation) {
                 this.id = Objects.requireNonNull(id);
                 this.state = Objects.requireNonNull(state);
                 this.failureReason = failureReason;
                 this.progress = Objects.requireNonNull(progress);
                 this.memoryUsage = memoryUsage;
+                this.analysisStats = analysisStats;
                 this.node = node;
                 this.assignmentExplanation = assignmentExplanation;
             }
@@ -197,6 +203,11 @@ public Stats(StreamInput in) throws IOException {
                 } else {
                     memoryUsage = null;
                 }
+                if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                    analysisStats = in.readOptionalNamedWriteable(AnalysisStats.class);
+                } else {
+                    analysisStats = null;
+                }
                 node = in.readOptionalWriteable(DiscoveryNode::new);
                 assignmentExplanation = in.readOptionalString();
             }
@@ -285,6 +296,11 @@ public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOExc
                 if (memoryUsage != null) {
                     builder.field("memory_usage", memoryUsage);
                 }
+                if (analysisStats != null) {
+                    builder.startObject("analysis_stats");
+                    builder.field(analysisStats.getWriteableName(), analysisStats);
+                    builder.endObject();
+                }
                 if (node != null) {
                     builder.startObject("node");
                     builder.field("id", node.getId());
@@ -318,6 +334,9 @@ public void writeTo(StreamOutput out) throws IOException {
                 if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                     out.writeOptionalWriteable(memoryUsage);
                 }
+                if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                    out.writeOptionalNamedWriteable(analysisStats);
+                }
                 out.writeOptionalWriteable(node);
                 out.writeOptionalString(assignmentExplanation);
             }
@@ -350,7 +369,7 @@ private void writeProgressToLegacy(StreamOutput out) throws IOException {
 
             @Override
             public int hashCode() {
-                return Objects.hash(id, state, failureReason, progress, memoryUsage, node, assignmentExplanation);
+                return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
             }
 
             @Override
@@ -367,6 +386,7 @@ public boolean equals(Object obj) {
                         && Objects.equals(this.failureReason, other.failureReason)
                         && Objects.equals(this.progress, other.progress)
                         && Objects.equals(this.memoryUsage, other.memoryUsage)
+                        && Objects.equals(this.analysisStats, other.analysisStats)
                         && Objects.equals(this.node, other.node)
                         && Objects.equals(this.assignmentExplanation, other.assignmentExplanation);
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStats.java
new file mode 100644
index 0000000000000..94b6d265345d1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStats.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+/**
+ * Statistics for the data frame analysis
+ */
+public interface AnalysisStats extends ToXContentObject, NamedWriteable {
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStatsNamedWriteablesProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStatsNamedWriteablesProvider.java
new file mode 100644
index 0000000000000..12fd35ebe97f7
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/AnalysisStatsNamedWriteablesProvider.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class AnalysisStatsNamedWriteablesProvider {
+
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return Arrays.asList(
+            new NamedWriteableRegistry.Entry(AnalysisStats.class, ClassificationStats.TYPE_VALUE, ClassificationStats::new),
+            new NamedWriteableRegistry.Entry(AnalysisStats.class, OutlierDetectionStats.TYPE_VALUE, OutlierDetectionStats::new),
+            new NamedWriteableRegistry.Entry(AnalysisStats.class, RegressionStats.TYPE_VALUE, RegressionStats::new)
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/Fields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/Fields.java
new file mode 100644
index 0000000000000..26b2424a63cc9
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/Fields.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats;
+
+import org.elasticsearch.common.ParseField;
+
+/**
+ * A collection of parse fields commonly used by stats objects
+ */
+public final class Fields {
+
+    public static final ParseField TYPE = new ParseField("type");
+    public static final ParseField JOB_ID = new ParseField("job_id");
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+
+    private Fields() {}
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/MemoryUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/MemoryUsage.java
index 5131d88d95924..b672c3809d163 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/MemoryUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/MemoryUsage.java
@@ -26,9 +26,6 @@ public class MemoryUsage implements Writeable, ToXContentObject {
 
     public static final String TYPE_VALUE = "analytics_memory_usage";
 
-    public static final ParseField TYPE = new ParseField("type");
-    public static final ParseField JOB_ID = new ParseField("job_id");
-    public static final ParseField TIMESTAMP = new ParseField("timestamp");
     public static final ParseField PEAK_USAGE_BYTES = new ParseField("peak_usage_bytes");
 
     public static final ConstructingObjectParser<MemoryUsage, Void> STRICT_PARSER = createParser(false);
@@ -38,11 +35,11 @@ private static ConstructingObjectParser<MemoryUsage, Void> createParser(boolean
         ConstructingObjectParser<MemoryUsage, Void> parser = new ConstructingObjectParser<>(TYPE_VALUE,
             ignoreUnknownFields, a -> new MemoryUsage((String) a[0], (Instant) a[1], (long) a[2]));
 
-        parser.declareString((bucket, s) -> {}, TYPE);
-        parser.declareString(ConstructingObjectParser.constructorArg(), JOB_ID);
+        parser.declareString((bucket, s) -> {}, Fields.TYPE);
+        parser.declareString(ConstructingObjectParser.constructorArg(), Fields.JOB_ID);
         parser.declareField(ConstructingObjectParser.constructorArg(),
-            p -> TimeUtils.parseTimeFieldToInstant(p, TIMESTAMP.getPreferredName()),
-            TIMESTAMP,
+            p -> TimeUtils.parseTimeFieldToInstant(p, Fields.TIMESTAMP.getPreferredName()),
+            Fields.TIMESTAMP,
             ObjectParser.ValueType.VALUE);
         parser.declareLong(ConstructingObjectParser.constructorArg(), PEAK_USAGE_BYTES);
         return parser;
@@ -56,7 +53,7 @@ public MemoryUsage(String jobId, Instant timestamp, long peakUsageBytes) {
         this.jobId = Objects.requireNonNull(jobId);
         // We intend to store this timestamp in millis granularity. Thus we're rounding here to ensure
         // internal representation matches toXContent
-        this.timestamp = Instant.ofEpochMilli(ExceptionsHelper.requireNonNull(timestamp, TIMESTAMP).toEpochMilli());
+        this.timestamp = Instant.ofEpochMilli(ExceptionsHelper.requireNonNull(timestamp, Fields.TIMESTAMP).toEpochMilli());
         this.peakUsageBytes = peakUsageBytes;
     }
 
@@ -77,10 +74,10 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(TYPE.getPreferredName(), TYPE_VALUE);
-            builder.field(JOB_ID.getPreferredName(), jobId);
+            builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
+            builder.field(Fields.JOB_ID.getPreferredName(), jobId);
         }
-        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
         builder.field(PEAK_USAGE_BYTES.getPreferredName(), peakUsageBytes);
         builder.endObject();
         return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java
new file mode 100644
index 0000000000000..7cd2186db7986
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class ClassificationStats implements AnalysisStats {
+
+    public static final String TYPE_VALUE = "classification_stats";
+
+    public static final ParseField ITERATION = new ParseField("iteration");
+    public static final ParseField HYPERPARAMETERS = new ParseField("hyperparameters");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
+    public static final ParseField VALIDATION_LOSS = new ParseField("validation_loss");
+
+    public static final ConstructingObjectParser<ClassificationStats, Void> STRICT_PARSER = createParser(false);
+    public static final ConstructingObjectParser<ClassificationStats, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<ClassificationStats, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<ClassificationStats, Void> parser = new ConstructingObjectParser<>(TYPE_VALUE, ignoreUnknownFields,
+            a -> new ClassificationStats(
+                (String) a[0],
+                (Instant) a[1],
+                (int) a[2],
+                (Hyperparameters) a[3],
+                (TimingStats) a[4],
+                (ValidationLoss) a[5]
+            )
+        );
+
+        parser.declareString((bucket, s) -> {}, Fields.TYPE);
+        parser.declareString(ConstructingObjectParser.constructorArg(), Fields.JOB_ID);
+        parser.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtils.parseTimeFieldToInstant(p, Fields.TIMESTAMP.getPreferredName()),
+            Fields.TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        parser.declareInt(ConstructingObjectParser.constructorArg(), ITERATION);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> Hyperparameters.fromXContent(p, ignoreUnknownFields), HYPERPARAMETERS);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> TimingStats.fromXContent(p, ignoreUnknownFields), TIMING_STATS);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> ValidationLoss.fromXContent(p, ignoreUnknownFields), VALIDATION_LOSS);
+        return parser;
+    }
+
+    private final String jobId;
+    private final Instant timestamp;
+    private final int iteration;
+    private final Hyperparameters hyperparameters;
+    private final TimingStats timingStats;
+    private final ValidationLoss validationLoss;
+
+    public ClassificationStats(String jobId, Instant timestamp, int iteration, Hyperparameters hyperparameters, TimingStats timingStats,
+                               ValidationLoss validationLoss) {
+        this.jobId = Objects.requireNonNull(jobId);
+        // We intend to store this timestamp in millis granularity. Thus we're rounding here to ensure
+        // internal representation matches toXContent
+        this.timestamp = Instant.ofEpochMilli(ExceptionsHelper.requireNonNull(timestamp, Fields.TIMESTAMP).toEpochMilli());
+        this.iteration = iteration;
+        this.hyperparameters = Objects.requireNonNull(hyperparameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+        this.validationLoss = Objects.requireNonNull(validationLoss);
+    }
+
+    public ClassificationStats(StreamInput in) throws IOException {
+        this.jobId = in.readString();
+        this.timestamp = in.readInstant();
+        this.iteration = in.readVInt();
+        this.hyperparameters = new Hyperparameters(in);
+        this.timingStats = new TimingStats(in);
+        this.validationLoss = new ValidationLoss(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE_VALUE;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(jobId);
+        out.writeInstant(timestamp);
+        out.writeVInt(iteration);
+        hyperparameters.writeTo(out);
+        timingStats.writeTo(out);
+        validationLoss.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+            builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
+            builder.field(Fields.JOB_ID.getPreferredName(), jobId);
+        }
+        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.field(ITERATION.getPreferredName(), iteration);
+        builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.field(VALIDATION_LOSS.getPreferredName(), validationLoss);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ClassificationStats that = (ClassificationStats) o;
+        return Objects.equals(jobId, that.jobId)
+            && Objects.equals(timestamp, that.timestamp)
+            && iteration == that.iteration
+            && Objects.equals(hyperparameters, that.hyperparameters)
+            && Objects.equals(timingStats, that.timingStats)
+            && Objects.equals(validationLoss, that.validationLoss);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, timestamp, iteration, hyperparameters, timingStats, validationLoss);
+    }
+
+    public String documentId(String jobId) {
+        return documentIdPrefix(jobId) + timestamp.toEpochMilli();
+    }
+
+    public static String documentIdPrefix(String jobId) {
+        return TYPE_VALUE + "_" + jobId + "_";
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/Hyperparameters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/Hyperparameters.java
new file mode 100644
index 0000000000000..c7f2c901428b1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/Hyperparameters.java
@@ -0,0 +1,237 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class Hyperparameters implements ToXContentObject, Writeable {
+
+    public static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
+    public static final ParseField DOWNSAMPLE_FACTOR = new ParseField("downsample_factor");
+    public static final ParseField ETA = new ParseField("eta");
+    public static final ParseField ETA_GROWTH_RATE_PER_TREE = new ParseField("eta_growth_rate_per_tree");
+    public static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
+    public static final ParseField MAX_ATTEMPTS_TO_ADD_TREE = new ParseField("max_attempts_to_add_tree");
+    public static final ParseField MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER = new ParseField(
+        "max_optimization_rounds_per_hyperparameter");
+    public static final ParseField MAX_TREES = new ParseField("max_trees");
+    public static final ParseField NUM_FOLDS = new ParseField("num_folds");
+    public static final ParseField NUM_SPLITS_PER_FEATURE = new ParseField("num_splits_per_feature");
+    public static final ParseField REGULARIZATION_DEPTH_PENALTY_MULTIPLIER = new ParseField("regularization_depth_penalty_multiplier");
+    public static final ParseField REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER
+        = new ParseField("regularization_leaf_weight_penalty_multiplier");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_LIMIT = new ParseField("regularization_soft_tree_depth_limit");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE = new ParseField("regularization_soft_tree_depth_tolerance");
+    public static final ParseField REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER =
+        new ParseField("regularization_tree_size_penalty_multiplier");
+
+    public static Hyperparameters fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<Hyperparameters, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<Hyperparameters, Void> parser = new ConstructingObjectParser<>("classification_hyperparameters",
+            ignoreUnknownFields,
+            a -> new Hyperparameters(
+                (String) a[0],
+                (double) a[1],
+                (double) a[2],
+                (double) a[3],
+                (double) a[4],
+                (int) a[5],
+                (int) a[6],
+                (int) a[7],
+                (int) a[8],
+                (int) a[9],
+                (double) a[10],
+                (double) a[11],
+                (double) a[12],
+                (double) a[13],
+                (double) a[14]
+            ));
+
+        parser.declareString(constructorArg(), CLASS_ASSIGNMENT_OBJECTIVE);
+        parser.declareDouble(constructorArg(), DOWNSAMPLE_FACTOR);
+        parser.declareDouble(constructorArg(), ETA);
+        parser.declareDouble(constructorArg(), ETA_GROWTH_RATE_PER_TREE);
+        parser.declareDouble(constructorArg(), FEATURE_BAG_FRACTION);
+        parser.declareInt(constructorArg(), MAX_ATTEMPTS_TO_ADD_TREE);
+        parser.declareInt(constructorArg(), MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER);
+        parser.declareInt(constructorArg(), MAX_TREES);
+        parser.declareInt(constructorArg(), NUM_FOLDS);
+        parser.declareInt(constructorArg(), NUM_SPLITS_PER_FEATURE);
+        parser.declareDouble(constructorArg(), REGULARIZATION_DEPTH_PENALTY_MULTIPLIER);
+        parser.declareDouble(constructorArg(), REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER);
+        parser.declareDouble(constructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_LIMIT);
+        parser.declareDouble(constructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE);
+        parser.declareDouble(constructorArg(), REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER);
+
+        return parser;
+    }
+
+    private final String classAssignmentObjective;
+    private final double downsampleFactor;
+    private final double eta;
+    private final double etaGrowthRatePerTree;
+    private final double featureBagFraction;
+    private final int maxAttemptsToAddTree;
+    private final int maxOptimizationRoundsPerHyperparameter;
+    private final int maxTrees;
+    private final int numFolds;
+    private final int numSplitsPerFeature;
+    private final double regularizationDepthPenaltyMultiplier;
+    private final double regularizationLeafWeightPenaltyMultiplier;
+    private final double regularizationSoftTreeDepthLimit;
+    private final double regularizationSoftTreeDepthTolerance;
+    private final double regularizationTreeSizePenaltyMultiplier;
+
+    public Hyperparameters(String classAssignmentObjective,
+                           double downsampleFactor,
+                           double eta,
+                           double etaGrowthRatePerTree,
+                           double featureBagFraction,
+                           int maxAttemptsToAddTree,
+                           int maxOptimizationRoundsPerHyperparameter,
+                           int maxTrees,
+                           int numFolds,
+                           int numSplitsPerFeature,
+                           double regularizationDepthPenaltyMultiplier,
+                           double regularizationLeafWeightPenaltyMultiplier,
+                           double regularizationSoftTreeDepthLimit,
+                           double regularizationSoftTreeDepthTolerance,
+                           double regularizationTreeSizePenaltyMultiplier) {
+        this.classAssignmentObjective = Objects.requireNonNull(classAssignmentObjective);
+        this.downsampleFactor = downsampleFactor;
+        this.eta = eta;
+        this.etaGrowthRatePerTree = etaGrowthRatePerTree;
+        this.featureBagFraction = featureBagFraction;
+        this.maxAttemptsToAddTree = maxAttemptsToAddTree;
+        this.maxOptimizationRoundsPerHyperparameter = maxOptimizationRoundsPerHyperparameter;
+        this.maxTrees = maxTrees;
+        this.numFolds = numFolds;
+        this.numSplitsPerFeature = numSplitsPerFeature;
+        this.regularizationDepthPenaltyMultiplier = regularizationDepthPenaltyMultiplier;
+        this.regularizationLeafWeightPenaltyMultiplier = regularizationLeafWeightPenaltyMultiplier;
+        this.regularizationSoftTreeDepthLimit = regularizationSoftTreeDepthLimit;
+        this.regularizationSoftTreeDepthTolerance = regularizationSoftTreeDepthTolerance;
+        this.regularizationTreeSizePenaltyMultiplier = regularizationTreeSizePenaltyMultiplier;
+    }
+
+    public Hyperparameters(StreamInput in) throws IOException {
+        this.classAssignmentObjective = in.readString();
+        this.downsampleFactor = in.readDouble();
+        this.eta = in.readDouble();
+        this.etaGrowthRatePerTree = in.readDouble();
+        this.featureBagFraction = in.readDouble();
+        this.maxAttemptsToAddTree = in.readVInt();
+        this.maxOptimizationRoundsPerHyperparameter = in.readVInt();
+        this.maxTrees = in.readVInt();
+        this.numFolds = in.readVInt();
+        this.numSplitsPerFeature = in.readVInt();
+        this.regularizationDepthPenaltyMultiplier = in.readDouble();
+        this.regularizationLeafWeightPenaltyMultiplier = in.readDouble();
+        this.regularizationSoftTreeDepthLimit = in.readDouble();
+        this.regularizationSoftTreeDepthTolerance = in.readDouble();
+        this.regularizationTreeSizePenaltyMultiplier = in.readDouble();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(classAssignmentObjective);
+        out.writeDouble(downsampleFactor);
+        out.writeDouble(eta);
+        out.writeDouble(etaGrowthRatePerTree);
+        out.writeDouble(featureBagFraction);
+        out.writeVInt(maxAttemptsToAddTree);
+        out.writeVInt(maxOptimizationRoundsPerHyperparameter);
+        out.writeVInt(maxTrees);
+        out.writeVInt(numFolds);
+        out.writeVInt(numSplitsPerFeature);
+        out.writeDouble(regularizationDepthPenaltyMultiplier);
+        out.writeDouble(regularizationLeafWeightPenaltyMultiplier);
+        out.writeDouble(regularizationSoftTreeDepthLimit);
+        out.writeDouble(regularizationSoftTreeDepthTolerance);
+        out.writeDouble(regularizationTreeSizePenaltyMultiplier);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
+        builder.field(DOWNSAMPLE_FACTOR.getPreferredName(), downsampleFactor);
+        builder.field(ETA.getPreferredName(), eta);
+        builder.field(ETA_GROWTH_RATE_PER_TREE.getPreferredName(), etaGrowthRatePerTree);
+        builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
+        builder.field(MAX_ATTEMPTS_TO_ADD_TREE.getPreferredName(), maxAttemptsToAddTree);
+        builder.field(MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER.getPreferredName(), maxOptimizationRoundsPerHyperparameter);
+        builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        builder.field(NUM_FOLDS.getPreferredName(), numFolds);
+        builder.field(NUM_SPLITS_PER_FEATURE.getPreferredName(), numSplitsPerFeature);
+        builder.field(REGULARIZATION_DEPTH_PENALTY_MULTIPLIER.getPreferredName(), regularizationDepthPenaltyMultiplier);
+        builder.field(REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER.getPreferredName(), regularizationLeafWeightPenaltyMultiplier);
+        builder.field(REGULARIZATION_SOFT_TREE_DEPTH_LIMIT.getPreferredName(), regularizationSoftTreeDepthLimit);
+        builder.field(REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE.getPreferredName(), regularizationSoftTreeDepthTolerance);
+        builder.field(REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER.getPreferredName(), regularizationTreeSizePenaltyMultiplier);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Hyperparameters that = (Hyperparameters) o;
+        return Objects.equals(classAssignmentObjective, that.classAssignmentObjective)
+            && downsampleFactor == that.downsampleFactor
+            && eta == that.eta
+            && etaGrowthRatePerTree == that.etaGrowthRatePerTree
+            && featureBagFraction == that.featureBagFraction
+            && maxAttemptsToAddTree == that.maxAttemptsToAddTree
+            && maxOptimizationRoundsPerHyperparameter == that.maxOptimizationRoundsPerHyperparameter
+            && maxTrees == that.maxTrees
+            && numFolds == that.numFolds
+            && numSplitsPerFeature == that.numSplitsPerFeature
+            && regularizationDepthPenaltyMultiplier == that.regularizationDepthPenaltyMultiplier
+            && regularizationLeafWeightPenaltyMultiplier == that.regularizationLeafWeightPenaltyMultiplier
+            && regularizationSoftTreeDepthLimit == that.regularizationSoftTreeDepthLimit
+            && regularizationSoftTreeDepthTolerance == that.regularizationSoftTreeDepthTolerance
+            && regularizationTreeSizePenaltyMultiplier == that.regularizationTreeSizePenaltyMultiplier;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            classAssignmentObjective,
+            downsampleFactor,
+            eta,
+            etaGrowthRatePerTree,
+            featureBagFraction,
+            maxAttemptsToAddTree,
+            maxOptimizationRoundsPerHyperparameter,
+            maxTrees,
+            numFolds,
+            numSplitsPerFeature,
+            regularizationDepthPenaltyMultiplier,
+            regularizationLeafWeightPenaltyMultiplier,
+            regularizationSoftTreeDepthLimit,
+            regularizationSoftTreeDepthTolerance,
+            regularizationTreeSizePenaltyMultiplier
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStats.java
new file mode 100644
index 0000000000000..07245c88f20e1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStats.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class TimingStats implements Writeable, ToXContentObject {
+
+    public static final ParseField ELAPSED_TIME = new ParseField("elapsed_time");
+    public static final ParseField ITERATION_TIME = new ParseField("iteration_time");
+
+    public static TimingStats fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<TimingStats, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<TimingStats, Void> parser = new ConstructingObjectParser<>("classification_timing_stats",
+            ignoreUnknownFields,
+            a -> new TimingStats(TimeValue.timeValueMillis((long) a[0]), TimeValue.timeValueMillis((long) a[1])));
+
+        parser.declareLong(ConstructingObjectParser.constructorArg(), ELAPSED_TIME);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), ITERATION_TIME);
+        return parser;
+    }
+
+    private final TimeValue elapsedTime;
+    private final TimeValue iterationTime;
+
+    public TimingStats(TimeValue elapsedTime, TimeValue iterationTime) {
+        this.elapsedTime = Objects.requireNonNull(elapsedTime);
+        this.iterationTime = Objects.requireNonNull(iterationTime);
+    }
+
+    public TimingStats(StreamInput in) throws IOException {
+        this.elapsedTime = in.readTimeValue();
+        this.iterationTime = in.readTimeValue();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeTimeValue(elapsedTime);
+        out.writeTimeValue(iterationTime);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.humanReadableField(ELAPSED_TIME.getPreferredName(), ELAPSED_TIME.getPreferredName() + "_string", elapsedTime);
+        builder.humanReadableField(ITERATION_TIME.getPreferredName(), ITERATION_TIME.getPreferredName() + "_string", iterationTime);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(elapsedTime, that.elapsedTime) && Objects.equals(iterationTime, that.iterationTime);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(elapsedTime, iterationTime);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLoss.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLoss.java
new file mode 100644
index 0000000000000..5526ae063a433
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLoss.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.FoldValues;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class ValidationLoss implements ToXContentObject, Writeable {
+
+    public static final ParseField LOSS_TYPE = new ParseField("loss_type");
+    public static final ParseField FOLD_VALUES = new ParseField("fold_values");
+
+    public static ValidationLoss fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<ValidationLoss, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<ValidationLoss, Void> parser = new ConstructingObjectParser<>("classification_validation_loss",
+            ignoreUnknownFields,
+            a -> new ValidationLoss((String) a[0], (List<FoldValues>) a[1]));
+
+        parser.declareString(ConstructingObjectParser.constructorArg(), LOSS_TYPE);
+        parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
+            (p, c) -> FoldValues.fromXContent(p, ignoreUnknownFields), FOLD_VALUES);
+        return parser;
+    }
+
+    private final String lossType;
+    private final List<FoldValues> foldValues;
+
+    public ValidationLoss(String lossType, List<FoldValues> values) {
+        this.lossType = Objects.requireNonNull(lossType);
+        this.foldValues = Objects.requireNonNull(values);
+    }
+
+    public ValidationLoss(StreamInput in) throws IOException {
+        lossType = in.readString();
+        foldValues = in.readList(FoldValues::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(lossType);
+        out.writeList(foldValues);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(LOSS_TYPE.getPreferredName(), lossType);
+        builder.field(FOLD_VALUES.getPreferredName(), foldValues);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ValidationLoss that = (ValidationLoss) o;
+        return Objects.equals(lossType, that.lossType) && Objects.equals(foldValues, that.foldValues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(lossType, foldValues);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValues.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValues.java
new file mode 100644
index 0000000000000..1638f18fd493b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValues.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class FoldValues implements Writeable, ToXContentObject {
+
+    public static final ParseField FOLD = new ParseField("fold");
+    public static final ParseField VALUES = new ParseField("values");
+
+    public static FoldValues fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<FoldValues, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<FoldValues, Void> parser = new ConstructingObjectParser<>("fold_values", ignoreUnknownFields,
+            a -> new FoldValues((int) a[0], (List<Double>) a[1]));
+        parser.declareInt(ConstructingObjectParser.constructorArg(), FOLD);
+        parser.declareDoubleArray(ConstructingObjectParser.constructorArg(), VALUES);
+        return parser;
+    }
+
+    private final int fold;
+    private final double[] values;
+
+    private FoldValues(int fold, List<Double> values) {
+        this(fold, values.stream().mapToDouble(Double::doubleValue).toArray());
+    }
+
+    public FoldValues(int fold, double[] values) {
+        this.fold = fold;
+        this.values = values;
+    }
+
+    public FoldValues(StreamInput in) throws IOException {
+        fold = in.readVInt();
+        values = in.readDoubleArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(fold);
+        out.writeDoubleArray(values);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(FOLD.getPreferredName(), fold);
+        builder.array(VALUES.getPreferredName(), values);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FoldValues other = (FoldValues) o;
+        return fold == other.fold && Arrays.equals(values, other.values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(fold, Arrays.hashCode(values));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
new file mode 100644
index 0000000000000..e1e24f36b87b9
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class OutlierDetectionStats implements AnalysisStats {
+
+    public static final String TYPE_VALUE = "outlier_detection_stats";
+
+    public static final ParseField PARAMETERS = new ParseField("parameters");
+    public static final ParseField TIMING_STATS = new ParseField("timings_stats");
+
+    public static final ConstructingObjectParser<OutlierDetectionStats, Void> STRICT_PARSER = createParser(false);
+    public static final ConstructingObjectParser<OutlierDetectionStats, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<OutlierDetectionStats, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<OutlierDetectionStats, Void> parser = new ConstructingObjectParser<>(TYPE_VALUE, ignoreUnknownFields,
+            a -> new OutlierDetectionStats((String) a[0], (Instant) a[1], (Parameters) a[2], (TimingStats) a[3]));
+
+        parser.declareString((bucket, s) -> {}, Fields.TYPE);
+        parser.declareString(ConstructingObjectParser.constructorArg(), Fields.JOB_ID);
+        parser.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtils.parseTimeFieldToInstant(p, Fields.TIMESTAMP.getPreferredName()),
+            Fields.TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> Parameters.fromXContent(p, ignoreUnknownFields), PARAMETERS);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> TimingStats.fromXContent(p, ignoreUnknownFields), TIMING_STATS);
+        return parser;
+    }
+
+    private final String jobId;
+    private final Instant timestamp;
+    private final Parameters parameters;
+    private final TimingStats timingStats;
+
+    public OutlierDetectionStats(String jobId, Instant timestamp, Parameters parameters, TimingStats timingStats) {
+        this.jobId = Objects.requireNonNull(jobId);
+        // We intend to store this timestamp in millis granularity. Thus we're rounding here to ensure
+        // internal representation matches toXContent
+        this.timestamp = Instant.ofEpochMilli(ExceptionsHelper.requireNonNull(timestamp, Fields.TIMESTAMP).toEpochMilli());
+        this.parameters = Objects.requireNonNull(parameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+    }
+
+    public OutlierDetectionStats(StreamInput in) throws IOException {
+        this.jobId = in.readString();
+        this.timestamp = in.readInstant();
+        this.parameters = new Parameters(in);
+        this.timingStats = new TimingStats(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE_VALUE;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(jobId);
+        out.writeInstant(timestamp);
+        parameters.writeTo(out);
+        timingStats.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+            builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
+            builder.field(Fields.JOB_ID.getPreferredName(), jobId);
+        }
+        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.field(PARAMETERS.getPreferredName(), parameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        OutlierDetectionStats that = (OutlierDetectionStats) o;
+        return Objects.equals(jobId, that.jobId)
+            && Objects.equals(timestamp, that.timestamp)
+            && Objects.equals(parameters, that.parameters)
+            && Objects.equals(timingStats, that.timingStats);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, timestamp, parameters, timingStats);
+    }
+
+    public String documentId(String jobId) {
+        return documentIdPrefix(jobId) + timestamp.toEpochMilli();
+    }
+
+    public static String documentIdPrefix(String jobId) {
+        return TYPE_VALUE + "_" + jobId + "_";
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/Parameters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/Parameters.java
new file mode 100644
index 0000000000000..79c74e457b36d
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/Parameters.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class Parameters implements Writeable, ToXContentObject {
+
+    public static final ParseField N_NEIGHBORS = new ParseField("n_neighbors");
+    public static final ParseField METHOD = new ParseField("method");
+    public static final ParseField FEATURE_INFLUENCE_THRESHOLD = new ParseField("feature_influence_threshold");
+    public static final ParseField COMPUTE_FEATURE_INFLUENCE = new ParseField("compute_feature_influence");
+    public static final ParseField OUTLIER_FRACTION = new ParseField("outlier_fraction");
+    public static final ParseField STANDARDIZATION_ENABLED = new ParseField("standardization_enabled");
+
+    public static Parameters fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<Parameters, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<Parameters, Void> parser = new ConstructingObjectParser<>("outlier_detection_parameters",
+            ignoreUnknownFields,
+            a -> new Parameters(
+                (int) a[0],
+                (String) a[1],
+                (boolean) a[2],
+                (double) a[3],
+                (double) a[4],
+                (boolean) a[5]
+            ));
+
+        parser.declareInt(constructorArg(), N_NEIGHBORS);
+        parser.declareString(constructorArg(), METHOD);
+        parser.declareBoolean(constructorArg(), COMPUTE_FEATURE_INFLUENCE);
+        parser.declareDouble(constructorArg(), FEATURE_INFLUENCE_THRESHOLD);
+        parser.declareDouble(constructorArg(), OUTLIER_FRACTION);
+        parser.declareBoolean(constructorArg(), STANDARDIZATION_ENABLED);
+
+        return parser;
+    }
+
+    private final int nNeighbors;
+    private final String method;
+    private final boolean computeFeatureInfluence;
+    private final double featureInfluenceThreshold;
+    private final double outlierFraction;
+    private final boolean standardizationEnabled;
+
+    public Parameters(int nNeighbors, String method, boolean computeFeatureInfluence, double featureInfluenceThreshold,
+                      double outlierFraction, boolean standardizationEnabled) {
+        this.nNeighbors = nNeighbors;
+        this.method = method;
+        this.computeFeatureInfluence = computeFeatureInfluence;
+        this.featureInfluenceThreshold = featureInfluenceThreshold;
+        this.outlierFraction = outlierFraction;
+        this.standardizationEnabled = standardizationEnabled;
+    }
+
+    public Parameters(StreamInput in) throws IOException {
+        this.nNeighbors = in.readVInt();
+        this.method = in.readString();
+        this.computeFeatureInfluence = in.readBoolean();
+        this.featureInfluenceThreshold = in.readDouble();
+        this.outlierFraction = in.readDouble();
+        this.standardizationEnabled = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(nNeighbors);
+        out.writeString(method);
+        out.writeBoolean(computeFeatureInfluence);
+        out.writeDouble(featureInfluenceThreshold);
+        out.writeDouble(outlierFraction);
+        out.writeBoolean(standardizationEnabled);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(N_NEIGHBORS.getPreferredName(), nNeighbors);
+        builder.field(METHOD.getPreferredName(), method);
+        builder.field(COMPUTE_FEATURE_INFLUENCE.getPreferredName(), computeFeatureInfluence);
+        builder.field(FEATURE_INFLUENCE_THRESHOLD.getPreferredName(), featureInfluenceThreshold);
+        builder.field(OUTLIER_FRACTION.getPreferredName(), outlierFraction);
+        builder.field(STANDARDIZATION_ENABLED.getPreferredName(), standardizationEnabled);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Parameters that = (Parameters) o;
+        return nNeighbors == that.nNeighbors
+            && Objects.equals(method, that.method)
+            && computeFeatureInfluence == that.computeFeatureInfluence
+            && featureInfluenceThreshold == that.featureInfluenceThreshold
+            && outlierFraction == that.outlierFraction
+            && standardizationEnabled == that.standardizationEnabled;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nNeighbors, method, computeFeatureInfluence, featureInfluenceThreshold, outlierFraction,
+            standardizationEnabled);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStats.java
new file mode 100644
index 0000000000000..7721cd3fb67d2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStats.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class TimingStats implements Writeable, ToXContentObject {
+
+    public static final ParseField ELAPSED_TIME = new ParseField("elapsed_time");
+
+    public static TimingStats fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<TimingStats, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<TimingStats, Void> parser = new ConstructingObjectParser<>("outlier_detection_timing_stats",
+            ignoreUnknownFields,
+            a -> new TimingStats(TimeValue.timeValueMillis((long) a[0])));
+
+        parser.declareLong(ConstructingObjectParser.constructorArg(), ELAPSED_TIME);
+        return parser;
+    }
+
+    private final TimeValue elapsedTime;
+
+    public TimingStats(TimeValue elapsedTime) {
+        this.elapsedTime = Objects.requireNonNull(elapsedTime);
+    }
+
+    public TimingStats(StreamInput in) throws IOException {
+        this.elapsedTime = in.readTimeValue();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeTimeValue(elapsedTime);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.humanReadableField(ELAPSED_TIME.getPreferredName(), ELAPSED_TIME.getPreferredName() + "_string", elapsedTime);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(elapsedTime, that.elapsedTime);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(elapsedTime);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/Hyperparameters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/Hyperparameters.java
new file mode 100644
index 0000000000000..332aa32a48b43
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/Hyperparameters.java
@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class Hyperparameters implements ToXContentObject, Writeable {
+
+    public static final ParseField DOWNSAMPLE_FACTOR = new ParseField("downsample_factor");
+    public static final ParseField ETA = new ParseField("eta");
+    public static final ParseField ETA_GROWTH_RATE_PER_TREE = new ParseField("eta_growth_rate_per_tree");
+    public static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
+    public static final ParseField MAX_ATTEMPTS_TO_ADD_TREE = new ParseField("max_attempts_to_add_tree");
+    public static final ParseField MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER = new ParseField(
+        "max_optimization_rounds_per_hyperparameter");
+    public static final ParseField MAX_TREES = new ParseField("max_trees");
+    public static final ParseField NUM_FOLDS = new ParseField("num_folds");
+    public static final ParseField NUM_SPLITS_PER_FEATURE = new ParseField("num_splits_per_feature");
+    public static final ParseField REGULARIZATION_DEPTH_PENALTY_MULTIPLIER = new ParseField("regularization_depth_penalty_multiplier");
+    public static final ParseField REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER
+        = new ParseField("regularization_leaf_weight_penalty_multiplier");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_LIMIT = new ParseField("regularization_soft_tree_depth_limit");
+    public static final ParseField REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE = new ParseField("regularization_soft_tree_depth_tolerance");
+    public static final ParseField REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER =
+        new ParseField("regularization_tree_size_penalty_multiplier");
+
+    public static Hyperparameters fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<Hyperparameters, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<Hyperparameters, Void> parser = new ConstructingObjectParser<>("regression_hyperparameters",
+            ignoreUnknownFields,
+            a -> new Hyperparameters(
+                (double) a[0],
+                (double) a[1],
+                (double) a[2],
+                (double) a[3],
+                (int) a[4],
+                (int) a[5],
+                (int) a[6],
+                (int) a[7],
+                (int) a[8],
+                (double) a[9],
+                (double) a[10],
+                (double) a[11],
+                (double) a[12],
+                (double) a[13]
+            ));
+
+        parser.declareDouble(constructorArg(), DOWNSAMPLE_FACTOR);
+        parser.declareDouble(constructorArg(), ETA);
+        parser.declareDouble(constructorArg(), ETA_GROWTH_RATE_PER_TREE);
+        parser.declareDouble(constructorArg(), FEATURE_BAG_FRACTION);
+        parser.declareInt(constructorArg(), MAX_ATTEMPTS_TO_ADD_TREE);
+        parser.declareInt(constructorArg(), MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER);
+        parser.declareInt(constructorArg(), MAX_TREES);
+        parser.declareInt(constructorArg(), NUM_FOLDS);
+        parser.declareInt(constructorArg(), NUM_SPLITS_PER_FEATURE);
+        parser.declareDouble(constructorArg(), REGULARIZATION_DEPTH_PENALTY_MULTIPLIER);
+        parser.declareDouble(constructorArg(), REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER);
+        parser.declareDouble(constructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_LIMIT);
+        parser.declareDouble(constructorArg(), REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE);
+        parser.declareDouble(constructorArg(), REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER);
+
+        return parser;
+    }
+
+    private final double downsampleFactor;
+    private final double eta;
+    private final double etaGrowthRatePerTree;
+    private final double featureBagFraction;
+    private final int maxAttemptsToAddTree;
+    private final int maxOptimizationRoundsPerHyperparameter;
+    private final int maxTrees;
+    private final int numFolds;
+    private final int numSplitsPerFeature;
+    private final double regularizationDepthPenaltyMultiplier;
+    private final double regularizationLeafWeightPenaltyMultiplier;
+    private final double regularizationSoftTreeDepthLimit;
+    private final double regularizationSoftTreeDepthTolerance;
+    private final double regularizationTreeSizePenaltyMultiplier;
+
+    public Hyperparameters(double downsampleFactor,
+                           double eta,
+                           double etaGrowthRatePerTree,
+                           double featureBagFraction,
+                           int maxAttemptsToAddTree,
+                           int maxOptimizationRoundsPerHyperparameter,
+                           int maxTrees,
+                           int numFolds,
+                           int numSplitsPerFeature,
+                           double regularizationDepthPenaltyMultiplier,
+                           double regularizationLeafWeightPenaltyMultiplier,
+                           double regularizationSoftTreeDepthLimit,
+                           double regularizationSoftTreeDepthTolerance,
+                           double regularizationTreeSizePenaltyMultiplier) {
+        this.downsampleFactor = downsampleFactor;
+        this.eta = eta;
+        this.etaGrowthRatePerTree = etaGrowthRatePerTree;
+        this.featureBagFraction = featureBagFraction;
+        this.maxAttemptsToAddTree = maxAttemptsToAddTree;
+        this.maxOptimizationRoundsPerHyperparameter = maxOptimizationRoundsPerHyperparameter;
+        this.maxTrees = maxTrees;
+        this.numFolds = numFolds;
+        this.numSplitsPerFeature = numSplitsPerFeature;
+        this.regularizationDepthPenaltyMultiplier = regularizationDepthPenaltyMultiplier;
+        this.regularizationLeafWeightPenaltyMultiplier = regularizationLeafWeightPenaltyMultiplier;
+        this.regularizationSoftTreeDepthLimit = regularizationSoftTreeDepthLimit;
+        this.regularizationSoftTreeDepthTolerance = regularizationSoftTreeDepthTolerance;
+        this.regularizationTreeSizePenaltyMultiplier = regularizationTreeSizePenaltyMultiplier;
+    }
+
+    public Hyperparameters(StreamInput in) throws IOException {
+        this.downsampleFactor = in.readDouble();
+        this.eta = in.readDouble();
+        this.etaGrowthRatePerTree = in.readDouble();
+        this.featureBagFraction = in.readDouble();
+        this.maxAttemptsToAddTree = in.readVInt();
+        this.maxOptimizationRoundsPerHyperparameter = in.readVInt();
+        this.maxTrees = in.readVInt();
+        this.numFolds = in.readVInt();
+        this.numSplitsPerFeature = in.readVInt();
+        this.regularizationDepthPenaltyMultiplier = in.readDouble();
+        this.regularizationLeafWeightPenaltyMultiplier = in.readDouble();
+        this.regularizationSoftTreeDepthLimit = in.readDouble();
+        this.regularizationSoftTreeDepthTolerance = in.readDouble();
+        this.regularizationTreeSizePenaltyMultiplier = in.readDouble();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(downsampleFactor);
+        out.writeDouble(eta);
+        out.writeDouble(etaGrowthRatePerTree);
+        out.writeDouble(featureBagFraction);
+        out.writeVInt(maxAttemptsToAddTree);
+        out.writeVInt(maxOptimizationRoundsPerHyperparameter);
+        out.writeVInt(maxTrees);
+        out.writeVInt(numFolds);
+        out.writeVInt(numSplitsPerFeature);
+        out.writeDouble(regularizationDepthPenaltyMultiplier);
+        out.writeDouble(regularizationLeafWeightPenaltyMultiplier);
+        out.writeDouble(regularizationSoftTreeDepthLimit);
+        out.writeDouble(regularizationSoftTreeDepthTolerance);
+        out.writeDouble(regularizationTreeSizePenaltyMultiplier);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(DOWNSAMPLE_FACTOR.getPreferredName(), downsampleFactor);
+        builder.field(ETA.getPreferredName(), eta);
+        builder.field(ETA_GROWTH_RATE_PER_TREE.getPreferredName(), etaGrowthRatePerTree);
+        builder.field(FEATURE_BAG_FRACTION.getPreferredName(), featureBagFraction);
+        builder.field(MAX_ATTEMPTS_TO_ADD_TREE.getPreferredName(), maxAttemptsToAddTree);
+        builder.field(MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER.getPreferredName(), maxOptimizationRoundsPerHyperparameter);
+        builder.field(MAX_TREES.getPreferredName(), maxTrees);
+        builder.field(NUM_FOLDS.getPreferredName(), numFolds);
+        builder.field(NUM_SPLITS_PER_FEATURE.getPreferredName(), numSplitsPerFeature);
+        builder.field(REGULARIZATION_DEPTH_PENALTY_MULTIPLIER.getPreferredName(), regularizationDepthPenaltyMultiplier);
+        builder.field(REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER.getPreferredName(), regularizationLeafWeightPenaltyMultiplier);
+        builder.field(REGULARIZATION_SOFT_TREE_DEPTH_LIMIT.getPreferredName(), regularizationSoftTreeDepthLimit);
+        builder.field(REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE.getPreferredName(), regularizationSoftTreeDepthTolerance);
+        builder.field(REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER.getPreferredName(), regularizationTreeSizePenaltyMultiplier);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Hyperparameters that = (Hyperparameters) o;
+        return downsampleFactor == that.downsampleFactor
+            && eta == that.eta
+            && etaGrowthRatePerTree == that.etaGrowthRatePerTree
+            && featureBagFraction == that.featureBagFraction
+            && maxAttemptsToAddTree == that.maxAttemptsToAddTree
+            && maxOptimizationRoundsPerHyperparameter == that.maxOptimizationRoundsPerHyperparameter
+            && maxTrees == that.maxTrees
+            && numFolds == that.numFolds
+            && numSplitsPerFeature == that.numSplitsPerFeature
+            && regularizationDepthPenaltyMultiplier == that.regularizationDepthPenaltyMultiplier
+            && regularizationLeafWeightPenaltyMultiplier == that.regularizationLeafWeightPenaltyMultiplier
+            && regularizationSoftTreeDepthLimit == that.regularizationSoftTreeDepthLimit
+            && regularizationSoftTreeDepthTolerance == that.regularizationSoftTreeDepthTolerance
+            && regularizationTreeSizePenaltyMultiplier == that.regularizationTreeSizePenaltyMultiplier;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            downsampleFactor,
+            eta,
+            etaGrowthRatePerTree,
+            featureBagFraction,
+            maxAttemptsToAddTree,
+            maxOptimizationRoundsPerHyperparameter,
+            maxTrees,
+            numFolds,
+            numSplitsPerFeature,
+            regularizationDepthPenaltyMultiplier,
+            regularizationLeafWeightPenaltyMultiplier,
+            regularizationSoftTreeDepthLimit,
+            regularizationSoftTreeDepthTolerance,
+            regularizationTreeSizePenaltyMultiplier
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java
new file mode 100644
index 0000000000000..c6afc7d4f370a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class RegressionStats implements AnalysisStats {
+
+    public static final String TYPE_VALUE = "regression_stats";
+
+    public static final ParseField ITERATION = new ParseField("iteration");
+    public static final ParseField HYPERPARAMETERS = new ParseField("hyperparameters");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
+    public static final ParseField VALIDATION_LOSS = new ParseField("validation_loss");
+
+    public static final ConstructingObjectParser<RegressionStats, Void> STRICT_PARSER = createParser(false);
+    public static final ConstructingObjectParser<RegressionStats, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<RegressionStats, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<RegressionStats, Void> parser = new ConstructingObjectParser<>(TYPE_VALUE, ignoreUnknownFields,
+            a -> new RegressionStats(
+                (String) a[0],
+                (Instant) a[1],
+                (int) a[2],
+                (Hyperparameters) a[3],
+                (TimingStats) a[4],
+                (ValidationLoss) a[5]
+            )
+        );
+
+        parser.declareString((bucket, s) -> {}, Fields.TYPE);
+        parser.declareString(ConstructingObjectParser.constructorArg(), Fields.JOB_ID);
+        parser.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtils.parseTimeFieldToInstant(p, Fields.TIMESTAMP.getPreferredName()),
+            Fields.TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        parser.declareInt(ConstructingObjectParser.constructorArg(), ITERATION);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> Hyperparameters.fromXContent(p, ignoreUnknownFields), HYPERPARAMETERS);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> TimingStats.fromXContent(p, ignoreUnknownFields), TIMING_STATS);
+        parser.declareObject(ConstructingObjectParser.constructorArg(),
+            (p, c) -> ValidationLoss.fromXContent(p, ignoreUnknownFields), VALIDATION_LOSS);
+        return parser;
+    }
+
+    private final String jobId;
+    private final Instant timestamp;
+    private final int iteration;
+    private final Hyperparameters hyperparameters;
+    private final TimingStats timingStats;
+    private final ValidationLoss validationLoss;
+
+    public RegressionStats(String jobId, Instant timestamp, int iteration, Hyperparameters hyperparameters, TimingStats timingStats,
+                           ValidationLoss validationLoss) {
+        this.jobId = Objects.requireNonNull(jobId);
+        // We intend to store this timestamp in millis granularity. Thus we're rounding here to ensure
+        // internal representation matches toXContent
+        this.timestamp = Instant.ofEpochMilli(ExceptionsHelper.requireNonNull(timestamp, Fields.TIMESTAMP).toEpochMilli());
+        this.iteration = iteration;
+        this.hyperparameters = Objects.requireNonNull(hyperparameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+        this.validationLoss = Objects.requireNonNull(validationLoss);
+    }
+
+    public RegressionStats(StreamInput in) throws IOException {
+        this.jobId = in.readString();
+        this.timestamp = in.readInstant();
+        this.iteration = in.readVInt();
+        this.hyperparameters = new Hyperparameters(in);
+        this.timingStats = new TimingStats(in);
+        this.validationLoss = new ValidationLoss(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE_VALUE;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(jobId);
+        out.writeInstant(timestamp);
+        out.writeVInt(iteration);
+        hyperparameters.writeTo(out);
+        timingStats.writeTo(out);
+        validationLoss.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+            builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
+            builder.field(Fields.JOB_ID.getPreferredName(), jobId);
+        }
+        builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        builder.field(ITERATION.getPreferredName(), iteration);
+        builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.field(VALIDATION_LOSS.getPreferredName(), validationLoss);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RegressionStats that = (RegressionStats) o;
+        return Objects.equals(jobId, that.jobId)
+            && Objects.equals(timestamp, that.timestamp)
+            && iteration == that.iteration
+            && Objects.equals(hyperparameters, that.hyperparameters)
+            && Objects.equals(timingStats, that.timingStats)
+            && Objects.equals(validationLoss, that.validationLoss);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, timestamp, iteration, hyperparameters, timingStats, validationLoss);
+    }
+
+    public String documentId(String jobId) {
+        return documentIdPrefix(jobId) + timestamp.toEpochMilli();
+    }
+
+    public static String documentIdPrefix(String jobId) {
+        return TYPE_VALUE + "_" + jobId + "_";
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStats.java
new file mode 100644
index 0000000000000..8dba89f601d37
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStats.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class TimingStats implements Writeable, ToXContentObject {
+
+    public static final ParseField ELAPSED_TIME = new ParseField("elapsed_time");
+    public static final ParseField ITERATION_TIME = new ParseField("iteration_time");
+
+    public static TimingStats fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<TimingStats, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<TimingStats, Void> parser = new ConstructingObjectParser<>("regression_timing_stats", ignoreUnknownFields,
+            a -> new TimingStats(TimeValue.timeValueMillis((long) a[0]), TimeValue.timeValueMillis((long) a[1])));
+
+        parser.declareLong(ConstructingObjectParser.constructorArg(), ELAPSED_TIME);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), ITERATION_TIME);
+        return parser;
+    }
+
+    private final TimeValue elapsedTime;
+    private final TimeValue iterationTime;
+
+    public TimingStats(TimeValue elapsedTime, TimeValue iterationTime) {
+        this.elapsedTime = Objects.requireNonNull(elapsedTime);
+        this.iterationTime = Objects.requireNonNull(iterationTime);
+    }
+
+    public TimingStats(StreamInput in) throws IOException {
+        this.elapsedTime = in.readTimeValue();
+        this.iterationTime = in.readTimeValue();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeTimeValue(elapsedTime);
+        out.writeTimeValue(iterationTime);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.humanReadableField(ELAPSED_TIME.getPreferredName(), ELAPSED_TIME.getPreferredName() + "_string", elapsedTime);
+        builder.humanReadableField(ITERATION_TIME.getPreferredName(), ITERATION_TIME.getPreferredName() + "_string", iterationTime);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(elapsedTime, that.elapsedTime) && Objects.equals(iterationTime, that.iterationTime);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(elapsedTime, iterationTime);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLoss.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLoss.java
new file mode 100644
index 0000000000000..c31eb72bce0d5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLoss.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.FoldValues;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class ValidationLoss implements ToXContentObject, Writeable {
+
+    public static final ParseField LOSS_TYPE = new ParseField("loss_type");
+    public static final ParseField FOLD_VALUES = new ParseField("fold_values");
+
+    public static ValidationLoss fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return createParser(ignoreUnknownFields).apply(parser, null);
+    }
+
+    private static ConstructingObjectParser<ValidationLoss, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<ValidationLoss, Void> parser = new ConstructingObjectParser<>("regression_validation_loss",
+                ignoreUnknownFields,
+            a -> new ValidationLoss((String) a[0], (List<FoldValues>) a[1]));
+
+        parser.declareString(ConstructingObjectParser.constructorArg(), LOSS_TYPE);
+        parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
+            (p, c) -> FoldValues.fromXContent(p, ignoreUnknownFields), FOLD_VALUES);
+        return parser;
+    }
+
+    private final String lossType;
+    private final List<FoldValues> foldValues;
+
+    public ValidationLoss(String lossType, List<FoldValues> values) {
+        this.lossType = Objects.requireNonNull(lossType);
+        this.foldValues = Objects.requireNonNull(values);
+    }
+
+    public ValidationLoss(StreamInput in) throws IOException {
+        lossType = in.readString();
+        foldValues = in.readList(FoldValues::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(lossType);
+        out.writeList(foldValues);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(LOSS_TYPE.getPreferredName(), lossType);
+        builder.field(FOLD_VALUES.getPreferredName(), foldValues);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ValidationLoss that = (ValidationLoss) o;
+        return Objects.equals(lossType, that.lossType) && Objects.equals(foldValues, that.foldValues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(lossType, foldValues);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
index 5a1215057bb22..5b742c9f6d91e 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
@@ -3,18 +3,120 @@
     "_meta": {
       "version" : "${xpack.ml.version}"
     },
+    "dynamic": false,
     "properties" : {
-      "type" : {
-        "type" : "keyword"
+      "iteration": {
+        "type": "integer"
+      },
+      "hyperparameters": {
+        "properties": {
+          "class_assignment_objective": {
+            "type": "keyword"
+          },
+          "downsample_factor": {
+            "type": "double"
+          },
+          "eta": {
+            "type": "double"
+          },
+          "eta_growth_rate_per_tree": {
+            "type": "double"
+          },
+          "feature_bag_fraction": {
+            "type": "double"
+          },
+          "max_attempts_to_add_tree": {
+            "type": "integer"
+          },
+          "max_optimization_rounds_per_hyperparameter": {
+            "type": "integer"
+          },
+          "max_trees": {
+            "type": "integer"
+          },
+          "num_folds": {
+            "type": "integer"
+          },
+          "num_splits_per_feature": {
+            "type": "integer"
+          },
+          "regularization_depth_penalty_multiplier": {
+            "type": "double"
+          },
+          "regularization_leaf_weight_penalty_multiplier": {
+            "type": "double"
+          },
+          "regularization_soft_tree_depth_limit": {
+            "type": "double"
+          },
+          "regularization_soft_tree_depth_tolerance": {
+            "type": "double"
+          },
+          "regularization_tree_size_penalty_multiplier": {
+            "type": "double"
+          }
+        }
       },
       "job_id" : {
         "type" : "keyword"
       },
-      "timestamp" : {
-        "type" : "date"
+      "parameters": {
+        "properties": {
+          "compute_feature_influence": {
+            "type": "boolean"
+          },
+          "feature_influence_threshold": {
+            "type": "double"
+          },
+          "method": {
+            "type": "keyword"
+          },
+          "n_neighbors": {
+            "type": "integer"
+          },
+          "outlier_fraction": {
+            "type": "double"
+          },
+          "standardization_enabled": {
+            "type": "boolean"
+          }
+        }
       },
       "peak_usage_bytes" : {
         "type" : "long"
+      },
+      "timestamp" : {
+        "type" : "date"
+      },
+      "timing_stats": {
+        "properties": {
+          "elapsed_time": {
+            "type": "long"
+          },
+          "iteration_time": {
+            "type": "long"
+          }
+        }
+      },
+      "type" : {
+        "type" : "keyword"
+      },
+      "validation_loss": {
+        "properties": {
+          "fold_values": {
+            "properties": {
+              "fold": {
+                "type": "integer"
+              },
+              "values": {
+                "type": "double"
+              }
+            }
+          },
+          "loss_type": {
+            "type": "keyword"
+          }
+        }
       }
     }
   }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
index f5dab116b38e1..5cb2b3fef5450 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
@@ -5,14 +5,20 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Response;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStatsNamedWriteablesProvider;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsageTests;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStatsTests;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStatsTests;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStatsTests;
 import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
 
 import java.util.ArrayList;
@@ -21,6 +27,13 @@
 
 public class GetDataFrameAnalyticsStatsActionResponseTests extends AbstractWireSerializingTestCase<Response> {
 
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+        namedWriteables.addAll(new AnalysisStatsNamedWriteablesProvider().getNamedWriteables());
+        return new NamedWriteableRegistry(namedWriteables);
+    }
+
     public static Response randomResponse(int listSize) {
         List<Response.Stats> analytics = new ArrayList<>(listSize);
         for (int j = 0; j < listSize; j++) {
@@ -30,8 +43,15 @@ public static Response randomResponse(int listSize) {
             IntStream.of(progressSize).forEach(progressIndex -> progress.add(
                 new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100))));
             MemoryUsage memoryUsage = randomBoolean() ? null : MemoryUsageTests.createRandom();
+            AnalysisStats analysisStats = randomBoolean() ? null :
+                randomFrom(
+                    ClassificationStatsTests.createRandom(),
+                    OutlierDetectionStatsTests.createRandom(),
+                    RegressionStatsTests.createRandom()
+                );
             Response.Stats stats = new Response.Stats(DataFrameAnalyticsConfigTests.randomValidId(),
-                randomFrom(DataFrameAnalyticsState.values()), failureReason, progress, memoryUsage, null, randomAlphaOfLength(20));
+                randomFrom(DataFrameAnalyticsState.values()), failureReason, progress, memoryUsage, analysisStats, null,
+                randomAlphaOfLength(20));
             analytics.add(stats);
         }
         return new Response(new QueryPage<>(analytics, analytics.size(), GetDataFrameAnalyticsAction.Response.RESULTS_FIELD));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java
new file mode 100644
index 0000000000000..4aba8d77fff21
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Collections;
+
+public class ClassificationStatsTests extends AbstractBWCSerializationTestCase<ClassificationStats> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected ClassificationStats mutateInstanceForVersion(ClassificationStats instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected ClassificationStats doParseInstance(XContentParser parser) throws IOException {
+        return lenient ? ClassificationStats.LENIENT_PARSER.apply(parser, null) : ClassificationStats.STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ToXContent.Params getToXContentParams() {
+        return new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
+    }
+
+    @Override
+    protected Writeable.Reader<ClassificationStats> instanceReader() {
+        return ClassificationStats::new;
+    }
+
+    @Override
+    protected ClassificationStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static ClassificationStats createRandom() {
+        return new ClassificationStats(
+            randomAlphaOfLength(10),
+            Instant.now(),
+            randomIntBetween(1, Integer.MAX_VALUE),
+            HyperparametersTests.createRandom(),
+            TimingStatsTests.createRandom(),
+            ValidationLossTests.createRandom()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java
new file mode 100644
index 0000000000000..4c67e3c36be19
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class HyperparametersTests extends AbstractBWCSerializationTestCase<Hyperparameters> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected Hyperparameters mutateInstanceForVersion(Hyperparameters instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected Hyperparameters doParseInstance(XContentParser parser) throws IOException {
+        return Hyperparameters.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<Hyperparameters> instanceReader() {
+        return Hyperparameters::new;
+    }
+
+    @Override
+    protected Hyperparameters createTestInstance() {
+        return createRandom();
+    }
+
+    public static Hyperparameters createRandom() {
+        return new Hyperparameters(
+            randomAlphaOfLength(10),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java
new file mode 100644
index 0000000000000..cff085e7082ff
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class TimingStatsTests extends AbstractBWCSerializationTestCase<TimingStats> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected TimingStats mutateInstanceForVersion(TimingStats instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) throws IOException {
+        return TimingStats.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<TimingStats> instanceReader() {
+        return TimingStats::new;
+    }
+
+    @Override
+    protected TimingStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static TimingStats createRandom() {
+        return new TimingStats(TimeValue.timeValueMillis(randomNonNegativeLong()), TimeValue.timeValueMillis(randomNonNegativeLong()));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java
new file mode 100644
index 0000000000000..2807a9e23d932
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.classification;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.FoldValuesTests;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class ValidationLossTests extends AbstractBWCSerializationTestCase<ValidationLoss> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseStrictOrLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected ValidationLoss doParseInstance(XContentParser parser) throws IOException {
+        return ValidationLoss.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<ValidationLoss> instanceReader() {
+        return ValidationLoss::new;
+    }
+
+    @Override
+    protected ValidationLoss createTestInstance() {
+        return createRandom();
+    }
+
+    public static ValidationLoss createRandom() {
+        return new ValidationLoss(
+            randomAlphaOfLength(10),
+            randomList(5, () -> FoldValuesTests.createRandom())
+        );
+    }
+
+    @Override
+    protected ValidationLoss mutateInstanceForVersion(ValidationLoss instance, Version version) {
+        return instance;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java
new file mode 100644
index 0000000000000..929929247655a
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.common;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class FoldValuesTests extends AbstractBWCSerializationTestCase<FoldValues> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected FoldValues doParseInstance(XContentParser parser) throws IOException {
+        return FoldValues.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<FoldValues> instanceReader() {
+        return FoldValues::new;
+    }
+
+    @Override
+    protected FoldValues createTestInstance() {
+        return createRandom();
+    }
+
+    public static FoldValues createRandom() {
+        int valuesSize = randomIntBetween(0, 10);
+        double[] values = new double[valuesSize];
+        for (int i = 0; i < valuesSize; i++) {
+            values[i] = randomDouble();
+        }
+        return new FoldValues(randomIntBetween(0, Integer.MAX_VALUE), values);
+    }
+
+    @Override
+    protected FoldValues mutateInstanceForVersion(FoldValues instance, Version version) {
+        return instance;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
new file mode 100644
index 0000000000000..40c10a8a5413f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Collections;
+
+public class OutlierDetectionStatsTests extends AbstractBWCSerializationTestCase<OutlierDetectionStats> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected OutlierDetectionStats mutateInstanceForVersion(OutlierDetectionStats instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected OutlierDetectionStats doParseInstance(XContentParser parser) throws IOException {
+        return lenient ? OutlierDetectionStats.LENIENT_PARSER.apply(parser, null)
+            : OutlierDetectionStats.STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ToXContent.Params getToXContentParams() {
+        return new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
+    }
+
+    @Override
+    protected Writeable.Reader<OutlierDetectionStats> instanceReader() {
+        return OutlierDetectionStats::new;
+    }
+
+    @Override
+    protected OutlierDetectionStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static OutlierDetectionStats createRandom() {
+        return new OutlierDetectionStats(
+            randomAlphaOfLength(10),
+            Instant.now(),
+            ParametersTests.createRandom(),
+            TimingStatsTests.createRandom()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java
new file mode 100644
index 0000000000000..eb84cc30ebf09
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class ParametersTests extends AbstractBWCSerializationTestCase<Parameters> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected Parameters mutateInstanceForVersion(Parameters instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected Parameters doParseInstance(XContentParser parser) throws IOException {
+        return Parameters.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<Parameters> instanceReader() {
+        return Parameters::new;
+    }
+
+    @Override
+    protected Parameters createTestInstance() {
+        return createRandom();
+    }
+
+    public static Parameters createRandom() {
+
+        return new Parameters(
+            randomIntBetween(1, Integer.MAX_VALUE),
+            randomAlphaOfLength(5),
+            randomBoolean(),
+            randomDouble(),
+            randomDouble(),
+            randomBoolean()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
new file mode 100644
index 0000000000000..389938a712a9a
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class TimingStatsTests extends AbstractBWCSerializationTestCase<TimingStats> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected TimingStats mutateInstanceForVersion(TimingStats instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) throws IOException {
+        return TimingStats.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<TimingStats> instanceReader() {
+        return TimingStats::new;
+    }
+
+    @Override
+    protected TimingStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static TimingStats createRandom() {
+        return new TimingStats(TimeValue.timeValueMillis(randomNonNegativeLong()));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java
new file mode 100644
index 0000000000000..bebdec390ecaf
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class HyperparametersTests extends AbstractBWCSerializationTestCase<Hyperparameters> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected Hyperparameters mutateInstanceForVersion(Hyperparameters instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected Hyperparameters doParseInstance(XContentParser parser) throws IOException {
+        return Hyperparameters.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<Hyperparameters> instanceReader() {
+        return Hyperparameters::new;
+    }
+
+    @Override
+    protected Hyperparameters createTestInstance() {
+        return createRandom();
+    }
+
+    public static Hyperparameters createRandom() {
+        return new Hyperparameters(
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java
new file mode 100644
index 0000000000000..dc4cc7b7ab3b4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Collections;
+
+public class RegressionStatsTests extends AbstractBWCSerializationTestCase<RegressionStats> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected RegressionStats mutateInstanceForVersion(RegressionStats instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected RegressionStats doParseInstance(XContentParser parser) throws IOException {
+        return lenient ? RegressionStats.LENIENT_PARSER.apply(parser, null) : RegressionStats.STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ToXContent.Params getToXContentParams() {
+        return new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
+    }
+
+    @Override
+    protected Writeable.Reader<RegressionStats> instanceReader() {
+        return RegressionStats::new;
+    }
+
+    @Override
+    protected RegressionStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static RegressionStats createRandom() {
+        return new RegressionStats(
+            randomAlphaOfLength(10),
+            Instant.now(),
+            randomIntBetween(1, Integer.MAX_VALUE),
+            HyperparametersTests.createRandom(),
+            TimingStatsTests.createRandom(),
+            ValidationLossTests.createRandom()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java
new file mode 100644
index 0000000000000..ec69602d754e6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class TimingStatsTests extends AbstractBWCSerializationTestCase<TimingStats> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected TimingStats mutateInstanceForVersion(TimingStats instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) throws IOException {
+        return TimingStats.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<TimingStats> instanceReader() {
+        return TimingStats::new;
+    }
+
+    @Override
+    protected TimingStats createTestInstance() {
+        return createRandom();
+    }
+
+    public static TimingStats createRandom() {
+        return new TimingStats(TimeValue.timeValueMillis(randomNonNegativeLong()), TimeValue.timeValueMillis(randomNonNegativeLong()));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java
new file mode 100644
index 0000000000000..922428bff5e1a
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.regression;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.FoldValuesTests;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class ValidationLossTests extends AbstractBWCSerializationTestCase<ValidationLoss> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseStrictOrLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected ValidationLoss doParseInstance(XContentParser parser) throws IOException {
+        return ValidationLoss.fromXContent(parser, lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<ValidationLoss> instanceReader() {
+        return ValidationLoss::new;
+    }
+
+    @Override
+    protected ValidationLoss createTestInstance() {
+        return createRandom();
+    }
+
+    public static ValidationLoss createRandom() {
+        return new ValidationLoss(
+            randomAlphaOfLength(10),
+            randomList(5, () -> FoldValuesTests.createRandom())
+        );
+    }
+
+    @Override
+    protected ValidationLoss mutateInstanceForVersion(ValidationLoss instance, Version version) {
+        return instance;
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 384063aa2e47a..2a22a37841cd1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -136,6 +136,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStatsNamedWriteablesProvider;
 import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
@@ -974,6 +975,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             DataFrameAnalyticsTaskState::new));
 
         namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
+        namedWriteables.addAll(new AnalysisStatsNamedWriteablesProvider().getNamedWriteables());
         namedWriteables.addAll(MlEvaluationNamedXContentProvider.getNamedWriteables());
         namedWriteables.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables());
         return namedWriteables;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
index db19a5e29f116..9ee28ba29cf3a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
@@ -41,7 +41,12 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
@@ -103,7 +108,8 @@ protected void taskOperation(GetDataFrameAnalyticsStatsAction.Request request, D
                 Stats stats = buildStats(
                     task.getParams().getId(),
                     statsHolder.getProgressTracker().report(),
-                    statsHolder.getMemoryUsage()
+                    statsHolder.getMemoryUsage(),
+                    statsHolder.getAnalysisStats()
                 );
                 listener.onResponse(new QueryPage<>(Collections.singletonList(stats), 1,
                     GetDataFrameAnalyticsAction.Response.RESULTS_FIELD));
@@ -194,7 +200,10 @@ private void searchStats(String configId, ActionListener<Stats> listener) {
 
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
         multiSearchRequest.add(buildStoredProgressSearch(configId));
-        multiSearchRequest.add(buildMemoryUsageSearch(configId));
+        multiSearchRequest.add(buildStatsDocSearch(configId, MemoryUsage.TYPE_VALUE));
+        multiSearchRequest.add(buildStatsDocSearch(configId, OutlierDetectionStats.TYPE_VALUE));
+        multiSearchRequest.add(buildStatsDocSearch(configId, ClassificationStats.TYPE_VALUE));
+        multiSearchRequest.add(buildStatsDocSearch(configId, RegressionStats.TYPE_VALUE));
 
         executeAsyncWithOrigin(client, ML_ORIGIN, MultiSearchAction.INSTANCE, multiSearchRequest, ActionListener.wrap(
             multiSearchResponse -> {
@@ -215,7 +224,8 @@ private void searchStats(String configId, ActionListener<Stats> listener) {
                 }
                 listener.onResponse(buildStats(configId,
                     retrievedStatsHolder.progress.get(),
-                    retrievedStatsHolder.memoryUsage
+                    retrievedStatsHolder.memoryUsage,
+                    retrievedStatsHolder.analysisStats
                 ));
             },
             e -> listener.onFailure(ExceptionsHelper.serverError("Error searching for stats", e))
@@ -230,15 +240,15 @@ private static SearchRequest buildStoredProgressSearch(String configId) {
         return searchRequest;
     }
 
-    private static SearchRequest buildMemoryUsageSearch(String configId) {
+    private static SearchRequest buildStatsDocSearch(String configId, String statsType) {
         SearchRequest searchRequest = new SearchRequest(MlStatsIndex.indexPattern());
         searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
         searchRequest.source().size(1);
         QueryBuilder query = QueryBuilders.boolQuery()
-            .filter(QueryBuilders.termQuery(MemoryUsage.JOB_ID.getPreferredName(), configId))
-            .filter(QueryBuilders.termQuery(MemoryUsage.TYPE.getPreferredName(), MemoryUsage.TYPE_VALUE));
+            .filter(QueryBuilders.termQuery(Fields.JOB_ID.getPreferredName(), configId))
+            .filter(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), statsType));
         searchRequest.source().query(query);
-        searchRequest.source().sort(SortBuilders.fieldSort(MemoryUsage.TIMESTAMP.getPreferredName()).order(SortOrder.DESC)
+        searchRequest.source().sort(SortBuilders.fieldSort(Fields.TIMESTAMP.getPreferredName()).order(SortOrder.DESC)
             // We need this for the search not to fail when there are no mappings yet in the index
             .unmappedType("long"));
         return searchRequest;
@@ -250,6 +260,12 @@ private static void parseHit(SearchHit hit, String configId, RetrievedStatsHolde
             retrievedStatsHolder.progress = MlParserUtils.parse(hit, StoredProgress.PARSER);
         } else if (hitId.startsWith(MemoryUsage.documentIdPrefix(configId))) {
             retrievedStatsHolder.memoryUsage = MlParserUtils.parse(hit, MemoryUsage.LENIENT_PARSER);
+        } else if (hitId.startsWith(OutlierDetectionStats.documentIdPrefix(configId))) {
+            retrievedStatsHolder.analysisStats = MlParserUtils.parse(hit, OutlierDetectionStats.LENIENT_PARSER);
+        } else if (hitId.startsWith(ClassificationStats.documentIdPrefix(configId))) {
+            retrievedStatsHolder.analysisStats = MlParserUtils.parse(hit, ClassificationStats.LENIENT_PARSER);
+        } else if (hitId.startsWith(RegressionStats.documentIdPrefix(configId))) {
+            retrievedStatsHolder.analysisStats = MlParserUtils.parse(hit, RegressionStats.LENIENT_PARSER);
         } else {
             throw ExceptionsHelper.serverError("unexpected doc id [" + hitId + "]");
         }
@@ -257,7 +273,8 @@ private static void parseHit(SearchHit hit, String configId, RetrievedStatsHolde
 
     private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concreteAnalyticsId,
                                                                        List<PhaseProgress> progress,
-                                                                       MemoryUsage memoryUsage) {
+                                                                       MemoryUsage memoryUsage,
+                                                                       AnalysisStats analysisStats) {
         ClusterState clusterState = clusterService.state();
         PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
         PersistentTasksCustomMetaData.PersistentTask<?> analyticsTask = MlTasks.getDataFrameAnalyticsTask(concreteAnalyticsId, tasks);
@@ -279,6 +296,7 @@ private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concre
             failureReason,
             progress,
             memoryUsage,
+            analysisStats,
             node,
             assignmentExplanation
         );
@@ -288,5 +306,6 @@ private static class RetrievedStatsHolder {
 
         private volatile StoredProgress progress = new StoredProgress(new ProgressTracker().report());
         private volatile MemoryUsage memoryUsage;
+        private volatile AnalysisStats analysisStats;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
index d5873eb29aa23..eaa202df5fc4a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -23,6 +23,9 @@
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
@@ -176,6 +179,21 @@ private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJo
             statsHolder.setMemoryUsage(memoryUsage);
             indexStatsResult(memoryUsage, memoryUsage::documentId);
         }
+        OutlierDetectionStats outlierDetectionStats = result.getOutlierDetectionStats();
+        if (outlierDetectionStats != null) {
+            statsHolder.setAnalysisStats(outlierDetectionStats);
+            indexStatsResult(outlierDetectionStats, outlierDetectionStats::documentId);
+        }
+        ClassificationStats classificationStats = result.getClassificationStats();
+        if (classificationStats != null) {
+            statsHolder.setAnalysisStats(classificationStats);
+            indexStatsResult(classificationStats, classificationStats::documentId);
+        }
+        RegressionStats regressionStats = result.getRegressionStats();
+        if (regressionStats != null) {
+            statsHolder.setAnalysisStats(regressionStats);
+            indexStatsResult(regressionStats, regressionStats::documentId);
+        }
     }
 
     private void createAndIndexInferenceModel(TrainedModelDefinition.Builder inferenceModel) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java
index fcac851fa13ef..c3e160b7cd74c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java
@@ -12,6 +12,9 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition;
 
 import java.io.IOException;
@@ -28,9 +31,20 @@ public class AnalyticsResult implements ToXContentObject {
     private static final ParseField PROGRESS_PERCENT = new ParseField("progress_percent");
     private static final ParseField INFERENCE_MODEL = new ParseField("inference_model");
     private static final ParseField ANALYTICS_MEMORY_USAGE = new ParseField("analytics_memory_usage");
+    private static final ParseField OUTLIER_DETECTION_STATS = new ParseField("outlier_detection_stats");
+    private static final ParseField CLASSIFICATION_STATS = new ParseField("classification_stats");
+    private static final ParseField REGRESSION_STATS = new ParseField("regression_stats");
 
     public static final ConstructingObjectParser<AnalyticsResult, Void> PARSER = new ConstructingObjectParser<>(TYPE.getPreferredName(),
-            a -> new AnalyticsResult((RowResults) a[0], (Integer) a[1], (TrainedModelDefinition.Builder) a[2], (MemoryUsage) a[3]));
+            a -> new AnalyticsResult(
+                (RowResults) a[0],
+                (Integer) a[1],
+                (TrainedModelDefinition.Builder) a[2],
+                (MemoryUsage) a[3],
+                (OutlierDetectionStats) a[4],
+                (ClassificationStats) a[5],
+                (RegressionStats) a[6]
+            ));
 
     static {
         PARSER.declareObject(optionalConstructorArg(), RowResults.PARSER, RowResults.TYPE);
@@ -38,6 +52,9 @@ public class AnalyticsResult implements ToXContentObject {
         // TODO change back to STRICT_PARSER once native side is aligned
         PARSER.declareObject(optionalConstructorArg(), TrainedModelDefinition.LENIENT_PARSER, INFERENCE_MODEL);
         PARSER.declareObject(optionalConstructorArg(), MemoryUsage.STRICT_PARSER, ANALYTICS_MEMORY_USAGE);
+        PARSER.declareObject(optionalConstructorArg(), OutlierDetectionStats.STRICT_PARSER, OUTLIER_DETECTION_STATS);
+        PARSER.declareObject(optionalConstructorArg(), ClassificationStats.STRICT_PARSER, CLASSIFICATION_STATS);
+        PARSER.declareObject(optionalConstructorArg(), RegressionStats.STRICT_PARSER, REGRESSION_STATS);
     }
 
     private final RowResults rowResults;
@@ -45,16 +62,25 @@ public class AnalyticsResult implements ToXContentObject {
     private final TrainedModelDefinition.Builder inferenceModelBuilder;
     private final TrainedModelDefinition inferenceModel;
     private final MemoryUsage memoryUsage;
+    private final OutlierDetectionStats outlierDetectionStats;
+    private final ClassificationStats classificationStats;
+    private final RegressionStats regressionStats;
 
     public AnalyticsResult(@Nullable RowResults rowResults,
                            @Nullable Integer progressPercent,
                            @Nullable TrainedModelDefinition.Builder inferenceModelBuilder,
-                           @Nullable MemoryUsage memoryUsage) {
+                           @Nullable MemoryUsage memoryUsage,
+                           @Nullable OutlierDetectionStats outlierDetectionStats,
+                           @Nullable ClassificationStats classificationStats,
+                           @Nullable RegressionStats regressionStats) {
         this.rowResults = rowResults;
         this.progressPercent = progressPercent;
         this.inferenceModelBuilder = inferenceModelBuilder;
         this.inferenceModel = inferenceModelBuilder == null ? null : inferenceModelBuilder.build();
         this.memoryUsage = memoryUsage;
+        this.outlierDetectionStats = outlierDetectionStats;
+        this.classificationStats = classificationStats;
+        this.regressionStats = regressionStats;
     }
 
     public RowResults getRowResults() {
@@ -73,6 +99,18 @@ public MemoryUsage getMemoryUsage() {
         return memoryUsage;
     }
 
+    public OutlierDetectionStats getOutlierDetectionStats() {
+        return outlierDetectionStats;
+    }
+
+    public ClassificationStats getClassificationStats() {
+        return classificationStats;
+    }
+
+    public RegressionStats getRegressionStats() {
+        return regressionStats;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -90,6 +128,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (memoryUsage != null) {
             builder.field(ANALYTICS_MEMORY_USAGE.getPreferredName(), memoryUsage, params);
         }
+        if (outlierDetectionStats != null) {
+            builder.field(OUTLIER_DETECTION_STATS.getPreferredName(), outlierDetectionStats, params);
+        }
+        if (classificationStats != null) {
+            builder.field(CLASSIFICATION_STATS.getPreferredName(), classificationStats, params);
+        }
+        if (regressionStats != null) {
+            builder.field(REGRESSION_STATS.getPreferredName(), regressionStats, params);
+        }
         builder.endObject();
         return builder;
     }
@@ -107,11 +154,15 @@ public boolean equals(Object other) {
         return Objects.equals(rowResults, that.rowResults)
             && Objects.equals(progressPercent, that.progressPercent)
             && Objects.equals(inferenceModel, that.inferenceModel)
-            && Objects.equals(memoryUsage, that.memoryUsage);
+            && Objects.equals(memoryUsage, that.memoryUsage)
+            && Objects.equals(outlierDetectionStats, that.outlierDetectionStats)
+            && Objects.equals(classificationStats, that.classificationStats)
+            && Objects.equals(regressionStats, that.regressionStats);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(rowResults, progressPercent, inferenceModel, memoryUsage);
+        return Objects.hash(rowResults, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats, classificationStats,
+            regressionStats);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
index d2e9bdd957ec6..ff6b9ec7bcfef 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.dataframe.stats;
 
+import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 
 import java.util.concurrent.atomic.AtomicReference;
@@ -17,10 +18,12 @@ public class StatsHolder {
 
     private final ProgressTracker progressTracker;
     private final AtomicReference<MemoryUsage> memoryUsageHolder;
+    private final AtomicReference<AnalysisStats> analysisStatsHolder;
 
     public StatsHolder() {
         progressTracker = new ProgressTracker();
         memoryUsageHolder = new AtomicReference<>();
+        analysisStatsHolder = new AtomicReference<>();
     }
 
     public ProgressTracker getProgressTracker() {
@@ -34,4 +37,12 @@ public void setMemoryUsage(MemoryUsage memoryUsage) {
     public MemoryUsage getMemoryUsage() {
         return memoryUsageHolder.get();
     }
+
+    public void setAnalysisStats(AnalysisStats analysisStats) {
+        analysisStatsHolder.set(analysisStats);
+    }
+
+    public AnalysisStats getAnalysisStats() {
+        return analysisStatsHolder.get();
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
index 29105eca8174e..7666984d109cc 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java
@@ -56,7 +56,7 @@ public class AnalyticsProcessManagerTests extends ESTestCase {
     private static final String CONFIG_ID = "config-id";
     private static final int NUM_ROWS = 100;
     private static final int NUM_COLS = 4;
-    private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null);
+    private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null);
 
     private Client client;
     private DataFrameAnalyticsAuditor auditor;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
index 2305116343961..da7bbff71d5a7 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
@@ -100,7 +100,9 @@ public void testProcess_GivenNoResults() {
 
     public void testProcess_GivenEmptyResults() {
         givenDataFrameRows(2);
-        givenProcessResults(Arrays.asList(new AnalyticsResult(null, 50, null, null), new AnalyticsResult(null, 100, null, null)));
+        givenProcessResults(Arrays.asList(
+            new AnalyticsResult(null, 50, null, null, null, null, null),
+            new AnalyticsResult(null, 100, null, null, null, null, null)));
         AnalyticsResultProcessor resultProcessor = createResultProcessor();
 
         resultProcessor.process(process);
@@ -115,8 +117,8 @@ public void testProcess_GivenRowResults() {
         givenDataFrameRows(2);
         RowResults rowResults1 = mock(RowResults.class);
         RowResults rowResults2 = mock(RowResults.class);
-        givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null, null),
-            new AnalyticsResult(rowResults2, 100, null, null)));
+        givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null, null, null, null, null),
+            new AnalyticsResult(rowResults2, 100, null, null, null, null, null)));
         AnalyticsResultProcessor resultProcessor = createResultProcessor();
 
         resultProcessor.process(process);
@@ -133,8 +135,8 @@ public void testProcess_GivenDataFrameRowsJoinerFails() {
         givenDataFrameRows(2);
         RowResults rowResults1 = mock(RowResults.class);
         RowResults rowResults2 = mock(RowResults.class);
-        givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null, null),
-            new AnalyticsResult(rowResults2, 100, null, null)));
+        givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null, null, null, null, null),
+            new AnalyticsResult(rowResults2, 100, null, null, null, null, null)));
 
         doThrow(new RuntimeException("some failure")).when(dataFrameRowsJoiner).processRowResults(any(RowResults.class));
 
@@ -167,7 +169,7 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
         extractedFieldList.add(new MultiField("bar", new DocValueField("bar.keyword", Collections.emptySet())));
         extractedFieldList.add(new DocValueField("baz", Collections.emptySet()));
         TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
-        givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null)));
+        givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
         AnalyticsResultProcessor resultProcessor = createResultProcessor(extractedFieldList);
 
         resultProcessor.process(process);
@@ -212,7 +214,7 @@ public void testProcess_GivenInferenceModelFailedToStore() {
         }).when(trainedModelProvider).storeTrainedModel(any(TrainedModelConfig.class), any(ActionListener.class));
 
         TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
-        givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null)));
+        givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
         AnalyticsResultProcessor resultProcessor = createResultProcessor();
 
         resultProcessor.process(process);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java
index 3b949d84ed0ef..e6a158bcfead9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java
@@ -11,7 +11,14 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsageTests;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStatsTests;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStatsTests;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStatsTests;
 import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinitionTests;
@@ -36,6 +43,10 @@ protected AnalyticsResult createTestInstance() {
         RowResults rowResults = null;
         Integer progressPercent = null;
         TrainedModelDefinition.Builder inferenceModel = null;
+        MemoryUsage memoryUsage = null;
+        OutlierDetectionStats outlierDetectionStats = null;
+        ClassificationStats classificationStats = null;
+        RegressionStats regressionStats = null;
         if (randomBoolean()) {
             rowResults = RowResultsTests.createRandom();
         }
@@ -45,7 +56,20 @@ protected AnalyticsResult createTestInstance() {
         if (randomBoolean()) {
             inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
         }
-        return new AnalyticsResult(rowResults, progressPercent, inferenceModel, MemoryUsageTests.createRandom());
+        if (randomBoolean()) {
+            memoryUsage = MemoryUsageTests.createRandom();
+        }
+        if (randomBoolean()) {
+            outlierDetectionStats = OutlierDetectionStatsTests.createRandom();
+        }
+        if (randomBoolean()) {
+            classificationStats = ClassificationStatsTests.createRandom();
+        }
+        if (randomBoolean()) {
+            regressionStats = RegressionStatsTests.createRandom();
+        }
+        return new AnalyticsResult(rowResults, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats, classificationStats,
+            regressionStats);
     }
 
     @Override

From bd79cbb0bfcbc86b775944e0725f898f83275548 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Fri, 20 Mar 2020 11:14:42 +0200
Subject: [PATCH 159/285] [ML] Mute data frame analytics BWC tests (#53843)

Until #53788 is backported to 7.x.
---
 .../test/mixed_cluster/90_ml_data_frame_analytics_crud.yml   | 5 +++++
 .../test/old_cluster/90_ml_data_frame_analytics_crud.yml     | 3 +++
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml     | 5 +++++
 3 files changed, 13 insertions(+)

diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index 5049f3360a42e..dacb1e1d0a31b 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53788"
+
 ---
 "Get old outlier_detection job":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index 923a56395e8a4..dd9535d0accc0 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53788"
 
   - do:
       index:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index 3af10eb33e286..e976b2a2ce7af 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53788"
+
 ---
 "Get old cluster outlier_detection job":
 

From 8aee647300f593d075b74c1873d7692510326176 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Mar 2020 11:18:23 +0100
Subject: [PATCH 160/285] Longer Timeout in S3 Retries Test (#53841)

The lower end of the timeout range of 100ms is prone to time out
on CI before the mock REST server gets to sending a response that
is not supposed to be a timeout.
Using 1-3s here should make this safe at the cost of randomly making
this test take a few seconds.

Closes #53506
---
 .../repositories/s3/S3BlobContainerRetriesTests.java        | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
index 7060082ffcdfa..a8affd31aa857 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -170,7 +170,7 @@ public void testReadBlobWithRetries() throws Exception {
             }
         });
 
-        final TimeValue readTimeout = TimeValue.timeValueMillis(between(100, 500));
+        final TimeValue readTimeout = TimeValue.timeValueSeconds(between(1, 3));
         final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null);
         try (InputStream inputStream = blobContainer.readBlob("read_blob_max_retries")) {
             assertArrayEquals(bytes, BytesReference.toBytes(Streams.readFully(inputStream)));
@@ -397,13 +397,15 @@ private static byte[] randomBlobContent() {
         return randomByteArrayOfLength(randomIntBetween(1, frequently() ? 512 : 1 << 20)); // rarely up to 1mb
     }
 
+    private static final Pattern RANGE_PATTERN = Pattern.compile("^bytes=([0-9]+)-9223372036854775806$");
+
     private static int getRangeStart(HttpExchange exchange) {
         final String rangeHeader = exchange.getRequestHeaders().getFirst("Range");
         if (rangeHeader == null) {
             return 0;
         }
 
-        final Matcher matcher = Pattern.compile("^bytes=([0-9]+)-9223372036854775806$").matcher(rangeHeader);
+        final Matcher matcher = RANGE_PATTERN.matcher(rangeHeader);
         assertTrue(rangeHeader + " matches expected pattern", matcher.matches());
         return Math.toIntExact(Long.parseLong(matcher.group(1)));
     }

From 12046084f98721efc8b86b95ce0651ef99b46577 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 20 Mar 2020 11:22:18 +0100
Subject: [PATCH 161/285] Initial data stream commit (#53666)

* Initial data stream commit

This commits adds a data stream feature flag, initial definition of a data stream and
the stubs for the data stream create, delete and get APIs. Also simple serialization
tests are added and a rest test to thest the data stream API stubs.

This is a large amount of code and mainly mechanical, but this commit should be
straightforward to review, because there isn't any real logic.

The data stream transport and rest action are behind the data stream feature flag and
are only intialized if the feature flag is enabled. The feature flag is enabled if
elasticsearch is build as snapshot or a release build and the
'es.datastreams_feature_flag_registered' is enabled.

The integ-test-zip sets the feature flag if building a release build, otherwise
rest tests would fail.

Relates to #53100

* fixed hlrc test

* ignore bwc until this change has been backported to 7.x branch

* changed data stream apis to be a cluster based action.

before this commit the data steams api were indices based actions,
but data streams aren't indices, data streams encapsulates indices,
but are indices themselves. It is a cluster level attribute, and
therefor cluster based action fits best for now.

Perhaps in the future we will have data stream based actions and
then this would be a right fit for the data stream crud apis.

* this should have been part of the previous commit

* fixed yaml test

* Also add feature flag in other modules that run the yaml test if a release build is executed

* Reverted the commits that make data stream a cluster based api

This reverts commit e362eeb669b8d20a6bede467155c6a832e1bf1ad.

* Make data stream crud apis work like a indices based api.

* renamed timestamp field

* fixed compile error after merging in master

* fixed merge mistake

* moved setting system property

* applied review comments
---
 .../client/RestHighLevelClientTests.java      |   5 +-
 .../archives/integ-test-zip/build.gradle      |   1 +
 qa/smoke-test-multinode/build.gradle          |   2 +
 .../api/indices.create_data_stream.json       |  31 ++++
 .../api/indices.delete_data_stream.json       |  26 +++
 .../api/indices.get_data_streams.json         |  33 ++++
 .../test/indices.data_stream/10_basic.yml     |  26 +++
 .../elasticsearch/action/ActionModule.java    |  38 ++++
 .../datastream/CreateDataStreamAction.java    | 128 +++++++++++++
 .../datastream/DeleteDataStreamAction.java    | 120 +++++++++++++
 .../datastream/GetDataStreamsAction.java      | 170 ++++++++++++++++++
 .../client/IndicesAdminClient.java            |  32 ++++
 .../client/support/AbstractClient.java        |  33 ++++
 .../cluster/metadata/DataStream.java          | 116 ++++++++++++
 .../indices/RestCreateDataStreamAction.java   |  58 ++++++
 .../indices/RestDeleteDataStreamAction.java   |  47 +++++
 .../indices/RestGetDataStreamsAction.java     |  52 ++++++
 .../CreateDataStreamRequestTests.java         |  38 ++++
 .../DeleteDataStreamRequestTests.java         |  36 ++++
 .../GetDataStreamsRequestTests.java           |  36 ++++
 .../GetDataStreamsResponseTests.java          |  60 +++++++
 .../cluster/metadata/DataStreamTests.java     |  50 ++++++
 .../privilege/ClusterPrivilegeResolver.java   |   7 +-
 .../build.gradle                              |   1 +
 24 files changed, 1143 insertions(+), 3 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index f7cacdfadccef..e35adbc7aa5e4 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -798,7 +798,10 @@ public void testApiNamingConventions() throws Exception {
             "scripts_painless_execute",
             "cluster.put_component_template",
             "cluster.get_component_template",
-            "cluster.delete_component_template"
+            "cluster.delete_component_template",
+            "indices.create_data_stream",
+            "indices.get_data_streams",
+            "indices.delete_data_stream"
         };
         //These API are not required for high-level client feature completeness
         String[] notRequiredApi = new String[] {
diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle
index 5fb5b36553b91..a0884b2555caf 100644
--- a/distribution/archives/integ-test-zip/build.gradle
+++ b/distribution/archives/integ-test-zip/build.gradle
@@ -37,5 +37,6 @@ integTest.runner {
 testClusters.integTest {
   if (BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.itv2_feature_flag_registered', 'true'
+    systemProperty 'es.datastreams_feature_flag_registered', 'true'
   }
 }
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index 1a2317fbb1667..3663d63ea4b51 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+import org.elasticsearch.gradle.info.BuildParams
 
 import org.elasticsearch.gradle.info.BuildParams
 
@@ -52,5 +53,6 @@ integTest.runner {
 testClusters.integTest {
   if (BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.itv2_feature_flag_registered', 'true'
+    systemProperty 'es.datastreams_feature_flag_registered', 'true'
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
new file mode 100644
index 0000000000000..ef8615a69b1ca
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
@@ -0,0 +1,31 @@
+{
+  "indices.create_data_stream":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
+      "description":"Creates or updates a data stream"
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_data_stream/{name}",
+          "methods":[
+            "PUT"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the data stream"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+    },
+    "body":{
+      "description":"The data stream definition",
+      "required":true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
new file mode 100644
index 0000000000000..71ed5808caefc
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
@@ -0,0 +1,26 @@
+{
+  "indices.delete_data_stream":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
+      "description":"Deletes a data stream."
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_data_stream/{name}",
+          "methods":[
+            "DELETE"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the data stream"
+            }
+          }
+        }
+      ]
+    },
+    "params":{}
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json
new file mode 100644
index 0000000000000..42415068d4a5d
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json
@@ -0,0 +1,33 @@
+{
+  "indices.get_data_streams":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
+      "description":"Returns data streams."
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_data_streams",
+          "methods":[
+            "GET"
+          ]
+        },
+        {
+          "path":"/_data_streams/{name}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "name":{
+              "type":"list",
+              "description":"The comma separated names of data streams"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
new file mode 100644
index 0000000000000..49754005b6db5
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -0,0 +1,26 @@
+---
+"Test stubs":
+  - skip:
+      version: " - 7.99.99"
+      reason: not backported yet
+
+  - do:
+      indices.create_data_stream:
+        name: data-stream2
+        body:
+          timestamp_field: "@timestamp"
+  - is_true: acknowledged
+
+  - do:
+      indices.get_data_streams: {}
+  - match: { 0.name: my_data_stream1 }
+  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.indices: ['my_data_stream1-000000'] }
+  - match: { 1.name: my_data_stream2 }
+  - match: { 1.timestamp_field: '@timestamp' }
+  - match: { 1.indices: [] }
+
+  - do:
+      indices.delete_data_stream:
+        name: data-stream2
+  - is_true: acknowledged
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index aa8d92129f577..894f0805564e3 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -28,6 +28,9 @@
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction;
+import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
+import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction;
@@ -250,9 +253,11 @@
 import org.elasticsearch.rest.action.admin.cluster.RestClusterStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestClusterUpdateSettingsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestCreateSnapshotAction;
+import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction;
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteSnapshotAction;
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
+import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -265,6 +270,7 @@
 import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
+import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.RestReloadSecureSettingsAction;
@@ -388,6 +394,24 @@ public class ActionModule extends AbstractModule {
         }
     }
 
+    private static final boolean DATASTREAMS_FEATURE_FLAG_REGISTERED;
+
+    static {
+        final String property = System.getProperty("es.datastreams_feature_flag_registered");
+        if (Build.CURRENT.isSnapshot() && property != null) {
+            throw new IllegalArgumentException("es.datastreams_feature_flag_registered is only supported in non-snapshot builds");
+        }
+        if (Build.CURRENT.isSnapshot() || "true".equals(property)) {
+            DATASTREAMS_FEATURE_FLAG_REGISTERED = true;
+        } else if ("false".equals(property) || property == null) {
+            DATASTREAMS_FEATURE_FLAG_REGISTERED = false;
+        } else {
+            throw new IllegalArgumentException(
+                "expected es.datastreams_feature_flag_registered to be unset or [true|false] but was [" + property + "]"
+            );
+        }
+    }
+
     private final Settings settings;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopedSettings;
@@ -565,6 +589,13 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
 
         actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register);
 
+        // Data streams:
+        if (DATASTREAMS_FEATURE_FLAG_REGISTERED) {
+            actions.register(CreateDataStreamAction.INSTANCE, CreateDataStreamAction.TransportAction.class);
+            actions.register(DeleteDataStreamAction.INSTANCE, DeleteDataStreamAction.TransportAction.class);
+            actions.register(GetDataStreamsAction.INSTANCE, GetDataStreamsAction.TransportAction.class);
+        }
+
         // Persistent tasks:
         actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class);
         actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class);
@@ -717,6 +748,13 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestDeletePipelineAction());
         registerHandler.accept(new RestSimulatePipelineAction());
 
+        // Data Stream API
+        if (DATASTREAMS_FEATURE_FLAG_REGISTERED) {
+            registerHandler.accept(new RestCreateDataStreamAction());
+            registerHandler.accept(new RestDeleteDataStreamAction());
+            registerHandler.accept(new RestGetDataStreamsAction());
+        }
+
         // CAT API
         registerHandler.accept(new RestAllocationAction());
         registerHandler.accept(new RestShardsAction());
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
new file mode 100644
index 0000000000000..df6e829a28af4
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class CreateDataStreamAction extends ActionType<AcknowledgedResponse> {
+
+    public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction();
+    public static final String NAME = "indices:admin/data_stream/create";
+
+    private CreateDataStreamAction() {
+        super(NAME, AcknowledgedResponse::new);
+    }
+
+    public static class Request extends MasterNodeRequest<Request> {
+
+        private final String name;
+        private String timestampFieldName;
+
+        public Request(String name) {
+            this.name = name;
+        }
+
+        public void setTimestampFieldName(String timestampFieldName) {
+            this.timestampFieldName = timestampFieldName;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.name = in.readString();
+            this.timestampFieldName = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(name);
+            out.writeString(timestampFieldName);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return name.equals(request.name) &&
+                timestampFieldName.equals(request.timestampFieldName);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(name, timestampFieldName);
+        }
+    }
+
+    public static class TransportAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
+
+        @Inject
+        public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+            super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver);
+        }
+
+        @Override
+        protected String executor() {
+            return ThreadPool.Names.SAME;
+        }
+
+        @Override
+        protected AcknowledgedResponse read(StreamInput in) throws IOException {
+            return new AcknowledgedResponse(in);
+        }
+
+        @Override
+        protected void masterOperation(Task task, Request request, ClusterState state,
+                                       ActionListener<AcknowledgedResponse> listener) throws Exception {
+            listener.onResponse(new AcknowledgedResponse(true));
+        }
+
+        @Override
+        protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+            return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
new file mode 100644
index 0000000000000..20a2ba4aa2cd6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteDataStreamAction extends ActionType<AcknowledgedResponse> {
+
+    public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction();
+    public static final String NAME = "indices:admin/data_stream/delete";
+
+    private DeleteDataStreamAction() {
+        super(NAME, AcknowledgedResponse::new);
+    }
+
+    public static class Request extends MasterNodeRequest<Request> {
+
+        private final String name;
+
+        public Request(String name) {
+            this.name = Objects.requireNonNull(name);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.name = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(name);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return name.equals(request.name);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(name);
+        }
+    }
+
+    public static class TransportAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
+
+        @Inject
+        public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+            super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver);
+        }
+
+        @Override
+        protected String executor() {
+            return ThreadPool.Names.SAME;
+        }
+
+        @Override
+        protected AcknowledgedResponse read(StreamInput in) throws IOException {
+            return new AcknowledgedResponse(in);
+        }
+
+        @Override
+        protected void masterOperation(Task task, Request request, ClusterState state,
+                                       ActionListener<AcknowledgedResponse> listener) throws Exception {
+            listener.onResponse(new AcknowledgedResponse(true));
+        }
+
+        @Override
+        protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+            return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
new file mode 100644
index 0000000000000..1549f056e811f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class GetDataStreamsAction extends ActionType<GetDataStreamsAction.Response> {
+
+    public static final GetDataStreamsAction INSTANCE = new GetDataStreamsAction();
+    public static final String NAME = "indices:admin/data_stream/get";
+
+    private GetDataStreamsAction() {
+        super(NAME, Response::new);
+    }
+
+    public static class Request extends MasterNodeReadRequest<Request> {
+
+        private final String[] names;
+
+        public Request(String[] names) {
+            this.names = Objects.requireNonNull(names);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.names = in.readStringArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(names);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Arrays.equals(names, request.names);
+        }
+
+        @Override
+        public int hashCode() {
+            return Arrays.hashCode(names);
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private final List<DataStream> dataStreams;
+
+        public Response(List<DataStream> dataStreams) {
+            this.dataStreams = dataStreams;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            this(in.readList(DataStream::new));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeList(dataStreams);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startArray();
+            for (DataStream dataStream : dataStreams) {
+                dataStream.toXContent(builder, params);
+            }
+            builder.endArray();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response response = (Response) o;
+            return dataStreams.equals(response.dataStreams);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(dataStreams);
+        }
+    }
+
+    public static class TransportAction extends TransportMasterNodeReadAction<Request, Response> {
+
+        @Inject
+        public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+            super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver);
+        }
+
+        @Override
+        protected String executor() {
+            return ThreadPool.Names.SAME;
+        }
+
+        @Override
+        protected Response read(StreamInput in) throws IOException {
+            return new Response(in);
+        }
+
+        @Override
+        protected void masterOperation(Task task, Request request, ClusterState state,
+                                       ActionListener<Response> listener) throws Exception {
+            List<DataStream> dataStreams = List.of(
+                new DataStream("my_data_stream1", "@timestamp", List.of("my_data_stream1-000000")),
+                new DataStream("my_data_stream2", "@timestamp", List.of())
+            );
+            listener.onResponse(new Response(dataStreams));
+        }
+
+        @Override
+        protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+            return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
index 36b34a7b24c85..13f28dbbce210 100644
--- a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
@@ -21,6 +21,9 @@
 
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
+import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
@@ -713,4 +716,33 @@ public interface IndicesAdminClient extends ElasticsearchClient {
      */
     void rolloverIndex(RolloverRequest request, ActionListener<RolloverResponse> listener);
 
+    /**
+     * Store a data stream
+     */
+    void createDataStream(CreateDataStreamAction.Request request, ActionListener<AcknowledgedResponse> listener);
+
+    /**
+     * Store a data stream
+     */
+    ActionFuture<AcknowledgedResponse> createDataStream(CreateDataStreamAction.Request request);
+
+    /**
+     * Delete a data stream
+     */
+    void deleteDataStream(DeleteDataStreamAction.Request request, ActionListener<AcknowledgedResponse> listener);
+
+    /**
+     * Delete a data stream
+     */
+    ActionFuture<AcknowledgedResponse> deleteDataStream(DeleteDataStreamAction.Request request);
+
+    /**
+     * Get data streams
+     */
+    void getDataStreams(GetDataStreamsAction.Request request, ActionListener<GetDataStreamsAction.Response> listener);
+
+    /**
+     * Get data streams
+     */
+    ActionFuture<GetDataStreamsAction.Response> getDataStreams(GetDataStreamsAction.Request request);
 }
diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 1ee480fb55edd..669df02de212b 100644
--- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -30,6 +30,9 @@
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequestBuilder;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
+import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
+import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
@@ -1657,6 +1660,36 @@ public ActionFuture<GetSettingsResponse> getSettings(GetSettingsRequest request)
         public void getSettings(GetSettingsRequest request, ActionListener<GetSettingsResponse> listener) {
             execute(GetSettingsAction.INSTANCE, request, listener);
         }
+
+        @Override
+        public void createDataStream(CreateDataStreamAction.Request request, ActionListener<AcknowledgedResponse> listener) {
+            execute(CreateDataStreamAction.INSTANCE, request, listener);
+        }
+
+        @Override
+        public ActionFuture<AcknowledgedResponse> createDataStream(CreateDataStreamAction.Request request) {
+            return execute(CreateDataStreamAction.INSTANCE, request);
+        }
+
+        @Override
+        public void deleteDataStream(DeleteDataStreamAction.Request request, ActionListener<AcknowledgedResponse> listener) {
+            execute(DeleteDataStreamAction.INSTANCE, request, listener);
+        }
+
+        @Override
+        public ActionFuture<AcknowledgedResponse> deleteDataStream(DeleteDataStreamAction.Request request) {
+            return execute(DeleteDataStreamAction.INSTANCE, request);
+        }
+
+        @Override
+        public void getDataStreams(GetDataStreamsAction.Request request, ActionListener<GetDataStreamsAction.Response> listener) {
+            execute(GetDataStreamsAction.INSTANCE, request, listener);
+        }
+
+        @Override
+        public ActionFuture<GetDataStreamsAction.Response> getDataStreams(GetDataStreamsAction.Request request) {
+            return execute(GetDataStreamsAction.INSTANCE, request);
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
new file mode 100644
index 0000000000000..f6bba191a173b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public final class DataStream extends AbstractDiffable<DataStream> implements ToXContentObject {
+
+    private final String name;
+    private final String timeStampField;
+    private final List<String> indices;
+
+    public DataStream(String name, String timeStampField, List<String> indices) {
+        this.name = name;
+        this.timeStampField = timeStampField;
+        this.indices = indices;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getTimeStampField() {
+        return timeStampField;
+    }
+
+    public List<String> getIndices() {
+        return indices;
+    }
+
+    public DataStream(StreamInput in) throws IOException {
+        this(in.readString(), in.readString(), in.readStringList());
+    }
+
+    public static Diff<DataStream> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(DataStream::new, in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(timeStampField);
+        out.writeStringCollection(indices);
+    }
+
+    public static final ParseField NAME_FIELD = new ParseField("name");
+    public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
+    public static final ParseField INDICES_FIELD = new ParseField("indices");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
+        args -> new DataStream((String) args[0], (String) args[1], (List<String>) args[2]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), TIMESTAMP_FIELD_FIELD);
+        PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD);
+    }
+
+    public static DataStream fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(NAME_FIELD.getPreferredName(), name);
+        builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName(), timeStampField);
+        builder.field(INDICES_FIELD.getPreferredName(), indices);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataStream that = (DataStream) o;
+        return name.equals(that.name) &&
+            timeStampField.equals(that.timeStampField) &&
+            indices.equals(that.indices);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, timeStampField, indices);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java
new file mode 100644
index 0000000000000..67b67677195dc
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class RestCreateDataStreamAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "create_data_stream_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(RestRequest.Method.PUT, "/_data_stream/{name}")
+        );
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        CreateDataStreamAction.Request putDataStreamRequest = new CreateDataStreamAction.Request(request.param("name"));
+        request.withContentOrSourceParamParserOrNull(parser -> {
+            Map<String, Object> body = parser.map();
+            String timeStampFieldName = (String) body.get(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName());
+            if (timeStampFieldName != null) {
+                putDataStreamRequest.setTimestampFieldName(timeStampFieldName);
+            }
+        });
+        return channel -> client.admin().indices().createDataStream(putDataStreamRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java
new file mode 100644
index 0000000000000..b69ee981d12b3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+public class RestDeleteDataStreamAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "delete_data_stream_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(RestRequest.Method.DELETE, "/_data_stream/{name}"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(request.param("name"));
+        return channel -> client.admin().indices().deleteDataStream(deleteDataStreamRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java
new file mode 100644
index 0000000000000..fdbe63829882e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+public class RestGetDataStreamsAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "get_data_streams_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(RestRequest.Method.GET, "/_data_streams"),
+            new Route(RestRequest.Method.GET, "/_data_streams/{name}")
+        );
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String[] names = Strings.splitStringByCommaToArray(request.param("name"));
+        GetDataStreamsAction.Request getDataStreamsRequest = new GetDataStreamsAction.Request(names);
+        return channel -> client.admin().indices().getDataStreams(getDataStreamsRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
new file mode 100644
index 0000000000000..d6a846c205fb3
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction.Request;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class CreateDataStreamRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request(randomAlphaOfLength(8));
+        request.setTimestampFieldName(randomAlphaOfLength(8));
+        return request;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
new file mode 100644
index 0000000000000..f460065699795
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction.Request;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(randomAlphaOfLength(8));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
new file mode 100644
index 0000000000000..062bdef629cc9
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction.Request;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(generateRandomStringArray(8, 8, false));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java
new file mode 100644
index 0000000000000..5c74c515634cc
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.datastream;
+
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction.Response;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class GetDataStreamsResponseTests extends AbstractSerializingTestCase<Response> {
+
+    @Override
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
+    }
+
+    @Override
+    protected Response doParseInstance(XContentParser parser) throws IOException {
+        List<DataStream> dataStreams = new ArrayList<>();
+        for (Token token = parser.nextToken(); token != Token.END_ARRAY; token = parser.nextToken()) {
+            if (token == Token.START_OBJECT) {
+                dataStreams.add(DataStream.fromXContent(parser));
+            }
+        }
+        return new Response(dataStreams);
+    }
+
+    @Override
+    protected Response createTestInstance() {
+        int numDataStreams = randomIntBetween(0, 8);
+        List<DataStream> dataStreams = new ArrayList<>();
+        for (int i = 0; i < numDataStreams; i++) {
+            dataStreams.add(new DataStream(randomAlphaOfLength(4), randomAlphaOfLength(4),
+                List.of(generateRandomStringArray(8, 4, false))));
+        }
+        return new Response(dataStreams);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
new file mode 100644
index 0000000000000..072165ab098ef
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
+
+    @Override
+    protected DataStream doParseInstance(XContentParser parser) throws IOException {
+        return DataStream.fromXContent(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<DataStream> instanceReader() {
+        return DataStream::new;
+    }
+
+    @Override
+    protected DataStream createTestInstance() {
+        int numIndices = randomIntBetween(0, 128);
+        List<String> indices = new ArrayList<>(numIndices);
+        for (int i = 0; i < numIndices; i++) {
+            indices.add(randomAlphaOfLength(10));
+        }
+        return new DataStream(randomAlphaOfLength(10), randomAlphaOfLength(10), indices);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
index 7fa41f033ea44..23023001e27f1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
@@ -55,7 +55,7 @@ public class ClusterPrivilegeResolver {
     private static final Set<String> MONITOR_TRANSFORM_PATTERN = Set.of("cluster:monitor/data_frame/*", "cluster:monitor/transform/*");
     private static final Set<String> MONITOR_WATCHER_PATTERN = Set.of("cluster:monitor/xpack/watcher/*");
     private static final Set<String> MONITOR_ROLLUP_PATTERN = Set.of("cluster:monitor/xpack/rollup/*");
-    private static final Set<String> ALL_CLUSTER_PATTERN = Set.of("cluster:*", "indices:admin/template/*");
+    private static final Set<String> ALL_CLUSTER_PATTERN = Set.of("cluster:*", "indices:admin/template/*", "indices:admin/data_stream/*");
     private static final Set<String> MANAGE_ML_PATTERN = Set.of("cluster:admin/xpack/ml/*", "cluster:monitor/xpack/ml/*");
     private static final Set<String> MANAGE_TRANSFORM_PATTERN = Set.of("cluster:admin/data_frame/*", "cluster:monitor/data_frame/*",
             "cluster:monitor/transform/*", "cluster:admin/transform/*");
@@ -197,7 +197,10 @@ public static Set<String> names() {
     }
 
     public static boolean isClusterAction(String actionName) {
-        return actionName.startsWith("cluster:") || actionName.startsWith("indices:admin/template/");
+        return actionName.startsWith("cluster:") ||
+            actionName.startsWith("indices:admin/template/") ||
+            // todo: hack until we implement security of data_streams
+            actionName.startsWith("indices:admin/data_stream/");
     }
 
     private static String actionToPattern(String text) {
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 139fb7cd2cabe..651c6af0308a2 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -43,5 +43,6 @@ testClusters.integTest {
 testClusters.integTest {
   if (BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.itv2_feature_flag_registered', 'true'
+    systemProperty 'es.datastreams_feature_flag_registered', 'true'
   }
 }

From bb582f1317f1e76ee36bd862c3e46ee1a9ee9f7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= <przemyslaw.witek@elastic.co>
Date: Fri, 20 Mar 2020 11:29:45 +0100
Subject: [PATCH 162/285] Delete empty .ml-state* indices during nightly
 maintenance task. (#53587)

---
 .../ml/integration/DeleteExpiredDataIT.java   |  42 +++++
 .../TransportDeleteExpiredDataAction.java     |   4 +-
 .../job/retention/EmptyStateIndexRemover.java | 109 +++++++++++
 .../ml/job/retention/UnusedStateRemover.java  |   4 +-
 .../EmptyStateIndexRemoverTests.java          | 176 ++++++++++++++++++
 5 files changed, 332 insertions(+), 3 deletions(-)
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemoverTests.java

diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
index 017c11f4b3b10..cdd12a8fe10e5 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
@@ -6,6 +6,8 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -14,6 +16,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -41,6 +44,7 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -277,6 +281,44 @@ public void testDeleteExpiredData() throws Exception {
             nonExistingJobDocsCount, equalTo(0));
     }
 
+    /**
+     * Verifies empty state indices deletion. Here is the summary of indices used by the test:
+     *
+     * +------------------+--------+----------+-------------------------+
+     * | index name       | empty? | current? | expected to be removed? |
+     * +------------------+--------+----------+-------------------------+
+     * | .ml-state        | yes    | no       | yes                     |
+     * | .ml-state-000001 | no     | no       | no                      |
+     * | .ml-state-000003 | yes    | no       | yes                     |
+     * | .ml-state-000005 | no     | no       | no                      |
+     * | .ml-state-000007 | yes    | yes      | no                      |
+     * +------------------+--------+----------+-------------------------+
+     */
+    public void testDeleteExpiredDataActionDeletesEmptyStateIndices() throws Exception {
+        client().admin().indices().prepareCreate(".ml-state").get();
+        client().admin().indices().prepareCreate(".ml-state-000001").get();
+        client().prepareIndex(".ml-state-000001").setSource("field_1", "value_1").get();
+        client().admin().indices().prepareCreate(".ml-state-000003").get();
+        client().admin().indices().prepareCreate(".ml-state-000005").get();
+        client().prepareIndex(".ml-state-000005").setSource("field_5", "value_5").get();
+        client().admin().indices().prepareCreate(".ml-state-000007").addAlias(new Alias(".ml-state-write").isHidden(true)).get();
+        refresh();
+
+        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(".ml-state*").get();
+        assertThat(Strings.toString(getIndexResponse),
+            getIndexResponse.getIndices(),
+            is(arrayContaining(".ml-state", ".ml-state-000001", ".ml-state-000003", ".ml-state-000005", ".ml-state-000007")));
+
+        client().execute(DeleteExpiredDataAction.INSTANCE, new DeleteExpiredDataAction.Request()).get();
+        refresh();
+
+        getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(".ml-state*").get();
+        assertThat(Strings.toString(getIndexResponse),
+            getIndexResponse.getIndices(),
+            // Only non-empty or current indices should survive deletion process
+            is(arrayContaining(".ml-state-000001", ".ml-state-000005", ".ml-state-000007")));
+    }
+
     private static Job.Builder newJobBuilder(String id) {
         Detector.Builder detector = new Detector.Builder();
         detector.setFunction("count");
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
index 273fe202e478b..fe3d1a73c89d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.action.DeleteExpiredDataAction;
 import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.job.retention.EmptyStateIndexRemover;
 import org.elasticsearch.xpack.ml.job.retention.ExpiredForecastsRemover;
 import org.elasticsearch.xpack.ml.job.retention.ExpiredModelSnapshotsRemover;
 import org.elasticsearch.xpack.ml.job.retention.ExpiredResultsRemover;
@@ -84,7 +85,8 @@ private void deleteExpiredData(ActionListener<DeleteExpiredDataAction.Response>
                 new ExpiredResultsRemover(client, auditor, threadPool),
                 new ExpiredForecastsRemover(client, threadPool),
                 new ExpiredModelSnapshotsRemover(client, threadPool),
-                new UnusedStateRemover(client, clusterService)
+                new UnusedStateRemover(client, clusterService),
+                new EmptyStateIndexRemover(client)
         );
         Iterator<MlDataRemover> dataRemoversIterator = new VolatileCursorIterator<>(dataRemovers);
         deleteExpiredData(dataRemoversIterator, listener, isTimedOutSupplier, true);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java
new file mode 100644
index 0000000000000..8b8fbe0f26325
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.retention;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
+
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import static java.util.stream.Collectors.toSet;
+
+/**
+ * This class deletes empty indices matching .ml-state* pattern that are not pointed at by the .ml-state-write alias.
+ */
+public class EmptyStateIndexRemover implements MlDataRemover {
+    
+    private final OriginSettingClient client;
+
+    public EmptyStateIndexRemover(OriginSettingClient client) {
+        this.client = Objects.requireNonNull(client);
+    }
+
+    @Override
+    public void remove(ActionListener<Boolean> listener, Supplier<Boolean> isTimedOutSupplier) {
+        try {
+            if (isTimedOutSupplier.get()) {
+                listener.onResponse(false);
+                return;
+            }
+            getEmptyStateIndices(
+                ActionListener.wrap(
+                    emptyStateIndices -> {
+                        if (emptyStateIndices.isEmpty()) {
+                            listener.onResponse(true);
+                            return;
+                        }
+                        getCurrentStateIndices(
+                            ActionListener.wrap(
+                                currentStateIndices -> {
+                                    Set<String> stateIndicesToRemove = Sets.difference(emptyStateIndices, currentStateIndices);
+                                    if (stateIndicesToRemove.isEmpty()) {
+                                        listener.onResponse(true);
+                                        return;
+                                    }
+                                    executeDeleteEmptyStateIndices(stateIndicesToRemove, listener);
+                                },
+                                listener::onFailure
+                            )
+                        );
+                    },
+                    listener::onFailure
+                )
+            );
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
+    }
+
+    private void getEmptyStateIndices(ActionListener<Set<String>> listener) {
+        IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest().indices(AnomalyDetectorsIndex.jobStateIndexPattern());
+        client.admin().indices().stats(
+            indicesStatsRequest,
+            ActionListener.wrap(
+                indicesStatsResponse -> {
+                    Set<String> emptyStateIndices =
+                        indicesStatsResponse.getIndices().values().stream()
+                            .filter(stats -> stats.getTotal().getDocs().getCount() == 0)
+                            .map(IndexStats::getIndex)
+                            .collect(toSet());
+                    listener.onResponse(emptyStateIndices);
+                },
+                listener::onFailure
+            )
+        );
+    }
+
+    private void getCurrentStateIndices(ActionListener<Set<String>> listener) {
+        GetIndexRequest getIndexRequest = new GetIndexRequest().indices(AnomalyDetectorsIndex.jobStateIndexWriteAlias());
+        client.admin().indices().getIndex(
+            getIndexRequest,
+            ActionListener.wrap(
+                getIndexResponse -> listener.onResponse(Set.of(getIndexResponse.getIndices())),
+                listener::onFailure
+            )
+        );
+    }
+
+    private void executeDeleteEmptyStateIndices(Set<String> emptyStateIndices, ActionListener<Boolean> listener) {
+        DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(emptyStateIndices.toArray(new String[0]));
+        client.admin().indices().delete(
+            deleteIndexRequest,
+            ActionListener.wrap(
+                deleteResponse -> listener.onResponse(deleteResponse.isAcknowledged()),
+                listener::onFailure
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
index cf1a9aaae4ede..6d0a9de1569db 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
@@ -41,9 +41,9 @@
 import java.util.function.Supplier;
 
 /**
- * If for any reason a job is deleted by some of its state documents
+ * If for any reason a job is deleted but some of its state documents
  * are left behind, this class deletes any unused documents stored
- * in the .ml-state index.
+ * in the .ml-state* indices.
  */
 public class UnusedStateRemover implements MlDataRemover {
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemoverTests.java
new file mode 100644
index 0000000000000..ea6d829be4c77
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemoverTests.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.retention;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexAction;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.ml.test.MockOriginSettingClient;
+import org.junit.After;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.stubbing.Answer;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class EmptyStateIndexRemoverTests extends ESTestCase {
+
+    private Client client;
+    private ActionListener<Boolean> listener;
+    private ArgumentCaptor<DeleteIndexRequest> deleteIndexRequestCaptor;
+
+    private EmptyStateIndexRemover remover;
+
+    @SuppressWarnings("unchecked")
+    @Before
+    public void setUpTests() {
+        client = mock(Client.class);
+        OriginSettingClient originSettingClient = MockOriginSettingClient.mockOriginSettingClient(client, ClientHelper.ML_ORIGIN);
+        listener = mock(ActionListener.class);
+        deleteIndexRequestCaptor = ArgumentCaptor.forClass(DeleteIndexRequest.class);
+
+        remover = new EmptyStateIndexRemover(originSettingClient);
+    }
+
+    @After
+    public void verifyNoOtherInteractionsWithMocks() {
+        verify(client).settings();
+        verify(client, atLeastOnce()).threadPool();
+        verifyNoMoreInteractions(client, listener);
+    }
+
+    public void testRemove_TimedOut() {
+        remover.remove(listener, () -> true);
+
+        InOrder inOrder = inOrder(client, listener);
+        inOrder.verify(listener).onResponse(false);
+    }
+
+    public void testRemove_NoStateIndices() {
+        IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
+        when(indicesStatsResponse.getIndices()).thenReturn(Map.of());
+        doAnswer(withResponse(indicesStatsResponse)).when(client).execute(any(), any(), any());
+
+        remover.remove(listener, () -> false);
+
+        InOrder inOrder = inOrder(client, listener);
+        inOrder.verify(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+        inOrder.verify(listener).onResponse(true);
+    }
+
+    public void testRemove_NoEmptyStateIndices() {
+        IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
+        doReturn(
+            Map.of(
+                ".ml-state-a", indexStats(".ml-state-a", 1),
+                ".ml-state-b", indexStats(".ml-state-b", 2),
+                ".ml-state-c", indexStats(".ml-state-c", 1),
+                ".ml-state-d", indexStats(".ml-state-d", 2))).when(indicesStatsResponse).getIndices();
+        doAnswer(withResponse(indicesStatsResponse)).when(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+
+        remover.remove(listener, () -> false);
+
+        InOrder inOrder = inOrder(client, listener);
+        inOrder.verify(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+        inOrder.verify(listener).onResponse(true);
+    }
+
+    private void assertDeleteActionExecuted(boolean acknowledged) {
+        IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
+        doReturn(
+            Map.of(
+                ".ml-state-a", indexStats(".ml-state-a", 1),
+                ".ml-state-b", indexStats(".ml-state-b", 0),
+                ".ml-state-c", indexStats(".ml-state-c", 2),
+                ".ml-state-d", indexStats(".ml-state-d", 0),
+                ".ml-state-e", indexStats(".ml-state-e", 0))).when(indicesStatsResponse).getIndices();
+        doAnswer(withResponse(indicesStatsResponse)).when(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+
+        GetIndexResponse getIndexResponse = new GetIndexResponse(new String[] { ".ml-state-e" }, null, null, null, null);
+        doAnswer(withResponse(getIndexResponse)).when(client).execute(eq(GetIndexAction.INSTANCE), any(), any());
+
+        AcknowledgedResponse deleteIndexResponse = new AcknowledgedResponse(acknowledged);
+        doAnswer(withResponse(deleteIndexResponse)).when(client).execute(eq(DeleteIndexAction.INSTANCE), any(), any());
+
+        remover.remove(listener, () -> false);
+
+        InOrder inOrder = inOrder(client, listener);
+        inOrder.verify(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+        inOrder.verify(client).execute(eq(GetIndexAction.INSTANCE), any(), any());
+        inOrder.verify(client).execute(eq(DeleteIndexAction.INSTANCE), deleteIndexRequestCaptor.capture(), any());
+        inOrder.verify(listener).onResponse(acknowledged);
+
+        DeleteIndexRequest deleteIndexRequest = deleteIndexRequestCaptor.getValue();
+        assertThat(deleteIndexRequest.indices(), arrayContainingInAnyOrder(".ml-state-b", ".ml-state-d"));
+    }
+
+    public void testRemove_DeleteAcknowledged() {
+        assertDeleteActionExecuted(true);
+    }
+
+    public void testRemove_DeleteNotAcknowledged() {
+        assertDeleteActionExecuted(false);
+    }
+
+    public void testRemove_NoIndicesToRemove() {
+        IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
+        doReturn(Map.of(".ml-state-a", indexStats(".ml-state-a", 0))).when(indicesStatsResponse).getIndices();
+        doAnswer(withResponse(indicesStatsResponse)).when(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+
+        GetIndexResponse getIndexResponse = new GetIndexResponse(new String[] { ".ml-state-a" }, null, null, null, null);
+        doAnswer(withResponse(getIndexResponse)).when(client).execute(eq(GetIndexAction.INSTANCE), any(), any());
+
+        remover.remove(listener, () -> false);
+
+        InOrder inOrder = inOrder(client, listener);
+        inOrder.verify(client).execute(eq(IndicesStatsAction.INSTANCE), any(), any());
+        inOrder.verify(client).execute(eq(GetIndexAction.INSTANCE), any(), any());
+        inOrder.verify(listener).onResponse(true);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <Response> Answer<Response> withResponse(Response response) {
+        return invocationOnMock -> {
+            ActionListener<Response> listener = (ActionListener<Response>) invocationOnMock.getArguments()[2];
+            listener.onResponse(response);
+            return null;
+        };
+    }
+
+    private static IndexStats indexStats(String index, int docCount) {
+        CommonStats indexTotalStats = mock(CommonStats.class);
+        when(indexTotalStats.getDocs()).thenReturn(new DocsStats(docCount, 0, 0));
+        IndexStats indexStats = mock(IndexStats.class);
+        when(indexStats.getIndex()).thenReturn(index);
+        when(indexStats.getTotal()).thenReturn(indexTotalStats);
+        return indexStats;
+    }
+}

From 7a1502db6c394254886a1aa16107a72be4de8197 Mon Sep 17 00:00:00 2001
From: lgypro <42293361+lgypro@users.noreply.github.com>
Date: Fri, 20 Mar 2020 18:44:53 +0800
Subject: [PATCH 163/285] =?UTF-8?q?[Docs]=C2=A0Fix=20typo=20=20in=20=5Fana?=
 =?UTF-8?q?lyze=20api=20docs=20(#53837)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/reference/analysis/testing.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/analysis/testing.asciidoc b/docs/reference/analysis/testing.asciidoc
index ba3300802ac87..845f275455eb2 100644
--- a/docs/reference/analysis/testing.asciidoc
+++ b/docs/reference/analysis/testing.asciidoc
@@ -55,7 +55,7 @@ The API returns the following response:
 You can also test combinations of:
 
 * A tokenizer
-* Zero or token filters
+* Zero or more token filters
 * Zero or more character filters
 
 [source,console]

From 76cd6385ceff72e3856dc99e1e98d3d90a089d8c Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 20 Mar 2020 11:22:52 +0000
Subject: [PATCH 164/285] Revert "Apply cluster states in system context
 (#53785)" (#53842)

This reverts commit c1dc5238da165be524e33a20ac95b5c4bf6566f3.
---
 .../PublicationTransportHandler.java          |  2 -
 .../service/ClusterApplierService.java        |  5 +-
 .../cluster/coordination/NodeJoinTests.java   |  5 +-
 .../snapshots/SnapshotResiliencyTests.java    |  8 +--
 .../AbstractCoordinatorTestCase.java          | 30 +++++------
 .../MockSinglePrioritizingExecutor.java       |  5 +-
 .../ClusterApplierAssertionPlugin.java        | 50 -------------------
 .../service/FakeThreadPoolMasterService.java  | 22 ++++----
 .../elasticsearch/test/ESIntegTestCase.java   |  4 --
 .../MockSinglePrioritizingExecutorTests.java  |  2 +-
 .../FakeThreadPoolMasterServiceTests.java     | 10 +---
 11 files changed, 40 insertions(+), 103 deletions(-)
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java

diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index 1a6db35a8091e..a4b2f509b2ed8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -161,7 +161,6 @@ public PublicationContext newPublicationContext(ClusterChangedEvent clusterChang
             public void sendPublishRequest(DiscoveryNode destination, PublishRequest publishRequest,
                                            ActionListener<PublishWithJoinResponse> originalListener) {
                 assert publishRequest.getAcceptedState() == clusterChangedEvent.state() : "state got switched on us";
-                assert transportService.getThreadPool().getThreadContext().isSystemContext();
                 final ActionListener<PublishWithJoinResponse> responseActionListener;
                 if (destination.equals(nodes.getLocalNode())) {
                     // if publishing to self, use original request instead (see currentPublishRequestToSelf for explanation)
@@ -198,7 +197,6 @@ public void onFailure(Exception e) {
             @Override
             public void sendApplyCommit(DiscoveryNode destination, ApplyCommitRequest applyCommitRequest,
                                         ActionListener<TransportResponse.Empty> responseActionListener) {
-                assert transportService.getThreadPool().getThreadContext().isSystemContext();
                 transportService.sendRequest(destination, COMMIT_STATE_ACTION_NAME, applyCommitRequest, stateRequestOptions,
                     new TransportResponseHandler<TransportResponse.Empty>() {
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index 4912c8f17a8e7..574919f6751ab 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -46,7 +46,6 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -348,9 +347,7 @@ private void submitStateUpdateTask(final String source, final ClusterStateTaskCo
         if (!lifecycle.started()) {
             return;
         }
-        final ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-            threadContext.markAsSystemContext();
+        try {
             UpdateTask updateTask = new UpdateTask(config.priority(), source, new SafeClusterApplyListener(listener, logger), executor);
             if (config.timeout() != null) {
                 threadPoolExecutor.execute(updateTask, config.timeout(),
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index 6557d647c75fa..ac3075597dc57 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -122,10 +122,9 @@ private static ClusterState initialState(DiscoveryNode localNode, long term, lon
     private void setupFakeMasterServiceAndCoordinator(long term, ClusterState initialState) {
         deterministicTaskQueue
             = new DeterministicTaskQueue(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), random());
-        final ThreadPool fakeThreadPool = deterministicTaskQueue.getThreadPool();
         FakeThreadPoolMasterService fakeMasterService = new FakeThreadPoolMasterService("test_node","test",
-            fakeThreadPool, deterministicTaskQueue::scheduleNow);
-        setupMasterServiceAndCoordinator(term, initialState, fakeMasterService, fakeThreadPool, Randomness.get());
+                deterministicTaskQueue::scheduleNow);
+        setupMasterServiceAndCoordinator(term, initialState, fakeMasterService, deterministicTaskQueue.getThreadPool(), Randomness.get());
         fakeMasterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
             coordinator.handlePublishRequest(new PublishRequest(event.state()));
             publishListener.onResponse(null);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 2139268fe3888..062e64ba55b74 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1163,15 +1163,15 @@ private final class TestClusterNode {
             TestClusterNode(DiscoveryNode node) throws IOException {
                 this.node = node;
                 final Environment environment = createEnvironment(node.getName());
-                threadPool = deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable));
-                masterService = new FakeThreadPoolMasterService(node.getName(), "test", threadPool, deterministicTaskQueue::scheduleNow);
+                masterService = new FakeThreadPoolMasterService(node.getName(), "test", deterministicTaskQueue::scheduleNow);
                 final Settings settings = environment.settings();
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+                threadPool = deterministicTaskQueue.getThreadPool();
                 clusterService = new ClusterService(settings, clusterSettings, masterService,
                     new ClusterApplierService(node.getName(), settings, clusterSettings, threadPool) {
                         @Override
                         protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-                            return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool);
+                            return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue);
                         }
 
                         @Override
@@ -1211,7 +1211,7 @@ protected NamedWriteableRegistry writeableRegistry() {
                     }
                 };
                 transportService = mockTransport.createTransportService(
-                    settings, threadPool,
+                    settings, deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable)),
                     new TransportInterceptor() {
                         @Override
                         public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
index 491bd7d858091..5239827f2b171 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -911,7 +911,6 @@ class ClusterNode {
             }
 
             private void setUp() {
-                final ThreadPool threadPool = deterministicTaskQueue.getThreadPool(this::onNode);
                 mockTransport = new DisruptableMockTransport(localNode, logger) {
                     @Override
                     protected void execute(Runnable runnable) {
@@ -929,20 +928,24 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                             .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
                     }
                 };
+
                 final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
                     nodeSettings : Settings.builder().put(nodeSettings)
                     .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
                         ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
-                transportService = mockTransport.createTransportService(settings, threadPool,
-                    getTransportInterceptor(localNode, threadPool), a -> localNode, null, emptySet());
-                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test", threadPool,
+                transportService = mockTransport.createTransportService(
+                    settings, deterministicTaskQueue.getThreadPool(this::onNode),
+                    getTransportInterceptor(localNode, deterministicTaskQueue.getThreadPool(this::onNode)),
+                    a -> localNode, null, emptySet());
+                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
                     runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
                 clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
-                    deterministicTaskQueue, threadPool);
+                    deterministicTaskQueue, this::onNode);
                 clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
                 clusterService.setNodeConnectionsService(
-                    new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService));
+                    new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
+                        transportService));
                 final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
                     Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
                 final AllocationService allocationService = ESAllocationTestCase.createAllocationService(Settings.EMPTY);
@@ -951,8 +954,8 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get(), (s, p, r) -> {},
                     getElectionStrategy());
                 masterService.setClusterStatePublisher(coordinator);
-                final GatewayService gatewayService
-                    = new GatewayService(settings, allocationService, clusterService, threadPool, coordinator, null);
+                final GatewayService gatewayService = new GatewayService(settings, allocationService, clusterService,
+                    deterministicTaskQueue.getThreadPool(this::onNode), coordinator, null);
 
                 logger.trace("starting up [{}]", localNode);
                 transportService.start();
@@ -1289,9 +1292,8 @@ static class AckedFakeThreadPoolMasterService extends FakeThreadPoolMasterServic
 
         AckCollector nextAckCollector = new AckCollector();
 
-        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, ThreadPool threadPool,
-                                         Consumer<Runnable> onTaskAvailableToRun) {
-            super(nodeName, serviceName, threadPool, onTaskAvailableToRun);
+        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
+            super(nodeName, serviceName, onTaskAvailableToRun);
         }
 
         @Override
@@ -1321,8 +1323,8 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
         private boolean applicationMayFail;
 
         DisruptableClusterApplierService(String nodeName, Settings settings, ClusterSettings clusterSettings,
-                                         DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
-            super(nodeName, settings, clusterSettings, threadPool);
+                                         DeterministicTaskQueue deterministicTaskQueue, Function<Runnable, Runnable> runnableWrapper) {
+            super(nodeName, settings, clusterSettings, deterministicTaskQueue.getThreadPool(runnableWrapper));
             this.nodeName = nodeName;
             this.deterministicTaskQueue = deterministicTaskQueue;
             addStateApplier(event -> {
@@ -1342,7 +1344,7 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
 
         @Override
         protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue, threadPool);
+            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue);
         }
 
         @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
index 553188ac24539..add5014e555fc 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.TimeUnit;
 
@@ -30,7 +29,7 @@
  */
 public class MockSinglePrioritizingExecutor extends PrioritizedEsThreadPoolExecutor {
 
-    public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
+    public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue) {
         super(name, 0, 1, 0L, TimeUnit.MILLISECONDS,
             r -> new Thread() {
                 @Override
@@ -52,7 +51,7 @@ public String toString() {
                     });
                 }
             },
-            threadPool.getThreadContext(), threadPool.scheduler());
+            deterministicTaskQueue.getThreadPool().getThreadContext(), deterministicTaskQueue.getThreadPool().scheduler());
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java b/test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java
deleted file mode 100644
index b44760ae37195..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/cluster/service/ClusterApplierAssertionPlugin.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.service;
-
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.watcher.ResourceWatcherService;
-
-import java.util.Collection;
-import java.util.Collections;
-
-public class ClusterApplierAssertionPlugin extends Plugin {
-    @Override
-    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
-                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
-                                               NamedXContentRegistry xContentRegistry, Environment environment,
-                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
-                                               IndexNameExpressionResolver indexNameExpressionResolver) {
-        clusterService.addStateApplier(event -> {
-            assert threadPool.getThreadContext().isSystemContext();
-        });
-        clusterService.addListener(event -> {
-            assert threadPool.getThreadContext().isSystemContext();
-        });
-        return Collections.emptyList();
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java b/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
index 100662f24a6ab..74d9cc17a5325 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
@@ -41,6 +41,8 @@
 
 import static org.apache.lucene.util.LuceneTestCase.random;
 import static org.elasticsearch.test.ESTestCase.randomInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class FakeThreadPoolMasterService extends MasterService {
     private static final Logger logger = LogManager.getLogger(FakeThreadPoolMasterService.class);
@@ -52,14 +54,21 @@ public class FakeThreadPoolMasterService extends MasterService {
     private boolean taskInProgress = false;
     private boolean waitForPublish = false;
 
-    public FakeThreadPoolMasterService(String nodeName, String serviceName, ThreadPool threadPool,
-                                       Consumer<Runnable> onTaskAvailableToRun) {
+    public FakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
         super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build(),
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool);
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            createMockThreadPool());
         this.name = serviceName;
         this.onTaskAvailableToRun = onTaskAvailableToRun;
     }
 
+    private static ThreadPool createMockThreadPool() {
+        final ThreadContext context = new ThreadContext(Settings.EMPTY);
+        final ThreadPool mockThreadPool = mock(ThreadPool.class);
+        when(mockThreadPool.getThreadContext()).thenReturn(context);
+        return mockThreadPool;
+    }
+
     @Override
     protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
         return new PrioritizedEsThreadPoolExecutor(name, 1, 1, 1, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(name),
@@ -101,11 +110,7 @@ public void run() {
                     final Runnable task = pendingTasks.remove(taskIndex);
                     taskInProgress = true;
                     scheduledNextTask = false;
-                    final ThreadContext threadContext = threadPool.getThreadContext();
-                    try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-                        threadContext.markAsSystemContext();
-                        task.run();
-                    }
+                    task.run();
                     if (waitForPublish == false) {
                         taskInProgress = false;
                     }
@@ -163,5 +168,4 @@ public void onFailure(Exception e) {
     protected AckListener wrapAckListener(AckListener ackListener) {
         return ackListener;
     }
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 4a482d6d412cf..3116792c502d6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -72,7 +72,6 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
-import org.elasticsearch.cluster.service.ClusterApplierAssertionPlugin;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
@@ -1785,9 +1784,6 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
             if (randomBoolean()) {
                 mocks.add(MockFieldFilterPlugin.class);
             }
-            if (randomBoolean()) {
-                mocks.add(ClusterApplierAssertionPlugin.class);
-            }
         }
 
         if (addMockTransportService()) {
diff --git a/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java b/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
index 66424de7f75d6..427f0d32c1270 100644
--- a/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
@@ -29,7 +29,7 @@ public class MockSinglePrioritizingExecutorTests extends ESTestCase {
 
     public void testPrioritizedEsThreadPoolExecutor() {
         final DeterministicTaskQueue taskQueue = DeterministicTaskQueueTests.newTaskQueue();
-        final PrioritizedEsThreadPoolExecutor executor = new MockSinglePrioritizingExecutor("test", taskQueue, taskQueue.getThreadPool());
+        final PrioritizedEsThreadPoolExecutor executor = new MockSinglePrioritizingExecutor("test", taskQueue);
         final AtomicBoolean called1 = new AtomicBoolean();
         final AtomicBoolean called2 = new AtomicBoolean();
         executor.execute(new PrioritizedRunnable(Priority.NORMAL) {
diff --git a/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java b/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
index c3e162254421b..f97477707e461 100644
--- a/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
@@ -27,10 +27,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -40,8 +37,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class FakeThreadPoolMasterServiceTests extends ESTestCase {
 
@@ -53,10 +48,7 @@ public void testFakeMasterService() {
         lastClusterStateRef.set(ClusterStateCreationUtils.state(discoveryNode, discoveryNode));
         long firstClusterStateVersion = lastClusterStateRef.get().version();
         AtomicReference<ActionListener<Void>> publishingCallback = new AtomicReference<>();
-        final ThreadContext context = new ThreadContext(Settings.EMPTY);
-        final ThreadPool mockThreadPool = mock(ThreadPool.class);
-        when(mockThreadPool.getThreadContext()).thenReturn(context);
-        FakeThreadPoolMasterService masterService = new FakeThreadPoolMasterService("test_node","test", mockThreadPool, runnableTasks::add);
+        FakeThreadPoolMasterService masterService = new FakeThreadPoolMasterService("test_node","test", runnableTasks::add);
         masterService.setClusterStateSupplier(lastClusterStateRef::get);
         masterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
             lastClusterStateRef.set(event.state());

From 2794ab797535a4dda2c10cf5757b13906daa4254 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 Mar 2020 07:34:37 -0400
Subject: [PATCH 165/285] Execute retention lease syncs under system context
 (#53838)

The retention lease syncs need to occur under the system context,
because they are internal actions executed on behalf of the user. Today
we are relying on this happening for background syncs by virtue of the
fact that the context the syncs are created under is the system
context. This is due to these occurring on the cluster state applier
thread. However, there are situations where this does not hold such as
when a timed out cluster state publication occurs, and the node where
the shard is allocated is the elected master node. In that case, the
context will be empty due to the fact that we do not reschedule
publication under the system context. Currently, doing so runs us into
some troubles with losing the existing context, possibly dropping
deprecation headers. We could copy that context over when marking the
current context as the system context, but the implications of that
require some more investigation. For now, we explicitly mark the
retention lease syncs as executing under the system context, as this is
situation that we can reason about.
---
 .../util/concurrent/AbstractAsyncTask.java    |  2 +-
 .../RetentionLeaseBackgroundSyncAction.java   | 78 ++++++++++---------
 .../index/seqno/RetentionLeaseSyncAction.java | 72 +++++++++--------
 3 files changed, 82 insertions(+), 70 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java
index 58e48f0215586..1ef9a484a2777 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java
@@ -91,7 +91,7 @@ public synchronized void rescheduleIfNecessary() {
             if (logger.isTraceEnabled()) {
                 logger.trace("scheduling {} every {}", toString(), interval);
             }
-            cancellable = threadPool.schedule(threadPool.preserveContext(this), interval, getThreadPool());
+            cancellable = threadPool.schedule(this, interval, getThreadPool());
             isScheduledOrRunning = true;
         } else {
             logger.trace("scheduled {} disabled", toString());
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
index 1f6dd80a9f82e..e4b82b3417551 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.gateway.WriteStateException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
@@ -101,44 +102,49 @@ protected void doExecute(Task task, Request request, ActionListener<ReplicationR
     }
 
     final void backgroundSync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases) {
-        final Request request = new Request(shardId, retentionLeases);
-        final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_background_sync", request);
-        transportService.sendChildRequest(clusterService.localNode(), transportPrimaryAction,
-            new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm),
-            task,
-            transportOptions,
-            new TransportResponseHandler<ReplicationResponse>() {
-                @Override
-                public ReplicationResponse read(StreamInput in) throws IOException {
-                    return newResponseInstance(in);
-                }
-
-                @Override
-                public String executor() {
-                    return ThreadPool.Names.SAME;
-                }
-
-                @Override
-                public void handleResponse(ReplicationResponse response) {
-                    task.setPhase("finished");
-                    taskManager.unregister(task);
-                }
-
-                @Override
-                public void handleException(TransportException e) {
-                    task.setPhase("finished");
-                    taskManager.unregister(task);
-                    if (ExceptionsHelper.unwrap(e, NodeClosedException.class) != null) {
-                        // node shutting down
-                        return;
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            // we have to execute under the system context so that if security is enabled the sync is authorized
+            threadContext.markAsSystemContext();
+            final Request request = new Request(shardId, retentionLeases);
+            final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_background_sync", request);
+            transportService.sendChildRequest(clusterService.localNode(), transportPrimaryAction,
+                new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm),
+                task,
+                transportOptions,
+                new TransportResponseHandler<ReplicationResponse>() {
+                    @Override
+                    public ReplicationResponse read(StreamInput in) throws IOException {
+                        return newResponseInstance(in);
                     }
-                    if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) != null) {
-                        // the shard is closed
-                        return;
+
+                    @Override
+                    public String executor() {
+                        return ThreadPool.Names.SAME;
+                    }
+
+                    @Override
+                    public void handleResponse(ReplicationResponse response) {
+                        task.setPhase("finished");
+                        taskManager.unregister(task);
                     }
-                    getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e);
-                }
-            });
+
+                    @Override
+                    public void handleException(TransportException e) {
+                        task.setPhase("finished");
+                        taskManager.unregister(task);
+                        if (ExceptionsHelper.unwrap(e, NodeClosedException.class) != null) {
+                            // node shutting down
+                            return;
+                        }
+                        if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) != null) {
+                            // the shard is closed
+                            return;
+                        }
+                        getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e);
+                    }
+                });
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
index 44da33186983a..6b906792afc05 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.gateway.WriteStateException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
@@ -99,40 +100,45 @@ protected void doExecute(Task parentTask, Request request, ActionListener<Respon
 
     final void sync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases,
                     ActionListener<ReplicationResponse> listener) {
-        final Request request = new Request(shardId, retentionLeases);
-        final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_sync", request);
-        transportService.sendChildRequest(clusterService.localNode(), transportPrimaryAction,
-            new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm),
-            task,
-            transportOptions,
-            new TransportResponseHandler<ReplicationResponse>() {
-                @Override
-                public ReplicationResponse read(StreamInput in) throws IOException {
-                    return newResponseInstance(in);
-                }
-
-                @Override
-                public String executor() {
-                    return ThreadPool.Names.SAME;
-                }
-
-                @Override
-                public void handleResponse(ReplicationResponse response) {
-                    task.setPhase("finished");
-                    taskManager.unregister(task);
-                    listener.onResponse(response);
-                }
-
-                @Override
-                public void handleException(TransportException e) {
-                    if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) {
-                        getLogger().warn(new ParameterizedMessage("{} retention lease sync failed", shardId), e);
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            // we have to execute under the system context so that if security is enabled the sync is authorized
+            threadContext.markAsSystemContext();
+            final Request request = new Request(shardId, retentionLeases);
+            final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_sync", request);
+            transportService.sendChildRequest(clusterService.localNode(), transportPrimaryAction,
+                new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm),
+                task,
+                transportOptions,
+                new TransportResponseHandler<ReplicationResponse>() {
+                    @Override
+                    public ReplicationResponse read(StreamInput in) throws IOException {
+                        return newResponseInstance(in);
                     }
-                    task.setPhase("finished");
-                    taskManager.unregister(task);
-                    listener.onFailure(e);
-                }
-            });
+
+                    @Override
+                    public String executor() {
+                        return ThreadPool.Names.SAME;
+                    }
+
+                    @Override
+                    public void handleResponse(ReplicationResponse response) {
+                        task.setPhase("finished");
+                        taskManager.unregister(task);
+                        listener.onResponse(response);
+                    }
+
+                    @Override
+                    public void handleException(TransportException e) {
+                        if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) {
+                            getLogger().warn(new ParameterizedMessage("{} retention lease sync failed", shardId), e);
+                        }
+                        task.setPhase("finished");
+                        taskManager.unregister(task);
+                        listener.onFailure(e);
+                    }
+                });
+        }
     }
 
     @Override

From 561222badcf371616a8b492ca6270ad366947fb3 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Fri, 20 Mar 2020 11:40:27 +0000
Subject: [PATCH 166/285] Report parser name and location in XContent
 deprecation warnings (#53805)

It's simple to deprecate a field used in an ObjectParser just by adding deprecation
markers to the relevant ParseField objects. The warnings themselves don't currently
have any context - they simply say that a deprecated field has been used, but not
where in the input xcontent it appears. This commit adds the parent object parser
name and XContentLocation to these deprecation messages.

Note that the context is automatically stripped from warning messages when they
are asserted on by integration tests and REST tests, because randomization of
xcontent type during these tests means that the XContentLocation is not constant
* Strip xcontentloc in yaml tests

* Handle negative pos values
---
 .../org/elasticsearch/common/ParseField.java  | 24 ++++++++--
 .../common/xcontent/DeprecationHandler.java   | 48 +++++++++++++------
 .../common/xcontent/ObjectParser.java         |  2 +-
 .../common/xcontent/XContentLocation.java     |  5 +-
 .../elasticsearch/common/ParseFieldTests.java |  6 +--
 .../common/xcontent/ObjectParserTests.java    |  2 +-
 .../common/logging/EvilLoggerTests.java       |  3 +-
 .../elasticsearch/http/DeprecationHttpIT.java |  3 +-
 .../common/logging/DeprecationLogger.java     | 14 ++++--
 .../xcontent/LoggingDeprecationHandler.java   | 20 +++++---
 .../logging/DeprecationLoggerTests.java       | 12 ++++-
 .../util/concurrent/ThreadContextTests.java   |  4 +-
 .../org/elasticsearch/test/ESTestCase.java    |  7 ++-
 .../test/rest/yaml/section/DoSection.java     |  2 +-
 ...LoggingDeprecationAccumulationHandler.java | 29 ++++++-----
 .../utils/XContentObjectTransformerTests.java |  4 +-
 .../inference/ingest/InferenceProcessor.java  |  2 +-
 .../xpack/security/authc/ApiKeyService.java   | 23 +++++----
 18 files changed, 145 insertions(+), 65 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
index f4f12ed4f4c81..3872ef4852274 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
@@ -19,11 +19,13 @@
 package org.elasticsearch.common;
 
 import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentLocation;
 
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Supplier;
 
 /**
  * Holds a field that can be found in a request while parsing and its different
@@ -115,6 +117,22 @@ public ParseField withAllDeprecated() {
      *         names for this {@link ParseField}.
      */
     public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
+        return match(null, () -> XContentLocation.UNKNOWN, fieldName, deprecationHandler);
+    }
+
+    /**
+     * Does {@code fieldName} match this field?
+     * @param parserName
+     *            the name of the parent object holding this field
+     * @param location
+     *            the XContentLocation of the field
+     * @param fieldName
+     *            the field name to match against this {@link ParseField}
+     * @param deprecationHandler called if {@code fieldName} is deprecated
+     * @return true if <code>fieldName</code> matches any of the acceptable
+     *         names for this {@link ParseField}.
+     */
+    public boolean match(String parserName, Supplier<XContentLocation> location, String fieldName, DeprecationHandler deprecationHandler) {
         Objects.requireNonNull(fieldName, "fieldName cannot be null");
         // if this parse field has not been completely deprecated then try to
         // match the preferred name
@@ -127,11 +145,11 @@ public boolean match(String fieldName, DeprecationHandler deprecationHandler) {
         for (String depName : deprecatedNames) {
             if (fieldName.equals(depName)) {
                 if (fullyDeprecated) {
-                    deprecationHandler.usedDeprecatedField(fieldName);
+                    deprecationHandler.usedDeprecatedField(parserName, location, fieldName);
                 } else if (allReplacedWith == null) {
-                    deprecationHandler.usedDeprecatedName(fieldName, name);
+                    deprecationHandler.usedDeprecatedName(parserName, location, fieldName, name);
                 } else {
-                    deprecationHandler.usedDeprecatedField(fieldName, allReplacedWith);
+                    deprecationHandler.usedDeprecatedField(parserName, location, fieldName, allReplacedWith);
                 }
                 return true;
             }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
index 2cfda82eb4c48..65f52943f0b86 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common.xcontent;
 
+import java.util.function.Supplier;
+
 /**
  * Callback for notifying the creator of the {@link XContentParser} that
  * parsing hit a deprecated field.
@@ -32,20 +34,35 @@ public interface DeprecationHandler {
      */
     DeprecationHandler THROW_UNSUPPORTED_OPERATION = new DeprecationHandler() {
         @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {
-            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                + usedName + "] which is a deprecated name for [" + replacedWith + "]");
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+            if (parserName != null) {
+                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
+                    + usedName + "] at [" + location.get() + "] which is a deprecated name for [" + replacedWith + "]");
+            } else {
+                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                    + usedName + "] which is a deprecated name for [" + replacedWith + "]");
+            }
         }
         @Override
-        public void usedDeprecatedName(String usedName, String modernName) {
-            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                + usedName + "] which has been replaced with [" + modernName + "]");
+        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+            if (parserName != null) {
+                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
+                    + usedName + "] at [" + location.get() + "] which has been replaced with [" + modernName + "]");
+            } else {
+                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                    + usedName + "] which has been replaced with [" + modernName + "]");
+            }
         }
 
         @Override
-        public void usedDeprecatedField(String usedName) {
-            throw new UnsupportedOperationException("deprecated fields not supported here but got ["
-                + usedName + "] which has been deprecated entirely");
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+            if (parserName != null) {
+                throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got ["
+                    + usedName + "] at [" + location.get() + "] which has been deprecated entirely");
+            } else {
+                throw new UnsupportedOperationException("deprecated fields not supported here but got ["
+                    + usedName + "] which has been deprecated entirely");
+            }
         }
     };
 
@@ -54,17 +71,17 @@ public void usedDeprecatedField(String usedName) {
      */
     DeprecationHandler IGNORE_DEPRECATIONS = new DeprecationHandler() {
         @Override
-        public void usedDeprecatedName(String usedName, String modernName) {
+        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
 
         }
 
         @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
 
         }
 
         @Override
-        public void usedDeprecatedField(String usedName) {
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
 
         }
     };
@@ -74,7 +91,7 @@ public void usedDeprecatedField(String usedName) {
      * @param usedName the provided field name
      * @param modernName the modern name for the field
      */
-    void usedDeprecatedName(String usedName, String modernName);
+    void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName);
 
     /**
      * Called when the provided field name matches the current field but the entire
@@ -82,12 +99,13 @@ public void usedDeprecatedField(String usedName) {
      * @param usedName the provided field name
      * @param replacedWith the name of the field that replaced this field
      */
-    void usedDeprecatedField(String usedName, String replacedWith);
+    void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith);
 
     /**
      * Called when the provided field name matches the current field but the entire
      * field has been marked as deprecated with no replacement
      * @param usedName the provided field name
      */
-    void usedDeprecatedField(String usedName);
+    void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName);
+
 }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index b11f6afd4bb26..73bbf812f598c 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -571,7 +571,7 @@ private class FieldParser {
         }
 
         void assertSupports(String parserName, XContentParser parser, String currentFieldName) {
-            if (parseField.match(currentFieldName, parser.getDeprecationHandler()) == false) {
+            if (parseField.match(parserName, parser::getTokenLocation, currentFieldName, parser.getDeprecationHandler()) == false) {
                 throw new XContentParseException(parser.getTokenLocation(),
                         "[" + parserName  + "] parsefield doesn't accept: " + currentFieldName);
             }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
index 43ab7503cd1dd..1d5bfd6c2c429 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentLocation.java
@@ -25,7 +25,10 @@
  * position of a parsing error to end users and consequently have line and
  * column numbers starting from 1.
  */
-public class XContentLocation {
+public final class XContentLocation {
+
+    public static final XContentLocation UNKNOWN = new XContentLocation(-1, -1);
+
     public final int lineNumber;
     public final int columnNumber;
 
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
index 381c7c3e959be..e7420356c8fac 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java
@@ -66,11 +66,11 @@ public void testDeprecatedWithNoReplacement() {
         ParseField field = new ParseField(name).withDeprecation(alternatives).withAllDeprecated();
         assertFalse(field.match("not a field name", LoggingDeprecationHandler.INSTANCE));
         assertTrue(field.match("dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [dep] used, which has been removed entirely");
+        assertWarnings("Deprecated field [dep] used, this field is unused and will be removed entirely");
         assertTrue(field.match("old_dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [old_dep] used, which has been removed entirely");
+        assertWarnings("Deprecated field [old_dep] used, this field is unused and will be removed entirely");
         assertTrue(field.match("new_dep", LoggingDeprecationHandler.INSTANCE));
-        assertWarnings("Deprecated field [new_dep] used, which has been removed entirely");
+        assertWarnings("Deprecated field [new_dep] used, this field is unused and will be removed entirely");
 
 
     }
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index c99d1b10d6a4d..5eca7d527730b 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -223,7 +223,7 @@ class TestStruct {
         objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
         objectParser.parse(parser, s, null);
         assertEquals("foo", s.test);
-        assertWarnings("Deprecated field [old_test] used, expected [test] instead");
+        assertWarnings(false, "[foo][1:15] Deprecated field [old_test] used, expected [test] instead");
     }
 
     public void testFailOnValueType() throws IOException {
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
index e32447c47b092..2e35ac9581f9a 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
@@ -164,7 +164,8 @@ public void testConcurrentDeprecationLogger() throws IOException, UserException,
                  */
                 final List<String> warnings = threadContext.getResponseHeaders().get("Warning");
                 final Set<String> actualWarningValues =
-                        warnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toSet());
+                        warnings.stream().map(s -> DeprecationLogger.extractWarningValueFromWarningHeader(s, true))
+                            .collect(Collectors.toSet());
                 for (int j = 0; j < 128; j++) {
                     assertThat(
                             actualWarningValues,
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java
index fc686c5671997..c9bed57fb13e7 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DeprecationHttpIT.java
@@ -187,7 +187,8 @@ private void doTestDeprecationWarningsAppearInHeaders() throws IOException {
             assertThat(deprecatedWarning, matches(WARNING_HEADER_PATTERN.pattern()));
         }
         final List<String> actualWarningValues =
-                deprecatedWarnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toList());
+                deprecatedWarnings.stream().map(s -> DeprecationLogger.extractWarningValueFromWarningHeader(s, true))
+                    .collect(Collectors.toList());
         for (Matcher<String> headerMatcher : headerMatchers) {
             assertThat(actualWarningValues, hasItem(headerMatcher));
         }
diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
index be157e16de20c..52ee3b7a5d288 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
@@ -174,6 +174,8 @@ public void deprecatedAndMaybeLog(final String key, final String msg, final Obje
                     "GMT" + // GMT
                     "\")?"); // closing quote (optional, since an older version can still send a warn-date)
 
+    public static final Pattern WARNING_XCONTENT_LOCATION_PATTERN = Pattern.compile("^\\[.*?]\\[-?\\d+:-?\\d+] ");
+
     /**
      * Extracts the warning value from the value of a warning header that is formatted according to RFC 7234. That is, given a string
      * {@code 299 Elasticsearch-6.0.0 "warning value"}, the return value of this method would be {@code warning value}.
@@ -181,7 +183,7 @@ public void deprecatedAndMaybeLog(final String key, final String msg, final Obje
      * @param s the value of a warning header formatted according to RFC 7234.
      * @return the extracted warning value
      */
-    public static String extractWarningValueFromWarningHeader(final String s) {
+    public static String extractWarningValueFromWarningHeader(final String s, boolean stripXContentPosition) {
         /*
          * We know the exact format of the warning header, so to extract the warning value we can skip forward from the front to the first
          * quote and we know the last quote is at the end of the string
@@ -196,8 +198,14 @@ public static String extractWarningValueFromWarningHeader(final String s) {
          */
         final int firstQuote = s.indexOf('\"');
         final int lastQuote = s.length() - 1;
-        final String warningValue = s.substring(firstQuote + 1, lastQuote);
+        String warningValue = s.substring(firstQuote + 1, lastQuote);
         assert assertWarningValue(s, warningValue);
+        if (stripXContentPosition) {
+            Matcher matcher = WARNING_XCONTENT_LOCATION_PATTERN.matcher(warningValue);
+            if (matcher.find()) {
+                warningValue = warningValue.substring(matcher.end());
+            }
+        }
         return warningValue;
     }
 
@@ -232,7 +240,7 @@ void deprecated(final Set<ThreadContext> threadContexts, final String message, f
             final String formattedMessage = LoggerMessageFormat.format(message, params);
             final String warningHeaderValue = formatWarning(formattedMessage);
             assert WARNING_HEADER_PATTERN.matcher(warningHeaderValue).matches();
-            assert extractWarningValueFromWarningHeader(warningHeaderValue).equals(escapeAndEncode(formattedMessage));
+            assert extractWarningValueFromWarningHeader(warningHeaderValue, false).equals(escapeAndEncode(formattedMessage));
             while (iterator.hasNext()) {
                 try {
                     final ThreadContext next = iterator.next();
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
index 9a885b55d146c..5feca4bfaab5a 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/LoggingDeprecationHandler.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.logging.DeprecationLogger;
 
+import java.util.function.Supplier;
+
 /**
  * Logs deprecations to the {@link DeprecationLogger}.
  * <p>
@@ -49,17 +51,23 @@ private LoggingDeprecationHandler() {
     }
 
     @Override
-    public void usedDeprecatedName(String usedName, String modernName) {
-        deprecationLogger.deprecated("Deprecated field [{}] used, expected [{}] instead", usedName, modernName);
+    public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecationLogger.deprecated("{}Deprecated field [{}] used, expected [{}] instead",
+            prefix, usedName, modernName);
     }
 
     @Override
-    public void usedDeprecatedField(String usedName, String replacedWith) {
-        deprecationLogger.deprecated("Deprecated field [{}] used, replaced by [{}]", usedName, replacedWith);
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecationLogger.deprecated("{}Deprecated field [{}] used, replaced by [{}]",
+            prefix, usedName, replacedWith);
     }
 
     @Override
-    public void usedDeprecatedField(String usedName) {
-        deprecationLogger.deprecated("Deprecated field [{}] used, which has been removed entirely", usedName);
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecationLogger.deprecated("{}Deprecated field [{}] used, this field is unused and will be removed entirely",
+            prefix, usedName);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java b/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
index e7a78454e57ff..d904d2f0c8526 100644
--- a/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
+++ b/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
@@ -216,10 +216,18 @@ public void testFailsWhenRemovingUnknownThreadContext() throws IOException {
         expectThrows(IllegalStateException.class, () -> DeprecationLogger.removeThreadContext(threadContext));
     }
 
-    public void testWarningValueFromWarningHeader() throws InterruptedException {
+    public void testWarningValueFromWarningHeader() {
         final String s = randomAlphaOfLength(16);
         final String first = DeprecationLogger.formatWarning(s);
-        assertThat(DeprecationLogger.extractWarningValueFromWarningHeader(first), equalTo(s));
+        assertThat(DeprecationLogger.extractWarningValueFromWarningHeader(first, false), equalTo(s));
+
+        final String withPos = "[context][1:11] Blah blah blah";
+        final String formatted = DeprecationLogger.formatWarning(withPos);
+        assertThat(DeprecationLogger.extractWarningValueFromWarningHeader(formatted, true), equalTo("Blah blah blah"));
+
+        final String withNegativePos = "[context][-1:-1] Blah blah blah";
+        assertThat(DeprecationLogger.extractWarningValueFromWarningHeader(DeprecationLogger.formatWarning(withNegativePos), true),
+            equalTo("Blah blah blah"));
     }
 
     public void testEscapeBackslashesAndQuotes() {
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index 577b451d8f19e..5305b80c9828b 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -208,11 +208,11 @@ public void testResponseHeaders() {
         }
 
         final String value = DeprecationLogger.formatWarning("qux");
-        threadContext.addResponseHeader("baz", value, DeprecationLogger::extractWarningValueFromWarningHeader);
+        threadContext.addResponseHeader("baz", value, s -> DeprecationLogger.extractWarningValueFromWarningHeader(s, false));
         // pretend that another thread created the same response at a different time
         if (randomBoolean()) {
             final String duplicateValue = DeprecationLogger.formatWarning("qux");
-            threadContext.addResponseHeader("baz", duplicateValue, DeprecationLogger::extractWarningValueFromWarningHeader);
+            threadContext.addResponseHeader("baz", duplicateValue, s -> DeprecationLogger.extractWarningValueFromWarningHeader(s, false));
         }
 
         threadContext.addResponseHeader("Warning", "One is the loneliest number");
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 75e577680274a..0a27993b40d7e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -422,6 +422,10 @@ protected final void assertSettingDeprecationsAndWarnings(final String[] setting
     }
 
     protected final void assertWarnings(String... expectedWarnings) {
+        assertWarnings(true, expectedWarnings);
+    }
+
+    protected final void assertWarnings(boolean stripXContentPosition, String... expectedWarnings) {
         if (enableWarningsCheck() == false) {
             throw new IllegalStateException("unable to check warning headers if the test is not set to do so");
         }
@@ -429,7 +433,8 @@ protected final void assertWarnings(String... expectedWarnings) {
             final List<String> actualWarnings = threadContext.getResponseHeaders().get("Warning");
             assertNotNull("no warnings, expected: " + Arrays.asList(expectedWarnings), actualWarnings);
             final Set<String> actualWarningValues =
-                    actualWarnings.stream().map(DeprecationLogger::extractWarningValueFromWarningHeader).collect(Collectors.toSet());
+                    actualWarnings.stream().map(s -> DeprecationLogger.extractWarningValueFromWarningHeader(s, stripXContentPosition))
+                        .collect(Collectors.toSet());
             for (String msg : expectedWarnings) {
                 assertThat(actualWarningValues, hasItem(DeprecationLogger.escapeAndEncode(msg)));
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index fec634b526756..d7884dd79d510 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -331,7 +331,7 @@ void checkWarningHeaders(final List<String> warningHeaders, final Version master
             final Matcher matcher = WARNING_HEADER_PATTERN.matcher(header);
             final boolean matches = matcher.matches();
             if (matches) {
-                final String message = matcher.group(1);
+                final String message = DeprecationLogger.extractWarningValueFromWarningHeader(header, true);
                 if (allowed.contains(message)) {
                     continue;
                 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
index 5e2a4d7a25302..323e3a84cdf2e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/LoggingDeprecationAccumulationHandler.java
@@ -8,10 +8,12 @@
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentLocation;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
 
 /**
  * Very similar to {@link org.elasticsearch.common.xcontent.LoggingDeprecationHandler} main differences are:
@@ -26,24 +28,27 @@ public class LoggingDeprecationAccumulationHandler implements DeprecationHandler
     private final List<String> deprecations = new ArrayList<>();
 
     @Override
-    public void usedDeprecatedName(String usedName, String modernName) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(usedName, modernName);
-        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, expected [{}] instead",
-            new Object[] {usedName, modernName}));
+    public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(parserName, location, usedName, modernName);
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, expected [{}] instead",
+            new Object[]{prefix, usedName, modernName}));
     }
 
     @Override
-    public void usedDeprecatedField(String usedName, String replacedWith) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName, replacedWith);
-        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, replaced by [{}]",
-            new Object[] {usedName, replacedWith}));
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(parserName, location, usedName, replacedWith);
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, replaced by [{}]",
+            new Object[]{prefix, usedName, replacedWith}));
     }
 
     @Override
-    public void usedDeprecatedField(String usedName) {
-        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(usedName);
-        deprecations.add(LoggerMessageFormat.format("Deprecated field [{}] used, which has been deprecated entirely",
-            new Object[]{ usedName }));
+    public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+        LoggingDeprecationHandler.INSTANCE.usedDeprecatedField(parserName, location, usedName);
+        String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+        deprecations.add(LoggerMessageFormat.format("{}Deprecated field [{}] used, unused and will be removed entirely",
+            new Object[]{prefix, usedName}));
     }
 
     /**
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
index d6ebc9a2e150b..886602395238f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
@@ -127,8 +127,8 @@ public void testToMap() throws IOException {
     public void testDeprecationWarnings() throws IOException {
         XContentObjectTransformer<QueryBuilder> queryBuilderTransformer = new XContentObjectTransformer<>(NamedXContentRegistry.EMPTY,
             (p)-> {
-            p.getDeprecationHandler().usedDeprecatedField("oldField", "newField");
-            p.getDeprecationHandler().usedDeprecatedName("oldName", "modernName");
+            p.getDeprecationHandler().usedDeprecatedField(null, null, "oldField", "newField");
+            p.getDeprecationHandler().usedDeprecatedName(null, null, "oldName", "modernName");
             return new BoolQueryBuilder();
             });
         List<String> deprecations = new ArrayList<>();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index 2a4861c5247f7..b52b8b5b64e3f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -242,7 +242,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
                 fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
                 //TODO Remove in 8.x
                 if (fieldMap != null) {
-                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
+                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(null, () -> null, FIELD_MAPPINGS, FIELD_MAP);
                 }
             }
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index f36ce90660a23..c363ad1253509 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -618,21 +619,25 @@ private ApiKeyLoggingDeprecationHandler(DeprecationLogger logger, String apiKeyI
         }
 
         @Override
-        public void usedDeprecatedName(String usedName, String modernName) {
-            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], expected [{}] instead",
-                usedName, apiKeyId, modernName);
+        public void usedDeprecatedName(String parserName, Supplier<XContentLocation> location, String usedName, String modernName) {
+            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+            deprecationLogger.deprecated("{}Deprecated field [{}] used in api key [{}], expected [{}] instead",
+                prefix, usedName, apiKeyId, modernName);
         }
 
         @Override
-        public void usedDeprecatedField(String usedName, String replacedWith) {
-            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], replaced by [{}]",
-                usedName, apiKeyId, replacedWith);
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName, String replacedWith) {
+            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+            deprecationLogger.deprecated("{}Deprecated field [{}] used in api key [{}], replaced by [{}]",
+                prefix, usedName, apiKeyId, replacedWith);
         }
 
         @Override
-        public void usedDeprecatedField(String usedName) {
-            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], which has been removed entirely",
-                usedName);
+        public void usedDeprecatedField(String parserName, Supplier<XContentLocation> location, String usedName) {
+            String prefix = parserName == null ? "" : "[" + parserName + "][" + location.get() + "] ";
+            deprecationLogger.deprecated(
+                "{}Deprecated field [{}] used in api key [{}], which is unused and will be removed entirely",
+                prefix, usedName);
         }
     }
 

From 687c888cf81caa6e8c83158ff3183966e225e681 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Fri, 20 Mar 2020 14:15:16 +0200
Subject: [PATCH 167/285] [ML] Unmute DF analytics bwc tests and fix versions
 (#53850)

... after backporting #53788
---
 .../core/ml/action/GetDataFrameAnalyticsStatsAction.java     | 4 ++--
 .../test/mixed_cluster/90_ml_data_frame_analytics_crud.yml   | 5 -----
 .../test/old_cluster/90_ml_data_frame_analytics_crud.yml     | 3 ---
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml     | 5 -----
 4 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
index 370ac21020059..209058e00468b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
@@ -203,7 +203,7 @@ public Stats(StreamInput in) throws IOException {
                 } else {
                     memoryUsage = null;
                 }
-                if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
                     analysisStats = in.readOptionalNamedWriteable(AnalysisStats.class);
                 } else {
                     analysisStats = null;
@@ -334,7 +334,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                     out.writeOptionalWriteable(memoryUsage);
                 }
-                if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                     out.writeOptionalNamedWriteable(analysisStats);
                 }
                 out.writeOptionalWriteable(node);
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index dacb1e1d0a31b..5049f3360a42e 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53788"
-
 ---
 "Get old outlier_detection job":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index dd9535d0accc0..923a56395e8a4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,7 +1,4 @@
 setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53788"
 
   - do:
       index:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index e976b2a2ce7af..3af10eb33e286 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53788"
-
 ---
 "Get old cluster outlier_detection job":
 

From 1443ba6a6d7eb2fca87d054440156c83fd600af3 Mon Sep 17 00:00:00 2001
From: Aleksandr Maus <aleksandr.maus@elastic.co>
Date: Fri, 20 Mar 2020 09:03:37 -0400
Subject: [PATCH 168/285] EQL: Extract query folder tests definitions into
 resources (#53802)

---
 .../xpack/eql/planner/QueryFolderOkTests.java | 124 +++++++++---------
 .../src/test/resources/queryfolder_tests.txt  |  66 ++++++++++
 2 files changed, 129 insertions(+), 61 deletions(-)
 create mode 100644 x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
index b774ff916414c..0867c6e77c5c8 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
@@ -7,72 +7,19 @@
 package org.elasticsearch.xpack.eql.planner;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
 
-import java.util.Arrays;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.hamcrest.Matchers.containsString;
 
 public class QueryFolderOkTests extends AbstractQueryFolderTestCase {
-    private static Object[][] specs = {
-            {"basic", "process where true", null},
-            {"singleNumericFilterEquals", "process where serial_event_id = 1", "\"term\":{\"serial_event_id\":{\"value\":1"},
-            {"singleNumericFilterLess", "process where serial_event_id < 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":false"
-            },
-            {"singleNumericFilterLessSymmetry", "process where 4 > serial_event_id",
-                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":false"
-            },
-            {"singleNumericFilterLessEquals", "process where serial_event_id <= 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":null,\"to\":4,\"include_lower\":false,\"include_upper\":true"
-            },
-            {"singleNumericFilterGreater", "process where serial_event_id > 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":4,\"to\":null,\"include_lower\":false,\"include_upper\":false"
-            },
-            {"singleNumericFilterGreaterEquals", "process where serial_event_id >= 4",
-                    "\"range\":{\"serial_event_id\":{\"from\":4,\"to\":null,\"include_lower\":true,\"include_upper\":false"
-            },
-            {"mixedTypeFilter", "process where process_name == \"notepad.exe\" or (serial_event_id < 4.5 and serial_event_id >= 3.1)",
-                    new Object[]{
-                            "\"term\":{\"process_name\":{\"value\":\"notepad.exe\"",
-                            "\"range\":{\"serial_event_id\":{\"from\":3.1,\"to\":4.5,\"include_lower\":true,\"include_upper\":false"
-                    }
-            },
-            {"notFilter", "process where not (exit_code > -1)",
-                    "\"range\":{\"exit_code\":{\"from\":null,\"to\":-1,\"include_lower\":false,\"include_upper\":true"
-            },
-            {"inFilter", "process where process_name in (\"python.exe\", \"SMSS.exe\", \"explorer.exe\")",
-                    new Object[]{
-                            "\"term\":{\"process_name\":{\"value\":\"python.exe\"",
-                            "\"term\":{\"process_name\":{\"value\":\"SMSS.exe\"",
-                            "\"term\":{\"process_name\":{\"value\":\"explorer.exe\"",
-                    }
-            },
-            {"equalsAndInFilter", "process where process_path == \"*\\\\red_ttp\\\\wininit.*\" and opcode in (0,1,2,3)",
-                    new Object[]{
-                            "\"wildcard\":{\"process_path\":{\"wildcard\":\"*\\\\\\\\red_ttp\\\\\\\\wininit.*\"",
-                            "\"term\":{\"opcode\":{\"value\":0",
-                            "\"term\":{\"opcode\":{\"value\":1",
-                            "\"term\":{\"opcode\":{\"value\":2",
-                            "\"term\":{\"opcode\":{\"value\":3",
-                    }
-            },
-            {"substringFunction", "process where substring(file_name, -4) == '.exe'",
-                    new Object[]{
-                            "{\"script\":{\"source\":\""
-                            + "InternalSqlScriptUtils.nullSafeFilter("
-                            + "InternalSqlScriptUtils.eq("
-                            + "InternalSqlScriptUtils.substring("
-                            + "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))",
-                            "\"params\":{\"v0\":\"file_name.keyword\",\"v1\":-4,\"v2\":null,\"v3\":\".exe\"}"
-                            
-                    }
-            }
-    };
-
     private final String name;
     private final String query;
     private final Object expect;
@@ -83,9 +30,64 @@ public QueryFolderOkTests(String name, String query, Object expect) {
         this.expect = expect;
     }
 
-    @ParametersFactory(shuffle = false, argumentFormatting = "%1$s.test")
-    public static Iterable<Object[]> parameters() {
-        return Arrays.asList(specs);
+    @ParametersFactory(shuffle = false, argumentFormatting = "%1$s")
+    public static Iterable<Object[]> parameters() throws Exception {
+        ArrayList<Object[]> arr = new ArrayList<>();
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(
+                QueryFolderOkTests.class.getResourceAsStream("/queryfolder_tests.txt"), StandardCharsets.UTF_8))) {
+            String line;
+            String name = null;
+            String query = null;
+            ArrayList<Object> expectations = null;
+            int newLineCount = 0;
+
+            while ((line = reader.readLine()) != null) {
+                if (line.startsWith("//")) {
+                    continue;
+                }
+
+                line = line.trim();
+                if (Strings.isEmpty(line)) {
+                    if (name != null) {
+                        newLineCount++;
+                    }
+                    if (newLineCount >= 2) {
+                        // Add and zero out for the next spec
+                        addSpec(arr, name, query, expectations == null ? null : expectations.toArray());
+                        name = null;
+                        query = null;
+                        expectations = null;
+                        newLineCount = 0;
+                    }
+                    continue;
+                }
+
+                if (name == null) {
+                    name = line;
+                    continue;
+                }
+
+                if (query == null) {
+                    query = line;
+                    continue;
+                }
+
+                if (line.equals("null") == false) {  // special case for no expectations
+                    if (expectations == null) {
+                        expectations = new ArrayList<>();
+                    }
+                    expectations.add(line);
+                }
+            }
+            addSpec(arr, name, query, expectations.toArray());
+        }
+        return arr;
+    }
+
+    private static void addSpec(ArrayList<Object[]> arr, String name, String query, Object[] expectations) {
+        if ((Strings.isNullOrEmpty(name) == false) && (Strings.isNullOrEmpty(query) == false)) {
+            arr.add(new Object[]{name, query, expectations});
+        }
     }
 
     public void test() {
diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
new file mode 100644
index 0000000000000..8a2af74ab84a2
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
@@ -0,0 +1,66 @@
+//
+// QueryFolder test
+// Simple format of the following blocks, separated by two new lines
+// <name>
+// <eql query>
+// <expectation 1>
+// <expectation 2>
+// ...
+// <expectation n>
+
+
+basic
+process where true
+null
+
+
+singleNumericFilterEquals
+process where serial_event_id = 1
+"term":{"serial_event_id":{"value":1
+
+
+singleNumericFilterLess
+process where serial_event_id < 4
+"range":{"serial_event_id":{"from":null,"to":4,"include_lower":false,"include_upper":false
+
+
+singleNumericFilterLessEquals
+process where serial_event_id <= 4
+"range":{"serial_event_id":{"from":null,"to":4,"include_lower":false,"include_upper":true
+
+
+singleNumericFilterGreater
+process where serial_event_id > 4
+"range":{"serial_event_id":{"from":4,"to":null,"include_lower":false,"include_upper":false
+
+
+singleNumericFilterGreaterEquals
+process where serial_event_id >= 4
+"range":{"serial_event_id":{"from":4,"to":null,"include_lower":true,"include_upper":false
+
+
+mixedTypeFilter
+process where process_name == "notepad.exe" or (serial_event_id < 4.5 and serial_event_id >= 3.1)
+"term":{"process_name":{"value":"notepad.exe"
+"range":{"serial_event_id":{"from":3.1,"to":4.5,"include_lower":true,"include_upper":false
+
+
+notFilter
+process where not (exit_code > -1)
+"range":{"exit_code":{"from":null,"to":-1,"include_lower":false,"include_upper":true
+
+
+inFilter
+process where process_name in ("python.exe", "SMSS.exe", "explorer.exe")
+"term":{"process_name":{"value":"python.exe"
+"term":{"process_name":{"value":"SMSS.exe"
+"term":{"process_name":{"value":"explorer.exe"
+
+
+equalsAndInFilter
+process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
+"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
+"term":{"opcode":{"value":0
+"term":{"opcode":{"value":1
+"term":{"opcode":{"value":2
+"term":{"opcode":{"value":3

From 0a93a93069b5e4995cfa9dc87c0652755479121d Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Fri, 20 Mar 2020 14:04:33 +0100
Subject: [PATCH 169/285] [DOCS] add docs for async search (#53675)

Relates to #49091

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>
---
 .../doc/RestTestsFromSnippetsTask.groovy      |   3 -
 docs/reference/api-conventions.asciidoc       |   6 +-
 docs/reference/async-search.asciidoc          |  22 ++
 docs/reference/index.asciidoc                 |   2 +
 .../reference/indices/rollover-index.asciidoc |   4 +-
 docs/reference/redirects.asciidoc             |  27 +--
 docs/reference/search.asciidoc                |   2 +
 docs/reference/search/async-search.asciidoc   | 214 ++++++++++++++++++
 .../api/async_search.delete.json              |   2 +-
 .../rest-api-spec/api/async_search.get.json   |   2 +-
 .../api/async_search.submit.json              |  18 +-
 11 files changed, 267 insertions(+), 35 deletions(-)
 create mode 100644 docs/reference/async-search.asciidoc
 create mode 100644 docs/reference/search/async-search.asciidoc

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index 5b855a4d9c7b5..e271ff1a58750 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -314,10 +314,7 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
             if (path == null) {
                 path = '' // Catch requests to the root...
             } else {
-                // Escape some characters that are also escaped by sense
                 path = path.replace('<', '%3C').replace('>', '%3E')
-                path = path.replace('{', '%7B').replace('}', '%7D')
-                path = path.replace('|', '%7C')
             }
             current.println("  - do:")
             if (catchPart != null) {
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 41a09c9a15ff6..3921aab0d1354 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -87,7 +87,7 @@ GET /%3Clogstash-%7Bnow%2Fd%7D%3E/_search
 }
 ----------------------------------------------------------------------
 // TEST[s/^/PUT logstash-2016.09.20\n/]
-// TEST[s/now/2016.09.20||/]
+// TEST[s/now/2016.09.20%7C%7C/]
 
 [NOTE]
 .Percent encoding of date math characters
@@ -141,7 +141,7 @@ GET /%3Clogstash-%7Bnow%2Fd-2d%7D%3E%2C%3Clogstash-%7Bnow%2Fd-1d%7D%3E%2C%3Clogs
 }
 ----------------------------------------------------------------------
 // TEST[s/^/PUT logstash-2016.09.20\nPUT logstash-2016.09.19\nPUT logstash-2016.09.18\n/]
-// TEST[s/now/2016.09.20||/]
+// TEST[s/now/2016.09.20%7C%7C/]
 
 [[common-options]]
 === Common options
@@ -367,7 +367,7 @@ GET /_search?filter_path=hits.hits._source&_source=title&sort=rating:desc
 [float]
 ==== Flat Settings
 
-The `flat_settings` flag affects rendering of the lists of settings. When the 
+The `flat_settings` flag affects rendering of the lists of settings. When the
 `flat_settings` flag is `true`, settings are returned in a flat format:
 
 [source,console]
diff --git a/docs/reference/async-search.asciidoc b/docs/reference/async-search.asciidoc
new file mode 100644
index 0000000000000..d51e017b0f97c
--- /dev/null
+++ b/docs/reference/async-search.asciidoc
@@ -0,0 +1,22 @@
+[role="xpack"]
+[testenv="basic"]
+[[async-search-intro]]
+== Long-running searches
+
+{es} generally allows you to quickly search across big amounts of data. There are
+situations where a search executes on many many shards, possibly against
+<<frozen-indices,frozen indices>> and spanning multiple
+<<modules-remote-clusters,remote clusters>>, for which
+results are not expected to be returned in milliseconds. When you need to
+execute long-running searches, synchronously
+waiting for its results to be returned is not ideal. Instead, Async search lets
+you submit a search request that gets executed _asynchronously_,
+monitor the progress of the request, and retrieve results at a later stage.
+You can also retrieve partial results as they become available but
+before the search has completed.
+
+You can submit an async search request using the <<submit-async-search,submit
+async search>> API. The <<get-async-search,get async search>> API allows you to
+monitor the progress of an async search request and retrieve its results. An
+ongoing async search can be deleted through the <<delete-async-search,delete
+async search>> API.
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index fcfc85d7184c5..bce1d615f264d 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -26,6 +26,8 @@ include::query-dsl.asciidoc[]
 
 include::modules/cross-cluster-search.asciidoc[]
 
+include::async-search.asciidoc[]
+
 include::scripting.asciidoc[]
 
 include::mapping.asciidoc[]
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index b75961a52559e..42c4d9a8a5b44 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -300,7 +300,7 @@ POST /logs_write/_rollover <2>
   }
 }
 --------------------------------------------------
-// TEST[s/now/2016.10.31||/]
+// TEST[s/now/2016.10.31%7C%7C/]
 
 <1> Creates an index named with today's date (e.g.) `logs-2016.10.31-1`
 <2> Rolls over to a new index with today's date, e.g. `logs-2016.10.31-000002` if run immediately, or `logs-2016.11.01-000002` if run after 24 hours
@@ -339,7 +339,7 @@ over indices created in the last three days, you could do the following:
 GET /%3Clogs-%7Bnow%2Fd%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-1d%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-2d%7D-*%3E/_search
 --------------------------------------------------
 // TEST[continued]
-// TEST[s/now/2016.10.31||/]
+// TEST[s/now/2016.10.31%7C%7C/]
 
 
 [[rollover-index-api-dry-run-ex]]
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 655e1156f1938..f9073fb252e41 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -120,7 +120,7 @@ See <<native-realm-configuration>>.
 [role="exclude",id="native-settings"]
 ==== Native realm settings
 
-See <<ref-native-settings>>. 
+See <<ref-native-settings>>.
 
 [role="exclude",id="configuring-saml-realm"]
 === Configuring a SAML realm
@@ -130,27 +130,27 @@ See <<saml-guide>>.
 [role="exclude",id="saml-settings"]
 ==== SAML realm settings
 
-See <<ref-saml-settings>>. 
+See <<ref-saml-settings>>.
 
 [role="exclude",id="_saml_realm_signing_settings"]
 ==== SAML realm signing settings
 
-See <<ref-saml-signing-settings>>. 
+See <<ref-saml-signing-settings>>.
 
 [role="exclude",id="_saml_realm_encryption_settings"]
 ==== SAML realm encryption settings
 
-See <<ref-saml-encryption-settings>>. 
+See <<ref-saml-encryption-settings>>.
 
 [role="exclude",id="_saml_realm_ssl_settings"]
 ==== SAML realm SSL settings
 
-See <<ref-saml-ssl-settings>>. 
+See <<ref-saml-ssl-settings>>.
 
 [role="exclude",id="configuring-file-realm"]
 === Configuring a file realm
 
-See <<file-realm-configuration>>. 
+See <<file-realm-configuration>>.
 
 [role="exclude",id="ldap-user-search"]
 === User search mode and user DN templates mode
@@ -170,7 +170,7 @@ See <<ref-ldap-settings>>.
 [role="exclude",id="ldap-ssl"]
 === Setting up SSL between Elasticsearch and LDAP
 
-See <<tls-ldap>>. 
+See <<tls-ldap>>.
 
 [role="exclude",id="configuring-kerberos-realm"]
 === Configuring a Kerberos realm
@@ -211,7 +211,7 @@ See <<ref-ad-settings>>.
 [role="exclude",id="mapping-roles-ad"]
 === Mapping Active Directory users and groups to roles
 
-See <<ad-realm-configuration>>. 
+See <<ad-realm-configuration>>.
 
 [role="exclude",id="how-security-works"]
 === How security works
@@ -237,9 +237,9 @@ See the details in
 
 This page was deleted.
 [[ml-datafeed-chunking-config]]
-See the details in <<ml-put-datafeed>>, <<ml-update-datafeed>>, 
+See the details in <<ml-put-datafeed>>, <<ml-update-datafeed>>,
 [[ml-datafeed-delayed-data-check-config]]
-<<ml-get-datafeed>>, 
+<<ml-get-datafeed>>,
 [[ml-datafeed-counts]]
 <<ml-get-datafeed-stats>>.
 
@@ -323,7 +323,7 @@ See <<snapshots-register-repository>>.
 [role="exclude",id="ml-dfa-analysis-objects"]
 === Analysis configuration objects
 
-This page was deleted. 
+This page was deleted.
 See <<put-dfanalytics>>.
 
 [role="exclude",id="slm-api-delete"]
@@ -376,11 +376,6 @@ See <<slm-api-stop>>.
 
 See <<ccs-gateway-seed-nodes>> and <<ccs-min-roundtrips>>.
 
-[role="exclude",id="async-search"]
-=== Asynchronous search
-
-coming::[7.x]
-
 [role="exclude",id="indices-component-templates"]
 === Component template APIs
 
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
index e8a8a42da6a00..a33807cf698ac 100644
--- a/docs/reference/search.asciidoc
+++ b/docs/reference/search.asciidoc
@@ -152,6 +152,8 @@ high). This default value is `5`.
 
 include::search/search.asciidoc[]
 
+include::search/async-search.asciidoc[]
+
 include::search/uri-request.asciidoc[]
 
 include::search/request-body.asciidoc[]
diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
new file mode 100644
index 0000000000000..e58455146fb82
--- /dev/null
+++ b/docs/reference/search/async-search.asciidoc
@@ -0,0 +1,214 @@
+[role="xpack"]
+[testenv="basic"]
+[[async-search]]
+=== Async search
+
+The async search API let you asynchronously execute a
+search request, monitor its progress, and retrieve  partial results
+as they become available.
+
+[[submit-async-search]]
+==== Submit async search API
+
+Executes a search request asynchronously. It accepts the same
+parameters and request body as the <<search-search,search API>>.
+
+[source,console,id=submit-async-search-date-histogram-example]
+--------------------------------------------------
+POST /sales*/_async_search?size=0
+{
+    "sort" : [
+      { "date" : {"order" : "asc"} }
+    ],
+    "aggs" : {
+        "sale_date" : {
+             "date_histogram" : {
+                 "field" : "date",
+                 "calendar_interval": "1d"
+             }
+         }
+    }
+}
+--------------------------------------------------
+// TEST[setup:sales]
+// TEST[s/size=0/size=0&wait_for_completion=0/]
+
+The response contains an identifier of the search being executed.
+You can use this ID to later retrieve the search's final results.
+The currently available search
+results are returned as part of the <<search-api-response-body,`response`>> object.
+
+[source,console-result]
+--------------------------------------------------
+{
+  "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=", <1>
+  "version" : 0,
+  "is_partial" : true, <2>
+  "is_running" : true, <3>
+  "start_time_in_millis" : 1583945890986,
+  "expiration_time_in_millis" : 1584377890986,
+  "response" : {
+    "took" : 1122,
+    "timed_out" : false,
+    "num_reduce_phases" : 0,
+    "_shards" : {
+      "total" : 562, <4>
+      "successful" : 3, <5>
+      "skipped" : 0,
+      "failed" : 0
+    },
+    "hits" : {
+      "total" : {
+        "value" : 157483, <6>
+        "relation" : "gte"
+      },
+      "max_score" : null,
+      "hits" : [ ]
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/$body.id/]
+// TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/]
+// TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/"took" : 1122/"took": $body.response.took/]
+// TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/]
+// TESTRESPONSE[s/"successful" : 3/"successful": $body.response._shards.successful/]
+// TESTRESPONSE[s/"value" : 157483/"value": $body.response.hits.total.value/]
+
+<1> Identifier of the async search that can be used to monitor its progress, retrieve its results, and/or delete it.
+<2> Whether the returned search results are partial or final
+<3> Whether the search is still being executed or it has completed
+<4> How many shards the search will be executed on, overall
+<5> How many shards have successfully completed the search
+<6> How many documents are currently matching the query, which belong to the shards that have already completed the search
+
+It is possible to block and wait until the search is completed up to a certain
+timeout by providing the `wait_for_completion` parameter, which defaults to
+`1` second.
+
+You can also specify how long the async search needs to be
+available through the `keep_alive` parameter, which defaults to `5d` (five days).
+Ongoing async searches and any saved search results are deleted after this
+period.
+
+NOTE: When results are sorted by a numeric field, shards get sorted based on
+minimum and maximum value that they hold for that field, hence partial
+results become available following the sort criteria that was requested.
+
+The submit async search API supports the same <<search-search-api-query-params,parameters>>
+as the search API, though some have different default values:
+
+* `batched_reduce_size` defaults to `5`: this affects how often partial results
+become available, which happens whenever shard results are reduced. A partial
+reduction is performed every time the coordinating node has received a certain
+number of new shard responses (`5` by default).
+* `request_cache` defaults to `true`
+* `pre_filter_shard_size` defaults to `1`: this is to enforce the execution of
+a pre-filter roundtrip to retrieve statistics from each shard so that the ones
+that surely don't hold any document matching the query get skipped.
+* `ccs_minimize_roundtrips` defaults to `false`, which is also the only
+supported value
+
+WARNING: Async search does not support <<request-body-search-scroll,scroll>>
+nor search requests that only include the  <<search-suggesters,suggest section>>.
+{ccs} is supported only with <<ccs-min-roundtrips,`ccs_minimize_roundtrips`>>
+set to `false`.
+
+[[get-async-search]]
+==== Get async search
+
+The get async search API retrieves the results of a previously submitted
+async search request given its id. If the {es} {security-features} are enabled.
+the access to the results of a specific async search is restricted to the user
+that submitted it in the first place.
+
+[source,console,id=get-async-search-date-histogram-example]
+--------------------------------------------------
+GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=
+--------------------------------------------------
+// TEST[continued s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/\${body.id}/]
+
+[source,console-result]
+--------------------------------------------------
+{
+  "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=",
+  "version" : 2, <1>
+  "is_partial" : true, <2>
+  "is_running" : true, <3>
+  "start_time_in_millis" : 1583945890986,
+  "expiration_time_in_millis" : 1584377890986, <4>
+  "response" : {
+    "took" : 12144,
+    "timed_out" : false,
+    "num_reduce_phases" : 38,
+    "_shards" : {
+      "total" : 562,
+      "successful" : 188,
+      "skipped" : 0,
+      "failed" : 0
+    },
+    "hits" : {
+      "total" : {
+        "value" : 456433,
+        "relation" : "eq"
+      },
+      "max_score" : null,
+      "hits" : [ ]
+    },
+    "aggregations" : { <5>
+      "sale_date" :  {
+        "buckets" : []
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/$body.id/]
+// TESTRESPONSE[s/"is_partial" : true/"is_partial" : false/]
+// TESTRESPONSE[s/"is_running" : true/"is_running" : false/]
+// TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/]
+// TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/"took" : 12144/"took": $body.response.took/]
+// TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/]
+// TESTRESPONSE[s/"successful" : 188/"successful": $body.response._shards.successful/]
+// TESTRESPONSE[s/"value" : 456433/"value": $body.response.hits.total.value/]
+// TESTRESPONSE[s/"buckets" : \[\]/"buckets": $body.response.aggregations.sale_date.buckets/]
+// TESTRESPONSE[s/"num_reduce_phases" : 38,//]
+
+<1> The returned `version` is useful to identify whether the response contains
+additional results compared to previously obtained responses. If the version
+stays the same, no new results have become available, otherwise a higher version
+number indicates that more shards have completed their execution of the query
+and their partial results are also included in the response.
+<2> Whether the returned search results are partial or final
+<3> Whether the search is still being executed or it has completed
+<4> When the async search will expire
+<5> Partial aggregations results, coming from the shards that have already
+completed the execution of the query.
+
+The `wait_for_completion` parameter, which defaults to `1`, can also be provided
+when calling the Get Async Search API, in order to wait for the search to be
+completed up until the provided timeout. Final results will be returned if
+available before the timeout expires, otherwise the currently available results
+will be returned once the timeout expires.
+
+The `keep_alive` parameter specifies how long the async search should be
+available in the cluster. When not specified, the `keep_alive` set with the
+corresponding submit async request will be used. Otherwise, it is possible to
+override such value and extend the validity of the request. When this period
+expires, the search, if still running, is cancelled. If the search is
+completed, its saved results are deleted.
+
+[[delete-async-search]]
+==== Delete async search
+
+You can use the delete async search API to manually delete an async search
+by ID. If the search is still running, the search request will be cancelled.
+Otherwise, the saved search results are deleted.
+
+[source,console,id=delete-async-search-date-histogram-example]
+--------------------------------------------------
+DELETE /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=
+--------------------------------------------------
+// TEST[continued s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/\${body.id}/]
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
index 4486b90cb9a1d..10416b04ce5c1 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json
@@ -3,7 +3,7 @@
     "documentation":{
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
     },
-    "stability":"experimental",
+    "stability":"stable",
     "url":{
       "paths":[
         {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
index f5ea1424756e1..abe07c04869b7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
@@ -3,7 +3,7 @@
     "documentation":{
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
     },
-    "stability":"experimental",
+    "stability":"stable",
     "url":{
       "paths":[
         {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
index 033939acf0703..468ece1dfd2d3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
@@ -3,7 +3,7 @@
     "documentation":{
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"
     },
-    "stability":"experimental",
+    "stability":"stable",
     "url":{
       "paths":[
         {
@@ -32,6 +32,10 @@
         "description":"Specify the time that the request should block waiting for the final response",
         "default": "1s"
       },
+      "clean_on_completion":{
+        "type":"boolean",
+        "description":"Control whether the response should not be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)"
+      },
       "keep_alive": {
         "type": "time",
         "description": "Update the time interval in which the results (partial or final) for this search will be available"
@@ -41,6 +45,10 @@
         "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.",
         "default":5
       },
+      "request_cache":{
+        "type":"boolean",
+        "description":"Specify if request cache should be used for this request or not, defaults to true"
+      },
       "analyzer":{
         "type":"string",
         "description":"The analyzer to use for the query string"
@@ -204,18 +212,10 @@
         "type":"boolean",
         "description":"Specify whether to return sequence number and primary term of the last modification of each hit"
       },
-      "request_cache":{
-        "type":"boolean",
-        "description":"Specify if request cache should be used for this request or not, defaults to index level setting"
-      },
       "max_concurrent_shard_requests":{
         "type":"number",
         "description":"The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests",
         "default":5
-      },
-      "clean_on_completion":{
-        "type":"boolean",
-        "description":"Control whether the response should not be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)"
       }
     },
     "body":{

From 634c8c6ca34c5552f083f441e2b6327aba4a27f7 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Mar 2020 14:04:47 +0100
Subject: [PATCH 170/285] Rethrow Failed Assertion in BlobStoreTestUtils
 (#53859)

This fixes two issues:

1. Currently, the future here is never resolved on assertion error so a failing test would take a full minute
to complete until the future times out.
2. S3 tests overide this method to busy assert on this method. This only works if an assertion error makes it
to the calling thread.

Closes #53508
---
 .../blobstore/BlobStoreTestUtil.java          | 46 +++++++++++--------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
index 54ea0f26dcd0a..feabefd701447 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
@@ -112,27 +112,35 @@ public static boolean blobExists(BlobContainer container, String blobName) throw
      *                 of this assertion must pass an executor on those when using such an implementation.
      */
     public static void assertConsistency(BlobStoreRepository repository, Executor executor) {
-        final PlainActionFuture<Void> listener = PlainActionFuture.newFuture();
-        executor.execute(ActionRunnable.run(listener, () -> {
-            final BlobContainer blobContainer = repository.blobContainer();
-            final long latestGen;
-            try (DataInputStream inputStream = new DataInputStream(blobContainer.readBlob("index.latest"))) {
-                latestGen = inputStream.readLong();
-            } catch (NoSuchFileException e) {
-                throw new AssertionError("Could not find index.latest blob for repo [" + repository + "]");
-            }
-            assertIndexGenerations(blobContainer, latestGen);
-            final RepositoryData repositoryData;
-            try (InputStream blob = blobContainer.readBlob("index-" + latestGen);
-                 XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
-                     LoggingDeprecationHandler.INSTANCE, blob)) {
-                repositoryData = RepositoryData.snapshotsFromXContent(parser, latestGen);
+        final PlainActionFuture<AssertionError> listener = PlainActionFuture.newFuture();
+        executor.execute(ActionRunnable.supply(listener, () -> {
+            try {
+                final BlobContainer blobContainer = repository.blobContainer();
+                final long latestGen;
+                try (DataInputStream inputStream = new DataInputStream(blobContainer.readBlob("index.latest"))) {
+                    latestGen = inputStream.readLong();
+                } catch (NoSuchFileException e) {
+                    throw new AssertionError("Could not find index.latest blob for repo [" + repository + "]");
+                }
+                assertIndexGenerations(blobContainer, latestGen);
+                final RepositoryData repositoryData;
+                try (InputStream blob = blobContainer.readBlob("index-" + latestGen);
+                     XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                         LoggingDeprecationHandler.INSTANCE, blob)) {
+                    repositoryData = RepositoryData.snapshotsFromXContent(parser, latestGen);
+                }
+                assertIndexUUIDs(blobContainer, repositoryData);
+                assertSnapshotUUIDs(repository, repositoryData);
+                assertShardIndexGenerations(blobContainer, repositoryData.shardGenerations());
+                return null;
+            } catch (AssertionError e) {
+                return e;
             }
-            assertIndexUUIDs(blobContainer, repositoryData);
-            assertSnapshotUUIDs(repository, repositoryData);
-            assertShardIndexGenerations(blobContainer, repositoryData.shardGenerations());
         }));
-        listener.actionGet(TimeValue.timeValueMinutes(1L));
+        final AssertionError err = listener.actionGet(TimeValue.timeValueMinutes(1L));
+        if (err != null) {
+            throw new AssertionError(err);
+        }
     }
 
     private static void assertIndexGenerations(BlobContainer repoRoot, long latestGen) throws IOException {

From a8c501b5a14aada9fa0a6394df9ec86358394fb7 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Fri, 20 Mar 2020 09:40:51 -0400
Subject: [PATCH 171/285] [DOCS] Add redirects for missing data stream API docs
 (#53866)

The following API spec files contain a link to a not-yet-created
docs page for the data stream APIs:

* [indices.create_data_stream.json][0]
* [indices.delete_data_stream.jsonn][1]
* [indices.get_data_streams.json][2]

The Elaticsearch-JS client uses these spec files to create their docs.
This created a broken link in the Elaticsearch-JS, which has broken
the docs build.

This PR adds a temporary redirect for the docs page. This redirect
should be removed when the actual API docs are added.

[0]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
[1]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
[2]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json

Relates to #53558.

CC @martijnvg
---
 docs/reference/redirects.asciidoc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index f9073fb252e41..03d54def6a054 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -381,6 +381,11 @@ See <<ccs-gateway-seed-nodes>> and <<ccs-min-roundtrips>>.
 
 coming::[7.x]
 
+[role="exclude",id="data-streams"]
+=== Data stream APIs
+
+coming::[7.x]
+
 [role="exclude",id="cat-transform"]
 === cat transform API
 

From 4a663971d5f05b1b389cbd3e18c60da250736238 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 20 Mar 2020 14:56:51 +0000
Subject: [PATCH 172/285] Use consistent threadpools in CoordinatorTests
 (#53868)

Today in the `CoordinatorTests` each node uses multiple threadpools. This is
mostly fine as they are almost completely stateless, except for the
`ThreadContext`: by using multiple threadpools we cannot make assertions that
the thread context is/isn't preserved as we expect. This commit consolidates
the threadpool instances in use so that each node uses just one.
---
 .../PublicationTransportHandler.java          |  2 ++
 .../cluster/coordination/NodeJoinTests.java   |  5 ++--
 .../snapshots/SnapshotResiliencyTests.java    |  8 ++---
 .../AbstractCoordinatorTestCase.java          | 30 +++++++++----------
 .../MockSinglePrioritizingExecutor.java       |  5 ++--
 .../service/FakeThreadPoolMasterService.java  | 21 +++++--------
 .../MockSinglePrioritizingExecutorTests.java  |  2 +-
 .../FakeThreadPoolMasterServiceTests.java     | 10 ++++++-
 8 files changed, 44 insertions(+), 39 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index a4b2f509b2ed8..1a6db35a8091e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -161,6 +161,7 @@ public PublicationContext newPublicationContext(ClusterChangedEvent clusterChang
             public void sendPublishRequest(DiscoveryNode destination, PublishRequest publishRequest,
                                            ActionListener<PublishWithJoinResponse> originalListener) {
                 assert publishRequest.getAcceptedState() == clusterChangedEvent.state() : "state got switched on us";
+                assert transportService.getThreadPool().getThreadContext().isSystemContext();
                 final ActionListener<PublishWithJoinResponse> responseActionListener;
                 if (destination.equals(nodes.getLocalNode())) {
                     // if publishing to self, use original request instead (see currentPublishRequestToSelf for explanation)
@@ -197,6 +198,7 @@ public void onFailure(Exception e) {
             @Override
             public void sendApplyCommit(DiscoveryNode destination, ApplyCommitRequest applyCommitRequest,
                                         ActionListener<TransportResponse.Empty> responseActionListener) {
+                assert transportService.getThreadPool().getThreadContext().isSystemContext();
                 transportService.sendRequest(destination, COMMIT_STATE_ACTION_NAME, applyCommitRequest, stateRequestOptions,
                     new TransportResponseHandler<TransportResponse.Empty>() {
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index ac3075597dc57..6557d647c75fa 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -122,9 +122,10 @@ private static ClusterState initialState(DiscoveryNode localNode, long term, lon
     private void setupFakeMasterServiceAndCoordinator(long term, ClusterState initialState) {
         deterministicTaskQueue
             = new DeterministicTaskQueue(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), random());
+        final ThreadPool fakeThreadPool = deterministicTaskQueue.getThreadPool();
         FakeThreadPoolMasterService fakeMasterService = new FakeThreadPoolMasterService("test_node","test",
-                deterministicTaskQueue::scheduleNow);
-        setupMasterServiceAndCoordinator(term, initialState, fakeMasterService, deterministicTaskQueue.getThreadPool(), Randomness.get());
+            fakeThreadPool, deterministicTaskQueue::scheduleNow);
+        setupMasterServiceAndCoordinator(term, initialState, fakeMasterService, fakeThreadPool, Randomness.get());
         fakeMasterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
             coordinator.handlePublishRequest(new PublishRequest(event.state()));
             publishListener.onResponse(null);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 062e64ba55b74..2139268fe3888 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1163,15 +1163,15 @@ private final class TestClusterNode {
             TestClusterNode(DiscoveryNode node) throws IOException {
                 this.node = node;
                 final Environment environment = createEnvironment(node.getName());
-                masterService = new FakeThreadPoolMasterService(node.getName(), "test", deterministicTaskQueue::scheduleNow);
+                threadPool = deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable));
+                masterService = new FakeThreadPoolMasterService(node.getName(), "test", threadPool, deterministicTaskQueue::scheduleNow);
                 final Settings settings = environment.settings();
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-                threadPool = deterministicTaskQueue.getThreadPool();
                 clusterService = new ClusterService(settings, clusterSettings, masterService,
                     new ClusterApplierService(node.getName(), settings, clusterSettings, threadPool) {
                         @Override
                         protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-                            return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue);
+                            return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool);
                         }
 
                         @Override
@@ -1211,7 +1211,7 @@ protected NamedWriteableRegistry writeableRegistry() {
                     }
                 };
                 transportService = mockTransport.createTransportService(
-                    settings, deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable)),
+                    settings, threadPool,
                     new TransportInterceptor() {
                         @Override
                         public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
index 5239827f2b171..491bd7d858091 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -911,6 +911,7 @@ class ClusterNode {
             }
 
             private void setUp() {
+                final ThreadPool threadPool = deterministicTaskQueue.getThreadPool(this::onNode);
                 mockTransport = new DisruptableMockTransport(localNode, logger) {
                     @Override
                     protected void execute(Runnable runnable) {
@@ -928,24 +929,20 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                             .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
                     }
                 };
-
                 final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
                     nodeSettings : Settings.builder().put(nodeSettings)
                     .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
                         ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
-                transportService = mockTransport.createTransportService(
-                    settings, deterministicTaskQueue.getThreadPool(this::onNode),
-                    getTransportInterceptor(localNode, deterministicTaskQueue.getThreadPool(this::onNode)),
-                    a -> localNode, null, emptySet());
-                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
+                transportService = mockTransport.createTransportService(settings, threadPool,
+                    getTransportInterceptor(localNode, threadPool), a -> localNode, null, emptySet());
+                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test", threadPool,
                     runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
                 clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
-                    deterministicTaskQueue, this::onNode);
+                    deterministicTaskQueue, threadPool);
                 clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
                 clusterService.setNodeConnectionsService(
-                    new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
-                        transportService));
+                    new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService));
                 final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
                     Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
                 final AllocationService allocationService = ESAllocationTestCase.createAllocationService(Settings.EMPTY);
@@ -954,8 +951,8 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get(), (s, p, r) -> {},
                     getElectionStrategy());
                 masterService.setClusterStatePublisher(coordinator);
-                final GatewayService gatewayService = new GatewayService(settings, allocationService, clusterService,
-                    deterministicTaskQueue.getThreadPool(this::onNode), coordinator, null);
+                final GatewayService gatewayService
+                    = new GatewayService(settings, allocationService, clusterService, threadPool, coordinator, null);
 
                 logger.trace("starting up [{}]", localNode);
                 transportService.start();
@@ -1292,8 +1289,9 @@ static class AckedFakeThreadPoolMasterService extends FakeThreadPoolMasterServic
 
         AckCollector nextAckCollector = new AckCollector();
 
-        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
-            super(nodeName, serviceName, onTaskAvailableToRun);
+        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, ThreadPool threadPool,
+                                         Consumer<Runnable> onTaskAvailableToRun) {
+            super(nodeName, serviceName, threadPool, onTaskAvailableToRun);
         }
 
         @Override
@@ -1323,8 +1321,8 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
         private boolean applicationMayFail;
 
         DisruptableClusterApplierService(String nodeName, Settings settings, ClusterSettings clusterSettings,
-                                         DeterministicTaskQueue deterministicTaskQueue, Function<Runnable, Runnable> runnableWrapper) {
-            super(nodeName, settings, clusterSettings, deterministicTaskQueue.getThreadPool(runnableWrapper));
+                                         DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
+            super(nodeName, settings, clusterSettings, threadPool);
             this.nodeName = nodeName;
             this.deterministicTaskQueue = deterministicTaskQueue;
             addStateApplier(event -> {
@@ -1344,7 +1342,7 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
 
         @Override
         protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue);
+            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue, threadPool);
         }
 
         @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
index add5014e555fc..553188ac24539 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.TimeUnit;
 
@@ -29,7 +30,7 @@
  */
 public class MockSinglePrioritizingExecutor extends PrioritizedEsThreadPoolExecutor {
 
-    public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue) {
+    public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
         super(name, 0, 1, 0L, TimeUnit.MILLISECONDS,
             r -> new Thread() {
                 @Override
@@ -51,7 +52,7 @@ public String toString() {
                     });
                 }
             },
-            deterministicTaskQueue.getThreadPool().getThreadContext(), deterministicTaskQueue.getThreadPool().scheduler());
+            threadPool.getThreadContext(), threadPool.scheduler());
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java b/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
index 74d9cc17a5325..dfa11d8beb049 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java
@@ -41,8 +41,6 @@
 
 import static org.apache.lucene.util.LuceneTestCase.random;
 import static org.elasticsearch.test.ESTestCase.randomInt;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class FakeThreadPoolMasterService extends MasterService {
     private static final Logger logger = LogManager.getLogger(FakeThreadPoolMasterService.class);
@@ -54,21 +52,14 @@ public class FakeThreadPoolMasterService extends MasterService {
     private boolean taskInProgress = false;
     private boolean waitForPublish = false;
 
-    public FakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
+    public FakeThreadPoolMasterService(String nodeName, String serviceName, ThreadPool threadPool,
+                                       Consumer<Runnable> onTaskAvailableToRun) {
         super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build(),
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            createMockThreadPool());
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool);
         this.name = serviceName;
         this.onTaskAvailableToRun = onTaskAvailableToRun;
     }
 
-    private static ThreadPool createMockThreadPool() {
-        final ThreadContext context = new ThreadContext(Settings.EMPTY);
-        final ThreadPool mockThreadPool = mock(ThreadPool.class);
-        when(mockThreadPool.getThreadContext()).thenReturn(context);
-        return mockThreadPool;
-    }
-
     @Override
     protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
         return new PrioritizedEsThreadPoolExecutor(name, 1, 1, 1, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(name),
@@ -110,7 +101,11 @@ public void run() {
                     final Runnable task = pendingTasks.remove(taskIndex);
                     taskInProgress = true;
                     scheduledNextTask = false;
-                    task.run();
+                    final ThreadContext threadContext = threadPool.getThreadContext();
+                    try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                        threadContext.markAsSystemContext();
+                        task.run();
+                    }
                     if (waitForPublish == false) {
                         taskInProgress = false;
                     }
diff --git a/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java b/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
index 427f0d32c1270..66424de7f75d6 100644
--- a/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutorTests.java
@@ -29,7 +29,7 @@ public class MockSinglePrioritizingExecutorTests extends ESTestCase {
 
     public void testPrioritizedEsThreadPoolExecutor() {
         final DeterministicTaskQueue taskQueue = DeterministicTaskQueueTests.newTaskQueue();
-        final PrioritizedEsThreadPoolExecutor executor = new MockSinglePrioritizingExecutor("test", taskQueue);
+        final PrioritizedEsThreadPoolExecutor executor = new MockSinglePrioritizingExecutor("test", taskQueue, taskQueue.getThreadPool());
         final AtomicBoolean called1 = new AtomicBoolean();
         final AtomicBoolean called2 = new AtomicBoolean();
         executor.execute(new PrioritizedRunnable(Priority.NORMAL) {
diff --git a/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java b/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
index f97477707e461..c3e162254421b 100644
--- a/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterServiceTests.java
@@ -27,7 +27,10 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -37,6 +40,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class FakeThreadPoolMasterServiceTests extends ESTestCase {
 
@@ -48,7 +53,10 @@ public void testFakeMasterService() {
         lastClusterStateRef.set(ClusterStateCreationUtils.state(discoveryNode, discoveryNode));
         long firstClusterStateVersion = lastClusterStateRef.get().version();
         AtomicReference<ActionListener<Void>> publishingCallback = new AtomicReference<>();
-        FakeThreadPoolMasterService masterService = new FakeThreadPoolMasterService("test_node","test", runnableTasks::add);
+        final ThreadContext context = new ThreadContext(Settings.EMPTY);
+        final ThreadPool mockThreadPool = mock(ThreadPool.class);
+        when(mockThreadPool.getThreadContext()).thenReturn(context);
+        FakeThreadPoolMasterService masterService = new FakeThreadPoolMasterService("test_node","test", mockThreadPool, runnableTasks::add);
         masterService.setClusterStateSupplier(lastClusterStateRef::get);
         masterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
             lastClusterStateRef.set(event.state());

From fc083493d24d882eab7fb76bd011317424e88713 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Fri, 20 Mar 2020 15:56:49 +0100
Subject: [PATCH 173/285] [DOCS] correct async search note

The sort optimization kicks in whenever results are sorted by field.
---
 docs/reference/search/async-search.asciidoc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
index e58455146fb82..1ec22e98d65d1 100644
--- a/docs/reference/search/async-search.asciidoc
+++ b/docs/reference/search/async-search.asciidoc
@@ -92,9 +92,10 @@ available through the `keep_alive` parameter, which defaults to `5d` (five days)
 Ongoing async searches and any saved search results are deleted after this
 period.
 
-NOTE: When results are sorted by a numeric field, shards get sorted based on
-minimum and maximum value that they hold for that field, hence partial
-results become available following the sort criteria that was requested.
+NOTE: When the primary sort of the results is an indexed field, shards get
+sorted based on minimum and maximum value that they hold for that field,
+hence partial results become available following the sort criteria that
+was requested.
 
 The submit async search API supports the same <<search-search-api-query-params,parameters>>
 as the search API, though some have different default values:

From 0f7cf7b720492a6cf6dcf9e2271fdfcdcc593d42 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Fri, 20 Mar 2020 15:38:43 +0000
Subject: [PATCH 174/285] TermsLookup fields should be marked as Required in
 docs (#53784)

The terms-lookup section of our terms query docs currently state that the
index, id and path fields are optional. They should be marked instead
as required.
---
 docs/reference/query-dsl/terms-query.asciidoc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index 85605457ea315..05d6a85566678 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -22,7 +22,7 @@ GET /_search
     "query" : {
         "terms" : {
             "user" : ["kimchy", "elasticsearch"],
-            "boost" : 1.0 
+            "boost" : 1.0
         }
     }
 }
@@ -93,16 +93,16 @@ To perform a terms lookup, use the following parameters.
 [[query-dsl-terms-lookup-params]]
 ====== Terms lookup parameters
 `index`::
-(Optional, string) Name of the index from which to fetch field values.
+(Required, string) Name of the index from which to fetch field values.
 
 `id`::
-(Optional, string) <<mapping-id-field,ID>> of the document from which to fetch
+(Required, string) <<mapping-id-field,ID>> of the document from which to fetch
 field values.
 
 `path`::
 +
 --
-(Optional, string) Name of the field from which to fetch field values. {es} uses
+(Required, string) Name of the field from which to fetch field values. {es} uses
 these values as search terms for the query.
 
 If the field values include an array of nested inner objects, you can access
@@ -117,7 +117,7 @@ when the document was indexed, this parameter is required.
 [[query-dsl-terms-lookup-example]]
 ====== Terms lookup example
 
-To see how terms lookup works, try the following example.  
+To see how terms lookup works, try the following example.
 
 . Create an index with a `keyword` field named `color`.
 +
@@ -244,4 +244,4 @@ field, {es} returns both documents.
 }
 ----
 // TESTRESPONSE[s/"took" : 17/"took" : $body.took/]
---
\ No newline at end of file
+--

From 1b5d64d9ae0c574e3ee32cb92dd44ec220bbb40a Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Fri, 20 Mar 2020 12:12:29 -0400
Subject: [PATCH 175/285] Fix random failures in
 InternalTopHitsTests#testReduceRandom (#53832)

The test was randomly and very rarely failing due to generating the same sort
key for multiple records, which was making order of these records in the results
nondeterministic. While investigating the test I also found that the data wasn't
generated in the way that matches the actual data. Normally, the order of
documents in hits and scoreDocs in InternalTopHits should be the same. However,
in the test only scoreDocs were sorted which was cause very confusing failure
messages. This commit fixes this issue as well.

Fixes #53676
---
 .../metrics/InternalTopHitsTests.java         | 31 +++++++++++++++++--
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java
index b2973b67ce3c1..3ebe75442af2b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 
@@ -70,6 +71,12 @@ public class InternalTopHitsTests extends InternalAggregationTestCase<InternalTo
      */
     private SortField[] testInstancesSortFields;
 
+    /**
+     * Collects all generated scores and fields to ensure that all scores are unique. That is necessary for deterministic results
+     */
+    private Set<Float> usedScores = new HashSet<>();
+    private Set<Object> usedFields = new HashSet<>();
+
     @Override
     public void setUp() throws Exception {
         super.setUp();
@@ -88,7 +95,8 @@ protected InternalTopHits createTestInstance(String name, List<PipelineAggregato
         SearchHit[] hits = new SearchHit[actualSize];
         Set<Integer> usedDocIds = new HashSet<>();
         for (int i = 0; i < actualSize; i++) {
-            float score = randomFloat();
+            float score = randomValueOtherThanMany(usedScores::contains, ESTestCase::randomFloat);
+            usedScores.add(score);
             maxScore = max(maxScore, score);
             int docId = randomValueOtherThanMany(usedDocIds::contains, () -> between(0, IndexWriter.MAX_DOCS));
             usedDocIds.add(docId);
@@ -97,7 +105,9 @@ protected InternalTopHits createTestInstance(String name, List<PipelineAggregato
             if (testInstancesLookSortedByField) {
                 Object[] fields = new Object[testInstancesSortFields.length];
                 for (int f = 0; f < testInstancesSortFields.length; f++) {
-                    fields[f] = randomOfType(testInstancesSortFields[f].getType());
+                    final int ff = f;
+                    fields[f] = randomValueOtherThanMany(usedFields::contains, () -> randomOfType(testInstancesSortFields[ff].getType()));
+                    usedFields.add(fields[f]);
                 }
                 scoreDocs[i] = new FieldDoc(docId, score, fields);
             } else {
@@ -107,10 +117,10 @@ protected InternalTopHits createTestInstance(String name, List<PipelineAggregato
             hits[i].score(score);
         }
         int totalHits = between(actualSize, 500000);
+        sort(hits, scoreDocs, scoreDocComparator());
         SearchHits searchHits = new SearchHits(hits, new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), maxScore);
 
         TopDocs topDocs;
-        Arrays.sort(scoreDocs, scoreDocComparator());
         if (testInstancesLookSortedByField) {
             topDocs = new TopFieldDocs(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), scoreDocs, testInstancesSortFields);
         } else {
@@ -122,6 +132,21 @@ protected InternalTopHits createTestInstance(String name, List<PipelineAggregato
         return new InternalTopHits(name, from, requestedSize, topDocsAndMaxScore, searchHits, pipelineAggregators, metaData);
     }
 
+    /**
+     * Sorts both searchHits and scoreDocs together based on scoreDocComparator()
+     */
+    private void sort(SearchHit[] searchHits, ScoreDoc[] scoreDocs, Comparator<ScoreDoc> comparator) {
+        List<Tuple<SearchHit, ScoreDoc>> hitScores = new ArrayList<>();
+        for (int i = 0; i < searchHits.length; i++) {
+            hitScores.add(new Tuple<>(searchHits[i], scoreDocs[i]));
+        }
+        hitScores.sort((t1, t2) -> comparator.compare(t1.v2(), t2.v2()));
+        for (int i = 0; i < searchHits.length; i++) {
+            searchHits[i] = hitScores.get(i).v1();
+            scoreDocs[i] = hitScores.get(i).v2();
+        }
+    }
+
     @Override
     protected void assertFromXContent(InternalTopHits aggregation, ParsedAggregation parsedAggregation) throws IOException {
         final SearchHits expectedSearchHits = aggregation.getHits();

From 0bc8bb3e2e800b8f69833c8f7cb9058c53f2615e Mon Sep 17 00:00:00 2001
From: Gordon Brown <gordon.brown@elastic.co>
Date: Fri, 20 Mar 2020 10:15:27 -0600
Subject: [PATCH 176/285] Add isHidden to AliasActions equals/hashcode (#53700)

This commit adds the `isHidden` flag to the `equals` and
`hashCode` methods for `AliasActions`.
---
 .../action/admin/indices/alias/IndicesAliasesRequest.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 39f9d7239f5ce..8c77bfa24ebbf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -553,12 +553,13 @@ public boolean equals(Object obj) {
                     && Objects.equals(routing, other.routing)
                     && Objects.equals(indexRouting, other.indexRouting)
                     && Objects.equals(searchRouting, other.searchRouting)
-                    && Objects.equals(writeIndex, other.writeIndex);
+                    && Objects.equals(writeIndex, other.writeIndex)
+                    && Objects.equals(isHidden, other.isHidden);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(type, indices, aliases, filter, routing, indexRouting, searchRouting, writeIndex);
+            return Objects.hash(type, indices, aliases, filter, routing, indexRouting, searchRouting, writeIndex, isHidden);
         }
     }
 

From f31c6e969b445703ee22e6220864de9278bfb811 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Mar 2020 17:24:27 +0100
Subject: [PATCH 177/285] Upgrade to Azure SDK 8.6.2 (#53865)

This fixes some bugs around retrying and URL encoding and should enable a follow-up
that finally adds bulk deletes on Azure.
---
 plugins/repository-azure/build.gradle                          | 2 +-
 plugins/repository-azure/licenses/azure-storage-8.4.0.jar.sha1 | 1 -
 plugins/repository-azure/licenses/azure-storage-8.6.2.jar.sha1 | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/azure-storage-8.4.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/azure-storage-8.6.2.jar.sha1

diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 33ab95e6ea405..418eccc87ecfa 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -25,7 +25,7 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.microsoft.azure:azure-storage:8.4.0'
+  compile 'com.microsoft.azure:azure-storage:8.6.2'
   compile 'com.microsoft.azure:azure-keyvault-core:1.0.0'
   compile 'com.google.guava:guava:20.0'
   compile 'org.apache.commons:commons-lang3:3.4'
diff --git a/plugins/repository-azure/licenses/azure-storage-8.4.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-8.4.0.jar.sha1
deleted file mode 100644
index db3b2baba0644..0000000000000
--- a/plugins/repository-azure/licenses/azure-storage-8.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-002c6b7827f06869b8d04880bf913ce4efcc9ad4
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-8.6.2.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-8.6.2.jar.sha1
new file mode 100644
index 0000000000000..0f8f24231fbdb
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-storage-8.6.2.jar.sha1
@@ -0,0 +1 @@
+d1b6de7264205e2441c667dfee5b002bbac61644
\ No newline at end of file

From 0dddfa4ed2ddb82c30e843eb5ce41eda505f50d6 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Fri, 20 Mar 2020 10:25:21 -0700
Subject: [PATCH 178/285] [DOCS] Updates list of transform aggs (#53820)

---
 docs/reference/rest-api/common-parms.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index baaea29f7e7e6..d297b70fa5553 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -593,10 +593,12 @@ are supported:
 * <<search-aggregations-metrics-avg-aggregation,Average>>
 * <<search-aggregations-metrics-weight-avg-aggregation,Weighted average>>
 * <<search-aggregations-metrics-cardinality-aggregation,Cardinality>>
+* <<search-aggregations-bucket-filter-aggregation,Filter>>
 * <<search-aggregations-metrics-geobounds-aggregation,Geo bounds>>
 * <<search-aggregations-metrics-geocentroid-aggregation,Geo centroid>>
 * <<search-aggregations-metrics-max-aggregation,Max>>
 * <<search-aggregations-metrics-min-aggregation,Min>>
+* <<search-aggregations-metrics-percentile-aggregation,Percentiles>>
 * <<search-aggregations-metrics-scripted-metric-aggregation,Scripted metric>>
 * <<search-aggregations-metrics-sum-aggregation,Sum>>
 * <<search-aggregations-metrics-valuecount-aggregation,Value count>>

From 065fc151d4b6848a1189f0ecfe62accf17e8f6ea Mon Sep 17 00:00:00 2001
From: Aleksandr Maus <aleksandr.maus@elastic.co>
Date: Fri, 20 Mar 2020 13:33:46 -0400
Subject: [PATCH 179/285] EQL: EqlActionIT improvements (#53780)

Related to https://github.com/elastic/elasticsearch/issues/53598
---
 x-pack/plugin/eql/build.gradle                |  12 --
 x-pack/plugin/eql/qa/common/build.gradle      |   5 +
 .../qa/common/licenses/jtoml-2.0.0.jar.sha1   |   1 +
 .../eql/qa/common/licenses/jtoml-LICENSE      | 202 ++++++++++++++++++
 .../eql/qa/common/licenses/jtoml-NOTICE       |   0
 .../test/eql/CommonEqlActionTestCase.java}    | 105 ++++++---
 .../org/elasticsearch/test/eql}/EqlSpec.java  |   2 +-
 .../test/eql}/EqlSpecLoader.java              |   2 +-
 .../common/src/main}/resources/test_data.json |   0
 .../src/main}/resources/test_queries.toml     |   0
 .../resources/test_queries_unsupported.toml   |   0
 .../elasticsearch/xpack/eql/EqlActionIT.java  |  17 ++
 .../eql/action/AbstractEqlIntegTestCase.java  |  41 ----
 .../eql/action/LocalStateEqlXPackPlugin.java  |  28 ---
 14 files changed, 308 insertions(+), 107 deletions(-)
 create mode 100644 x-pack/plugin/eql/qa/common/licenses/jtoml-2.0.0.jar.sha1
 create mode 100644 x-pack/plugin/eql/qa/common/licenses/jtoml-LICENSE
 create mode 100644 x-pack/plugin/eql/qa/common/licenses/jtoml-NOTICE
 rename x-pack/plugin/eql/{src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java => qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java} (55%)
 rename x-pack/plugin/eql/{src/test/java/org/elasticsearch/xpack/eql/action => qa/common/src/main/java/org/elasticsearch/test/eql}/EqlSpec.java (97%)
 rename x-pack/plugin/eql/{src/test/java/org/elasticsearch/xpack/eql/action => qa/common/src/main/java/org/elasticsearch/test/eql}/EqlSpecLoader.java (98%)
 rename x-pack/plugin/eql/{src/test => qa/common/src/main}/resources/test_data.json (100%)
 rename x-pack/plugin/eql/{src/test => qa/common/src/main}/resources/test_queries.toml (100%)
 rename x-pack/plugin/eql/{src/test => qa/common/src/main}/resources/test_queries_unsupported.toml (100%)
 create mode 100644 x-pack/plugin/eql/qa/rest/src/test/java/org/elasticsearch/xpack/eql/EqlActionIT.java
 delete mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java
 delete mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEqlXPackPlugin.java

diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle
index 97713aa8f7ee2..13b12a0d88799 100644
--- a/x-pack/plugin/eql/build.gradle
+++ b/x-pack/plugin/eql/build.gradle
@@ -20,15 +20,6 @@ archivesBaseName = 'x-pack-eql'
 // All integration tests live in qa modules
 integTest.enabled = false
 
-// Instead we create a separate task to run the tests based on ESIntegTestCase
-task internalClusterTest(type: Test) {
-  mustRunAfter test
-  include '**/*IT.class'
-  systemProperty 'es.set.netty.runtime.available.processors', 'false'
-}
-
-check.dependsOn internalClusterTest
-
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   compileOnly(project(':modules:lang-painless')) {
@@ -43,9 +34,6 @@ dependencies {
   testCompile project(path: ':modules:reindex', configuration: 'runtime')
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
   testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
-
-  // TOML parser for EqlActionIT tests
-  testCompile 'io.ous:jtoml:2.0.0'
 }
 
 
diff --git a/x-pack/plugin/eql/qa/common/build.gradle b/x-pack/plugin/eql/qa/common/build.gradle
index e1c6fb4f95bea..6d98001739117 100644
--- a/x-pack/plugin/eql/qa/common/build.gradle
+++ b/x-pack/plugin/eql/qa/common/build.gradle
@@ -3,4 +3,9 @@ test.enabled = false
 
 dependencies {
   compile project(':test:framework')
+  compile project(path: xpackModule('core'), configuration: 'default')
+  compile project(path: xpackModule('core'), configuration: 'testArtifacts')
+
+  // TOML parser for EqlActionIT tests
+  compile 'io.ous:jtoml:2.0.0'
 }
diff --git a/x-pack/plugin/eql/qa/common/licenses/jtoml-2.0.0.jar.sha1 b/x-pack/plugin/eql/qa/common/licenses/jtoml-2.0.0.jar.sha1
new file mode 100644
index 0000000000000..ccf73fed50e38
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/licenses/jtoml-2.0.0.jar.sha1
@@ -0,0 +1 @@
+2085505b2b173e14b2a7702edd703ce185fd6206
diff --git a/x-pack/plugin/eql/qa/common/licenses/jtoml-LICENSE b/x-pack/plugin/eql/qa/common/licenses/jtoml-LICENSE
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/licenses/jtoml-LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/eql/qa/common/licenses/jtoml-NOTICE b/x-pack/plugin/eql/qa/common/licenses/jtoml-NOTICE
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java
similarity index 55%
rename from x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
rename to x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java
index b2e5c2c89abe1..a8d4cbc75defb 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java
@@ -4,19 +4,25 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.eql.action;
+package org.elasticsearch.test.eql;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.Build;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.eql.EqlSearchRequest;
+import org.elasticsearch.client.eql.EqlSearchResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -25,10 +31,11 @@
 import java.util.HashMap;
 import java.util.List;
 
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
-public class EqlActionIT extends AbstractEqlIntegTestCase {
+public abstract class CommonEqlActionTestCase extends ESRestTestCase {
+
+    private RestHighLevelClient highLevelClient;
 
     static final String indexPrefix = "endgame";
     static final String testIndexName = indexPrefix + "-1.4.0";
@@ -39,12 +46,20 @@ public static void checkForSnapshot() {
         assumeTrue("Only works on snapshot builds for now", Build.CURRENT.isSnapshot());
     }
 
-    @Before
+    private static boolean isSetUp = false;
+    private static int counter = 0;
+
     @SuppressWarnings("unchecked")
-    public void setUpData() throws Exception {
-        // Insert test data
-        BulkRequestBuilder bulkBuilder = client().prepareBulk();
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, EqlActionIT.class.getResourceAsStream("/test_data.json"))) {
+    private static void setupData(CommonEqlActionTestCase tc) throws Exception {
+        if (isSetUp) {
+            return;
+        }
+
+        BulkRequest bulk = new BulkRequest();
+        bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        try (XContentParser parser = tc.createParser(JsonXContent.jsonXContent,
+                CommonEqlActionTestCase.class.getResourceAsStream("/test_data.json"))) {
             List<Object> list = parser.list();
             for (Object item : list) {
                 assertThat(item, instanceOf(HashMap.class));
@@ -60,18 +75,40 @@ public void setUpData() throws Exception {
                 // Add @timestamp field
                 entry.put("@timestamp", entry.get("timestamp"));
 
-                bulkBuilder.add(new IndexRequest(testIndexName).source(entry, XContentType.JSON));
+                bulk.add(new IndexRequest(testIndexName).source(entry, XContentType.JSON));
             }
         }
-        BulkResponse bulkResponse = bulkBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
-        assertThat(bulkResponse.hasFailures() ? bulkResponse.buildFailureMessage() : "", bulkResponse.hasFailures(), equalTo(false));
 
-        ensureYellow(testIndexName);
+        if (bulk.numberOfActions() > 0) {
+            BulkResponse bulkResponse = tc.highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
+            assertEquals(RestStatus.OK, bulkResponse.status());
+            assertFalse(bulkResponse.hasFailures());
+            isSetUp = true;
+        }
+    }
+
+    private static void cleanupData(CommonEqlActionTestCase tc) throws Exception {
+        // Delete index after all tests ran
+        if (isSetUp && (--counter == 0)) {
+            deleteIndex(testIndexName);
+            isSetUp = false;
+        }
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        // Need to preserve data between parameterized tests runs
+        return true;
+    }
+
+    @Before
+    public void setup() throws Exception {
+        setupData(this);
     }
 
     @After
-    public void tearDownData() {
-        client().admin().indices().prepareDelete(testIndexName).get();
+    public void cleanup() throws Exception {
+        cleanupData(this);
     }
 
     @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
@@ -83,7 +120,6 @@ public static List<Object[]> readTestSpecs() throws Exception {
         List<EqlSpec> unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false);
 
         // Validate only currently supported specs
-        int num = 1; // Seq number of the test
         for (EqlSpec spec : specs) {
             boolean supported = true;
             // Check if spec is supported, simple iteration, cause the list is short.
@@ -95,20 +131,36 @@ public static List<Object[]> readTestSpecs() throws Exception {
             }
 
             if (supported) {
-                testSpecs.add(new Object[]{num++, spec});
+                String name = spec.description();
+                if (Strings.isNullOrEmpty(name)) {
+                    name = spec.note();
+                }
+                if (Strings.isNullOrEmpty(name)) {
+                    name = spec.query();
+                }
+
+                testSpecs.add(new Object[]{++counter, name, spec});
             }
         }
         return testSpecs;
     }
 
     private final int num;
+    private final String name;
     private final EqlSpec spec;
 
-    public EqlActionIT(int num, EqlSpec spec) {
+    public CommonEqlActionTestCase(int num, String name, EqlSpec spec) {
         this.num = num;
+        this.name = name;
         this.spec = spec;
     }
 
+    public void test() throws Exception {
+        EqlSearchRequest request = new EqlSearchRequest(testIndexName, spec.query());
+        EqlSearchResponse response = highLevelClient().eql().search(request, RequestOptions.DEFAULT);
+        assertSpec(response.hits().events());
+    }
+
     private static long[] extractIds(List<SearchHit> events) {
         final int len = events.size();
         final long ids[] = new long[len];
@@ -123,10 +175,15 @@ private void assertSpec(List<SearchHit> events) {
         assertArrayEquals("unexpected result for spec: [" + spec.toString() + "]", spec.expectedEventIds(), extractIds(events));
     }
 
-    public final void test() {
-        EqlSearchResponse response = new EqlSearchRequestBuilder(client(), EqlSearchAction.INSTANCE)
-                .indices(testIndexName).query(spec.query()).get();
-
-        assertSpec(response.hits().events());
+    private RestHighLevelClient highLevelClient() {
+        if (highLevelClient == null) {
+            highLevelClient = new RestHighLevelClient(
+                    client(),
+                    ignore -> {
+                    },
+                    List.of()) {
+            };
+        }
+        return highLevelClient;
     }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpec.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
similarity index 97%
rename from x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpec.java
rename to x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
index 1db8269656f6e..00ebde23eed39 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpec.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.eql.action;
+package org.elasticsearch.test.eql;
 
 import org.elasticsearch.common.Strings;
 
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpecLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
similarity index 98%
rename from x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpecLoader.java
rename to x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
index f5fe4f93c315d..932874bc0916f 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpecLoader.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.eql.action;
+package org.elasticsearch.test.eql;
 
 import io.ous.jtoml.JToml;
 import io.ous.jtoml.Toml;
diff --git a/x-pack/plugin/eql/src/test/resources/test_data.json b/x-pack/plugin/eql/qa/common/src/main/resources/test_data.json
similarity index 100%
rename from x-pack/plugin/eql/src/test/resources/test_data.json
rename to x-pack/plugin/eql/qa/common/src/main/resources/test_data.json
diff --git a/x-pack/plugin/eql/src/test/resources/test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml
similarity index 100%
rename from x-pack/plugin/eql/src/test/resources/test_queries.toml
rename to x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml
diff --git a/x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
similarity index 100%
rename from x-pack/plugin/eql/src/test/resources/test_queries_unsupported.toml
rename to x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
diff --git a/x-pack/plugin/eql/qa/rest/src/test/java/org/elasticsearch/xpack/eql/EqlActionIT.java b/x-pack/plugin/eql/qa/rest/src/test/java/org/elasticsearch/xpack/eql/EqlActionIT.java
new file mode 100644
index 0000000000000..c69cd9567902d
--- /dev/null
+++ b/x-pack/plugin/eql/qa/rest/src/test/java/org/elasticsearch/xpack/eql/EqlActionIT.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.test.eql.CommonEqlActionTestCase;
+import org.elasticsearch.test.eql.EqlSpec;
+
+public class EqlActionIT extends CommonEqlActionTestCase {
+    // Constructor for parameterized test
+    public EqlActionIT(int num, String name, EqlSpec spec) {
+        super(num, name, spec);
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java
deleted file mode 100644
index 386c0e337b310..0000000000000
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.eql.action;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.LicenseService;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
-
-import java.util.Collection;
-
-import static java.util.Collections.singletonList;
-import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
-
-@ESIntegTestCase.ClusterScope(scope = SUITE, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0)
-public abstract class AbstractEqlIntegTestCase extends ESIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
-        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
-        settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
-        settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
-        settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
-        settings.put(EqlPlugin.EQL_ENABLED_SETTING.getKey(), true);
-        settings.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
-        return settings.build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return singletonList(LocalStateEqlXPackPlugin.class);
-    }
-}
-
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEqlXPackPlugin.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEqlXPackPlugin.java
deleted file mode 100644
index 7a6cd355a6ffa..0000000000000
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEqlXPackPlugin.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.eql.action;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
-import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
-
-import java.nio.file.Path;
-
-public class LocalStateEqlXPackPlugin extends LocalStateCompositeXPackPlugin {
-
-    public LocalStateEqlXPackPlugin(final Settings settings, final Path configPath) throws Exception {
-        super(settings, configPath);
-        LocalStateEqlXPackPlugin thisVar = this;
-        plugins.add(new EqlPlugin(settings) {
-            @Override
-            protected XPackLicenseState getLicenseState() {
-                return thisVar.getLicenseState();
-            }
-        });
-    }
-}

From d864ee2ad9587f142c9ed3c7e8afeae615248dba Mon Sep 17 00:00:00 2001
From: Luca Belluccini <luca.belluccini@elastic.co>
Date: Fri, 20 Mar 2020 19:02:08 +0100
Subject: [PATCH 180/285] [DOCS] Clarify upgrade paths (#53417)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Unsupported upgrade paths:
- `6.8 to 7.0`.
Supported, but requires a full cluster restart:
- `6.0–6.7 directly to 7.x`
---
 docs/reference/upgrade.asciidoc | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/docs/reference/upgrade.asciidoc b/docs/reference/upgrade.asciidoc
index 72fffc50ac97a..56b4f1ff43de3 100644
--- a/docs/reference/upgrade.asciidoc
+++ b/docs/reference/upgrade.asciidoc
@@ -43,10 +43,18 @@ a|
 
 [WARNING]
 ====
-The following upgrade paths are *not* supported:
+The upgrade path from 6.8 to 7.0 is *not* supported (both full cluster restart and rolling upgrade).
+====
 
-* 6.8 to 7.0.
-* 6.7 to 7.1.–{prev-major-version}.
+To upgrade directly to {version} from 6.7 or earlier, you must shut down the
+cluster, install {version}, and restart. For more information, see
+<<restart-upgrade, Full cluster restart upgrade>>.
+
+[WARNING]
+====
+In-place downgrades to earlier versions are *not* supported. To downgrade to an
+earlier version, <<snapshots-restore-snapshot,restore a snapshot>> taken prior
+to the version upgrade.
 ====
 
 {es} can read indices created in the previous major version. If you
@@ -60,10 +68,6 @@ When upgrading to a new version of {es}, you need to upgrade each
 of the products in your Elastic Stack. For more information, see the
 {stack-ref}/upgrading-elastic-stack.html[Elastic Stack Installation and Upgrade Guide].
 
-To upgrade directly to {version} from 6.6 or earlier, you must shut down the
-cluster, install {version}, and restart. For more information, see
-<<restart-upgrade, Full cluster restart upgrade>>.
-
 --
 
 include::upgrade/rolling_upgrade.asciidoc[]

From 3cf3f60f93cd62fb7153abc56c6932354e1bbe12 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Fri, 20 Mar 2020 11:18:51 -0700
Subject: [PATCH 181/285] Mute submit-async-search-date-histogram-example test

---
 docs/reference/search/async-search.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
index 1ec22e98d65d1..1b94e4000f38f 100644
--- a/docs/reference/search/async-search.asciidoc
+++ b/docs/reference/search/async-search.asciidoc
@@ -30,6 +30,7 @@ POST /sales*/_async_search?size=0
     }
 }
 --------------------------------------------------
+// TEST[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/53887"]
 // TEST[setup:sales]
 // TEST[s/size=0/size=0&wait_for_completion=0/]
 

From f3243949d7319042c801473681d906e5ebc41ecc Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 20 Mar 2020 19:01:05 +0000
Subject: [PATCH 182/285] Describe STALE_STATE_CONFIG in ClusterFormationFH
 (#53878)

We mark cluster states persisted on master-ineligible nodes as
potentially-stale using the voting configuration `{STALE_STATE_CONFIG}` which
prevents these nodes from being elected as master if they are restarted as
master-eligible. Today we do not handle this special voting configuration
differently in the `ClusterFormationFailureHandler`, leading to a mysterious
message `an election requires a node with id [STALE_STATE_CONFIG]` if the
election does not succeed.

This commit adds a special case description for this situation to explain
better why this node cannot win an election.

Closes #53734
---
 .../coordination/ClusterFormationFailureHelper.java    |  8 +++++++-
 .../org/elasticsearch/gateway/GatewayMetaState.java    |  9 ++++++++-
 .../ClusterFormationFailureHelperTests.java            | 10 ++++++++++
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
index c249458dd23cf..e68af52eaed38 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.gateway.GatewayMetaState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
 
@@ -206,7 +207,12 @@ private String describeQuorum(VotingConfiguration votingConfiguration) {
             assert requiredNodes <= realNodeIds.size() : nodeIds;
 
             if (nodeIds.size() == 1) {
-                return "a node with id " + realNodeIds;
+                if (nodeIds.contains(GatewayMetaState.STALE_STATE_CONFIG_NODE_ID)) {
+                    return "one or more nodes that have already participated as master-eligible nodes in the cluster but this node was " +
+                        "not master-eligible the last time it joined the cluster";
+                } else {
+                    return "a node with id " + realNodeIds;
+                }
             } else if (nodeIds.size() == 2) {
                 return "two nodes with ids " + realNodeIds;
             } else {
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index 89e0ff5c9fb66..62b3eebccab38 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -78,6 +78,13 @@
  */
 public class GatewayMetaState implements Closeable {
 
+    /**
+     * Fake node ID for a voting configuration written by a master-ineligible data node to indicate that its on-disk state is potentially
+     * stale (since it is written asynchronously after application, rather than before acceptance). This node ID means that if the node is
+     * restarted as a master-eligible node then it does not win any elections until it has received a fresh cluster state.
+     */
+    public static final String STALE_STATE_CONFIG_NODE_ID = "STALE_STATE_CONFIG";
+
     // Set by calling start()
     private final SetOnce<PersistedState> persistedState = new SetOnce<>();
 
@@ -360,7 +367,7 @@ protected void doRun() {
         }
 
         static final CoordinationMetaData.VotingConfiguration staleStateConfiguration =
-            new CoordinationMetaData.VotingConfiguration(Collections.singleton("STALE_STATE_CONFIG"));
+            new CoordinationMetaData.VotingConfiguration(Collections.singleton(STALE_STATE_CONFIG_NODE_ID));
 
         static ClusterState resetVotingConfiguration(ClusterState clusterState) {
             CoordinationMetaData newCoordinationMetaData = CoordinationMetaData.builder(clusterState.coordinationMetaData())
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
index 850a1ec0b7abd..756cf6668b5ba 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.gateway.GatewayMetaState;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -395,5 +396,14 @@ public void testDescriptionAfterBootstrapping() {
                     "have discovered [] which is not a quorum; " +
                     "discovery will continue using [] from hosts providers and [" + otherMasterNode + ", " + localNode +
                     "] from last-known cluster state; node term 0, last-accepted version 0 in term 0")));
+
+        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, GatewayMetaState.STALE_STATE_CONFIG_NODE_ID), emptyList(),
+                emptyList(), 0L, electionStrategy).getDescription(),
+            is("master not discovered or elected yet, an election requires one or more nodes that have already participated as " +
+                "master-eligible nodes in the cluster but this node was not master-eligible the last time it joined the cluster, " +
+                "have discovered [] which is not a quorum; " +
+                "discovery will continue using [] from hosts providers and [" + localNode +
+                "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
+
     }
 }

From f8d4b0c1d5dab78921ef0b219b815489d82e9d3d Mon Sep 17 00:00:00 2001
From: Stuart Tettemer <stu@elastic.co>
Date: Fri, 20 Mar 2020 13:29:34 -0600
Subject: [PATCH 183/285] Scripting: Context script cache unlimited compile
 (#53769)

* Adds "unlimited" compilation rate for context script caches
* `script.context.${CONTEXT}.max_compilations_rate` = `unlimited`
  disables compilation rate limiting for `${CONTEXT}`'s script
  cache

Refs: #50152
---
 .../org/elasticsearch/script/ScriptCache.java | 11 ++++----
 .../elasticsearch/script/ScriptService.java   |  8 +++++-
 .../script/ScriptCacheTests.java              | 13 ++++++++++
 .../script/ScriptServiceTests.java            | 25 +++++++++++++++++--
 4 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java
index a37851cc81050..7ab5f6d131718 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java
@@ -40,14 +40,16 @@ public class ScriptCache {
 
     private static final Logger logger = LogManager.getLogger(ScriptService.class);
 
+    static final Tuple<Integer, TimeValue> UNLIMITED_COMPILATION_RATE = new Tuple<>(0, TimeValue.ZERO);
+
     private final Cache<CacheKey, Object> cache;
     private final ScriptMetrics scriptMetrics;
 
     private final Object lock = new Object();
 
-    // Mutable fields
-    private long lastInlineCompileTime;
-    private double scriptsPerTimeWindow;
+    // Mutable fields, visible for tests
+    long lastInlineCompileTime;
+    double scriptsPerTimeWindow;
 
     // Cache settings or derived from settings
     final int cacheSize;
@@ -150,8 +152,7 @@ public ScriptStats stats() {
      * is discarded - there can never be more water in the bucket than the size of the bucket.
      */
     void checkCompilationLimit() {
-        if (rate.v1() == 0 && rate.v2().getNanos() == 0) {
-            // unlimited
+        if (rate.equals(UNLIMITED_COMPILATION_RATE)) {
             return;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java
index 66ed3fee0d6ca..5f8edfda3a278 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -128,10 +128,16 @@ public class ScriptService implements Closeable, ClusterStateApplier {
             key -> Setting.positiveTimeSetting(key, SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, TimeValue.timeValueMillis(0),
                                                Property.NodeScope, Property.Dynamic));
 
+    // Unlimited compilation rate for context-specific script caches
+    static final String UNLIMITED_COMPILATION_RATE_KEY = "unlimited";
+
     public static final Setting.AffixSetting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE_SETTING =
         Setting.affixKeySetting(CONTEXT_PREFIX,
             "max_compilations_rate",
-            key -> new Setting<>(key, "75/5m", MAX_COMPILATION_RATE_FUNCTION, Property.NodeScope, Property.Dynamic));
+            key -> new Setting<>(key, "75/5m",
+                (String value) -> value.equals(UNLIMITED_COMPILATION_RATE_KEY) ? ScriptCache.UNLIMITED_COMPILATION_RATE:
+                                                                                 MAX_COMPILATION_RATE_FUNCTION.apply(value),
+                Property.NodeScope, Property.Dynamic));
 
     private static final Tuple<Integer, TimeValue> SCRIPT_COMPILATION_RATE_ZERO = new Tuple<>(0, TimeValue.ZERO);
 
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
index dfa946099306f..96c724b5720c6 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
@@ -52,4 +52,17 @@ public void testCompilationCircuitBreaking() throws Exception {
             cache.checkCompilationLimit();
         }
     }
+
+    public void testUnlimitedCompilationRate() {
+        final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY);
+        final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY);
+        ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE);
+        long lastInlineCompileTime = cache.lastInlineCompileTime;
+        double scriptsPerTimeWindow = cache.scriptsPerTimeWindow;
+        for(int i=0; i < 3000; i++) {
+            cache.checkCompilationLimit();
+            assertEquals(lastInlineCompileTime, cache.lastInlineCompileTime);
+            assertEquals(scriptsPerTimeWindow, cache.scriptsPerTimeWindow, 0.0); // delta of 0.0 because it should never change
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 77ce3fe127c7f..4130c723dbe33 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -450,6 +450,24 @@ public void testCacheHolderContextConstructor() {
         assertEquals(zero, holder.contextCache.get("baz").get().rate);
     }
 
+    public void testCompilationRateUnlimitedContextOnly() throws IOException {
+        IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> {
+            buildScriptService(Settings.builder()
+                .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
+                .build());
+        });
+        assertEquals("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [unlimited]", illegal.getMessage());
+
+        // Should not throw.
+        buildScriptService(Settings.builder()
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("ingest").getKey(),
+                 ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("field").getKey(),
+                 ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY)
+            .build());
+    }
+
     public void testCacheHolderChangeSettings() {
         String fooCompilationRate = "77/5m";
         String barCompilationRate = "78/6m";
@@ -459,7 +477,7 @@ public void testCacheHolderChangeSettings() {
         Settings s = Settings.builder()
             .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate)
             .build();
-        Set<String> contexts = Set.of("foo", "bar", "baz");
+        Set<String> contexts = Set.of("foo", "bar", "baz", "qux");
         ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, true);
 
         assertNotNull(holder.general);
@@ -470,16 +488,19 @@ public void testCacheHolderChangeSettings() {
             .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY)
             .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), fooCompilationRate)
             .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), barCompilationRate)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("qux").getKey(),
+                 ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
             .build()
         );
 
         assertNull(holder.general);
         assertNotNull(holder.contextCache);
-        assertEquals(3, holder.contextCache.size());
+        assertEquals(4, holder.contextCache.size());
         assertEquals(contexts, holder.contextCache.keySet());
 
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate);
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate);
+        assertEquals(ScriptCache.UNLIMITED_COMPILATION_RATE, holder.contextCache.get("qux").get().rate);
         assertEquals(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getDefault(Settings.EMPTY),
             holder.contextCache.get("baz").get().rate);
 

From 6332c40a2cfd7d9acf9a982b75f544f83f629cf6 Mon Sep 17 00:00:00 2001
From: Gordon Brown <gordon.brown@elastic.co>
Date: Fri, 20 Mar 2020 13:59:13 -0600
Subject: [PATCH 184/285] Transition Transforms to using hidden indices for
 notifcations index (#53773)

This commit changes the Transforms notifications index to be hidden
index, with a hidden alias.

This commit also removes the temporary hack in
MetaDataCreateIndexService that prevents deprecation warnings for known
dot-prefixed index names which are not hidden/system indices, as this
was the last index pattern to need that hack.
---
 .../metadata/MetaDataCreateIndexService.java  | 17 +--------
 .../MetaDataCreateIndexServiceTests.java      | 35 -------------------
 .../integration/TransformAuditorIT.java       | 13 ++++++-
 .../TransformClusterStateListener.java        |  5 ++-
 .../persistence/TransformInternalIndex.java   |  3 +-
 5 files changed, 19 insertions(+), 54 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 0d5910ab10093..229c050f00cc8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -23,7 +23,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.Version;
@@ -55,7 +54,6 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -88,7 +86,6 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -114,14 +111,6 @@ public class MetaDataCreateIndexService {
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
-    /**
-     * These index patterns will be converted to hidden indices, at which point they should be removed from this list.
-     */
-    private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
-        ".data-frame-notifications-*",
-        ".transform-notifications-*"
-    ));
-
     private final Settings settings;
     private final ClusterService clusterService;
     private final IndicesService indicesService;
@@ -192,11 +181,7 @@ public void validateDotIndex(String index, ClusterState state, @Nullable Boolean
             List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
                 .filter(descriptor -> descriptor.matchesIndexPattern(index))
                 .collect(toList());
-            if (DOT_INDICES_EXCLUSIONS.run(index)) {
-                assert Objects.isNull(isHidden) || Boolean.FALSE.equals(isHidden) : "when converting a special-cased index to be a " +
-                    "hidden index, it must be removed from the exclusions list";
-                logger.debug("not emitting deprecation warning about index [{}] because it is in the exclusions list", index);
-            } else if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
+            if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
                 deprecationLogger.deprecated("index name [{}] starts with a dot '.', in the next major version, index names " +
                     "starting with a dot are reserved for hidden indices and system indices", index);
             } else if (matchingDescriptors.size() > 1) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 1ad07109b8d07..f58d061fb0113 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -614,41 +614,6 @@ public void testValidateDotIndex() {
         }
     }
 
-    public void testIndexNameExclusionsList() {
-        // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
-        List<String> excludedNames = Arrays.asList(
-            ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
-            ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT)
-        );
-
-        ThreadPool testThreadPool = new TestThreadPool(getTestName());
-        try {
-            MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService(
-                Settings.EMPTY,
-                ClusterServiceUtils.createClusterService(testThreadPool),
-                null,
-                null,
-                null,
-                null,
-                null,
-                testThreadPool,
-                null,
-                Collections.emptyList(),
-                false
-            );
-
-            excludedNames.forEach(name -> {
-                checkerService.validateDotIndex(name, ClusterState.EMPTY_STATE, false);
-            });
-
-            excludedNames.forEach(name -> {
-                expectThrows(AssertionError.class, () -> checkerService.validateDotIndex(name, ClusterState.EMPTY_STATE, true));
-            });
-        } finally {
-            testThreadPool.shutdown();
-        }
-    }
-
     public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exception {
         IndexTemplateMetaData templateMetaData = addMatchingTemplate(templateBuilder -> {
             templateBuilder.putAlias(AliasMetaData.builder("alias1"));
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java
index 43121a1478035..df54fa6e51caf 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java
@@ -7,7 +7,9 @@
 package org.elasticsearch.xpack.transform.integration;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.junit.Before;
@@ -94,7 +96,16 @@ public void testAliasCreatedforBWCIndexes() throws Exception {
                 .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
                 .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
 
-        createIndex(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED, settings.build());
+        // These indices should only exist if created in previous versions, ignore the deprecation warning for this test
+        RequestOptions options = expectWarnings("index name [" + TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED + "] starts " +
+            "with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices " +
+            "and system indices");
+        Request request = new Request("PUT", "/" + TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED);
+        String entity = "{\"settings\": " + Strings.toString(settings.build()) + "}";
+        request.setJsonEntity(entity);
+        request.setOptions(options);
+        client().performRequest(request);
+
         assertBusy(() -> {
             assertTrue(aliasExists(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED,
                     TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS));
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java
index 8df8cf8d2644e..30ad79b974381 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java
@@ -78,7 +78,10 @@ private static void createAuditAliasForDataFrameBWC(ClusterState state, Client c
         }
 
         final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
-                .addAlias(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS)
+                .addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                    .index(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED)
+                    .alias(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS)
+                    .isHidden(true))
                 .request();
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), TRANSFORM_ORIGIN, request,
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
index adf685f3fe358..9874dc1f2586a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
@@ -101,9 +101,10 @@ public static IndexTemplateMetaData getAuditIndexTemplateMetaData() throws IOExc
                     // the audits are expected to be small
                     .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                     .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                    .put(IndexMetaData.SETTING_INDEX_HIDDEN, true)
             )
             .putMapping(MapperService.SINGLE_MAPPING_NAME, Strings.toString(auditMappings()))
-            .putAlias(AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS))
+            .putAlias(AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS).isHidden(true))
             .build();
         return transformTemplate;
     }

From 290d58b1b05d17a5628f87d20ec583004bac5f3e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Fri, 20 Mar 2020 13:26:22 -0700
Subject: [PATCH 185/285] Remove unnecessary fromValue method for
 ClusterHealthStatus (#53893)

We use readFrom as a standard name for deserializing an object, but
ClusterHealthStatus also had a `fromValue` method which was used during
outer deserialization. This commit collapses the two methods together.

closes #53569
---
 .../action/admin/cluster/health/ClusterHealthRequest.java    | 2 +-
 .../action/admin/cluster/health/ClusterHealthResponse.java   | 2 +-
 .../action/admin/cluster/stats/ClusterStatsNodeResponse.java | 2 +-
 .../admin/indices/shards/IndicesShardStoresRequest.java      | 2 +-
 .../elasticsearch/cluster/health/ClusterHealthStatus.java    | 5 +----
 .../org/elasticsearch/cluster/health/ClusterIndexHealth.java | 2 +-
 .../org/elasticsearch/cluster/health/ClusterShardHealth.java | 2 +-
 .../org/elasticsearch/cluster/health/ClusterStateHealth.java | 2 +-
 8 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
index bbe24d7c0443f..ef6538f210d45 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
@@ -73,7 +73,7 @@ public ClusterHealthRequest(StreamInput in) throws IOException {
         }
         timeout = in.readTimeValue();
         if (in.readBoolean()) {
-            waitForStatus = ClusterHealthStatus.fromValue(in.readByte());
+            waitForStatus = ClusterHealthStatus.readFrom(in);
         }
         waitForNoRelocatingShards = in.readBoolean();
         waitForActiveShards = ActiveShardCount.readFrom(in);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
index 163c66003f4d7..2bcae763201fa 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
@@ -147,7 +147,7 @@ public ClusterHealthResponse() {}
     public ClusterHealthResponse(StreamInput in) throws IOException {
         super(in);
         clusterName = in.readString();
-        clusterHealthStatus = ClusterHealthStatus.fromValue(in.readByte());
+        clusterHealthStatus = ClusterHealthStatus.readFrom(in);
         clusterStateHealth = new ClusterStateHealth(in);
         numberOfPendingTasks = in.readInt();
         timedOut = in.readBoolean();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
index b32ed7e39d239..d315fc4344a9e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
@@ -42,7 +42,7 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException {
         super(in);
         clusterStatus = null;
         if (in.readBoolean()) {
-            clusterStatus = ClusterHealthStatus.fromValue(in.readByte());
+            clusterStatus = ClusterHealthStatus.readFrom(in);
         }
         this.nodeInfo = new NodeInfo(in);
         this.nodeStats = new NodeStats(in);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java
index 45a9c7283d0ab..19f707aa13367 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java
@@ -55,7 +55,7 @@ public IndicesShardStoresRequest(StreamInput in) throws IOException {
         int nStatus = in.readVInt();
         statuses = EnumSet.noneOf(ClusterHealthStatus.class);
         for (int i = 0; i < nStatus; i++) {
-            statuses.add(ClusterHealthStatus.fromValue(in.readByte()));
+            statuses.add(ClusterHealthStatus.readFrom(in));
         }
         indicesOptions = IndicesOptions.readIndicesOptions(in);
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
index 8a255fb1ce816..c0738ee82226a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
@@ -52,10 +52,7 @@ public void writeTo(StreamOutput out) throws IOException {
      * @throws IllegalArgumentException if the value is unrecognized
      */
     public static ClusterHealthStatus readFrom(StreamInput in) throws IOException {
-        return fromValue(in.readByte());
-    }
-
-    public static ClusterHealthStatus fromValue(byte value) throws IOException {
+        byte value = in.readByte();
         switch (value) {
             case 0:
                 return GREEN;
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
index c1a52f2ffc548..1ee111b1f0e1a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
@@ -165,7 +165,7 @@ public ClusterIndexHealth(final StreamInput in) throws IOException {
         relocatingShards = in.readVInt();
         initializingShards = in.readVInt();
         unassignedShards = in.readVInt();
-        status = ClusterHealthStatus.fromValue(in.readByte());
+        status = ClusterHealthStatus.readFrom(in);
 
         int size = in.readVInt();
         shards = new HashMap<>(size);
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
index 1d3a3dcee7b95..a96aedb023a4e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
@@ -121,7 +121,7 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR
 
     public ClusterShardHealth(final StreamInput in) throws IOException {
         shardId = in.readVInt();
-        status = ClusterHealthStatus.fromValue(in.readByte());
+        status = ClusterHealthStatus.readFrom(in);
         activeShards = in.readVInt();
         relocatingShards = in.readVInt();
         initializingShards = in.readVInt();
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
index ad1561e4f0eda..dbaefb359e073 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
@@ -134,7 +134,7 @@ public ClusterStateHealth(final StreamInput in) throws IOException {
         unassignedShards = in.readVInt();
         numberOfNodes = in.readVInt();
         numberOfDataNodes = in.readVInt();
-        status = ClusterHealthStatus.fromValue(in.readByte());
+        status = ClusterHealthStatus.readFrom(in);
         int size = in.readVInt();
         indices = new HashMap<>(size);
         for (int i = 0; i < size; i++) {

From 984670b34d34fa0608fc00da6d941d3aaa41f5a7 Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Fri, 20 Mar 2020 15:11:49 -0600
Subject: [PATCH 186/285] =?UTF-8?q?Set=20feature=20flags=20for=20IndexTemp?=
 =?UTF-8?q?latesV2=20and=20Data=20Streams=20in=20top=E2=80=A6=20(#53897)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Resolves #53892
---
 build.gradle                                         | 12 ++++++++++++
 distribution/archives/integ-test-zip/build.gradle    |  8 --------
 qa/smoke-test-multinode/build.gradle                 | 10 ----------
 .../java/org/elasticsearch/action/ActionModule.java  |  6 ------
 x-pack/qa/core-rest-tests-with-security/build.gradle |  9 ---------
 5 files changed, 12 insertions(+), 33 deletions(-)

diff --git a/build.gradle b/build.gradle
index 932afc36f001f..17cd5f6b93cd0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -545,3 +545,15 @@ allprojects {
     }
   }
 }
+
+// TODO: remove this once 7.7 is released and the 7.x branch is 7.8
+subprojects {
+  pluginManager.withPlugin('elasticsearch.testclusters') {
+    testClusters.all {
+      if (org.elasticsearch.gradle.info.BuildParams.isSnapshotBuild() == false) {
+        systemProperty 'es.itv2_feature_flag_registered', 'true'
+        systemProperty 'es.datastreams_feature_flag_registered', 'true'
+      }
+    }
+  }
+}
diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle
index a0884b2555caf..ed8f893d7f840 100644
--- a/distribution/archives/integ-test-zip/build.gradle
+++ b/distribution/archives/integ-test-zip/build.gradle
@@ -1,4 +1,3 @@
-import org.elasticsearch.gradle.info.BuildParams
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -33,10 +32,3 @@ integTest.runner {
     systemProperty 'tests.logfile', '--external--'
   }
 }
-
-testClusters.integTest {
-  if (BuildParams.isSnapshotBuild() == false) {
-    systemProperty 'es.itv2_feature_flag_registered', 'true'
-    systemProperty 'es.datastreams_feature_flag_registered', 'true'
-  }
-}
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index 3663d63ea4b51..244b5d1e8af6e 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -16,9 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import org.elasticsearch.gradle.info.BuildParams
-
-import org.elasticsearch.gradle.info.BuildParams
 
 apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
@@ -49,10 +46,3 @@ integTest.runner {
     ].join(',')
   }
 }
-
-testClusters.integTest {
-  if (BuildParams.isSnapshotBuild() == false) {
-    systemProperty 'es.itv2_feature_flag_registered', 'true'
-    systemProperty 'es.datastreams_feature_flag_registered', 'true'
-  }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 894f0805564e3..09814a4059b71 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -381,9 +381,6 @@ public class ActionModule extends AbstractModule {
 
     static {
         final String property = System.getProperty("es.itv2_feature_flag_registered");
-        if (Build.CURRENT.isSnapshot() && property != null) {
-            throw new IllegalArgumentException("es.itv2_feature_flag_registered is only supported in non-snapshot builds");
-        }
         if (Build.CURRENT.isSnapshot() || "true".equals(property)) {
             ITV2_FEATURE_FLAG_REGISTERED = true;
         } else if ("false".equals(property) || property == null) {
@@ -398,9 +395,6 @@ public class ActionModule extends AbstractModule {
 
     static {
         final String property = System.getProperty("es.datastreams_feature_flag_registered");
-        if (Build.CURRENT.isSnapshot() && property != null) {
-            throw new IllegalArgumentException("es.datastreams_feature_flag_registered is only supported in non-snapshot builds");
-        }
         if (Build.CURRENT.isSnapshot() || "true".equals(property)) {
             DATASTREAMS_FEATURE_FLAG_REGISTERED = true;
         } else if ("false".equals(property) || property == null) {
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 651c6af0308a2..4fc4428c8ed6f 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.info.BuildParams
-
 apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
@@ -39,10 +37,3 @@ testClusters.integTest {
   user username: System.getProperty('tests.rest.cluster.username', 'test_user'),
     password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
 }
-
-testClusters.integTest {
-  if (BuildParams.isSnapshotBuild() == false) {
-    systemProperty 'es.itv2_feature_flag_registered', 'true'
-    systemProperty 'es.datastreams_feature_flag_registered', 'true'
-  }
-}

From 1b37d4bfc9616f39aa3a7d3d0dc63dc132e909a8 Mon Sep 17 00:00:00 2001
From: Stuart Tettemer <stu@elastic.co>
Date: Fri, 20 Mar 2020 15:15:19 -0600
Subject: [PATCH 187/285] Scripting: Increase ingest script cache defaults
 (#53765)

* Adds ability for contexts to specify their own defaults.
* Context defaults are applied if no context-specific or
  general setting exists.
* See 070ea7e for settings keys.

* Increases the per-context default for the `ingest` context.
  * Cache size is doubled, 200 compared to default of 100
  * Cache expiration is unchanged at no expiration
  * Cache max compilation is quintupled, 375/5m instead of 75/5m

Refs: #50152
---
 .../script/IngestConditionalScript.java       |   6 +-
 .../elasticsearch/script/IngestScript.java    |   6 +-
 .../elasticsearch/script/ScriptContext.java   |  28 +++-
 .../elasticsearch/script/ScriptService.java   |  54 +++++---
 .../script/ScriptServiceTests.java            | 120 ++++++++++++++++--
 5 files changed, 180 insertions(+), 34 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java b/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java
index 27ce29b95dc50..ed224204f8933 100644
--- a/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java
+++ b/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.unit.TimeValue;
+
 import java.util.Map;
 
 /**
@@ -29,7 +32,8 @@ public abstract class IngestConditionalScript {
     public static final String[] PARAMETERS = { "ctx" };
 
     /** The context used to compile {@link IngestConditionalScript} factories. */
-    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("processor_conditional", Factory.class);
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("processor_conditional", Factory.class,
+        200, TimeValue.timeValueMillis(0), new Tuple<>(375, TimeValue.timeValueMinutes(5)));
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;
diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java
index f357394ed31f0..ffd3475fc52c3 100644
--- a/server/src/main/java/org/elasticsearch/script/IngestScript.java
+++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java
@@ -20,6 +20,9 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.unit.TimeValue;
+
 import java.util.Map;
 
 /**
@@ -30,7 +33,8 @@ public abstract class IngestScript {
     public static final String[] PARAMETERS = { "ctx" };
 
     /** The context used to compile {@link IngestScript} factories. */
-    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class);
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class,
+        200, TimeValue.timeValueMillis(0), new Tuple<>(375, TimeValue.timeValueMinutes(5)));
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContext.java b/server/src/main/java/org/elasticsearch/script/ScriptContext.java
index 081a26d1e511a..abdcc99a1a3b1 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptContext.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptContext.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.unit.TimeValue;
+
 import java.lang.reflect.Method;
 
 /**
@@ -68,8 +71,18 @@ public final class ScriptContext<FactoryType> {
     /** A class that is an instance of a script. */
     public final Class<?> instanceClazz;
 
-    /** Construct a context with the related instance and compiled classes. */
-    public ScriptContext(String name, Class<FactoryType> factoryClazz) {
+    /** The default size of the cache for the context if not overridden */
+    public final int cacheSizeDefault;
+
+    /** The default expiration of a script in the cache for the context, if not overridden */
+    public final TimeValue cacheExpireDefault;
+
+    /** The default max compilation rate for scripts in this context.  Script compilation is throttled if this is exceeded */
+    public final Tuple<Integer, TimeValue> maxCompilationRateDefault;
+
+    /** Construct a context with the related instance and compiled classes with caller provided cache defaults */
+    public ScriptContext(String name, Class<FactoryType> factoryClazz, int cacheSizeDefault, TimeValue cacheExpireDefault,
+                        Tuple<Integer, TimeValue> maxCompilationRateDefault) {
         this.name = name;
         this.factoryClazz = factoryClazz;
         Method newInstanceMethod = findMethod("FactoryType", factoryClazz, "newInstance");
@@ -90,6 +103,17 @@ public ScriptContext(String name, Class<FactoryType> factoryClazz) {
                 + factoryClazz.getName() + "] for script context [" + name + "]");
         }
         instanceClazz = newInstanceMethod.getReturnType();
+
+        this.cacheSizeDefault = cacheSizeDefault;
+        this.cacheExpireDefault = cacheExpireDefault;
+        this.maxCompilationRateDefault = maxCompilationRateDefault;
+    }
+
+    /** Construct a context with the related instance and compiled classes with defaults for cacheSizeDefault, cacheExpireDefault and
+     *  maxCompilationRateDefault */
+    public ScriptContext(String name, Class<FactoryType> factoryClazz) {
+        // cache size default, cache expire default, max compilation rate are defaults from ScriptService.
+        this(name, factoryClazz, 100, TimeValue.timeValueMillis(0), new Tuple<>(75, TimeValue.timeValueMinutes(5)));
     }
 
     /** Returns a method with the given name, or throws an exception if multiple are found. */
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java
index 5f8edfda3a278..59422f7dd5034 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -46,6 +46,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -241,7 +242,7 @@ public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<S
 
         // Validation requires knowing which contexts exist.
         this.validateCacheSettings(settings);
-        cacheHolder = new AtomicReference<>(new CacheHolder(settings, contexts.keySet(), compilationLimitsEnabled()));
+        cacheHolder = new AtomicReference<>(new CacheHolder(settings, contexts.values(), compilationLimitsEnabled()));
     }
 
     /**
@@ -255,12 +256,12 @@ void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
         clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_SIZE_IN_BYTES, this::setMaxSizeInBytes);
 
         // Handle all updatable per-context settings at once for each context.
-        for (String context: contexts.keySet()) {
+        for (ScriptContext<?> context: contexts.values()) {
             clusterSettings.addSettingsUpdateConsumer(
                 (settings) -> cacheHolder.get().updateContextSettings(settings, context),
-                List.of(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context),
-                        SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context),
-                        SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context),
+                List.of(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name),
+                        SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context.name),
+                        SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name),
                         SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, // general settings used for fallbacks
                         SCRIPT_GENERAL_CACHE_SIZE_SETTING
                 )
@@ -580,17 +581,18 @@ static class CacheHolder {
         final ScriptCache general;
         final Map<String, AtomicReference<ScriptCache>> contextCache;
 
-        final Set<String> contexts;
+        final Set<ScriptContext<?>> contexts;
         final boolean compilationLimitsEnabled;
 
-        CacheHolder(Settings settings, Set<String> contexts, boolean compilationLimitsEnabled) {
+        CacheHolder(Settings settings, Collection<ScriptContext<?>> contexts, boolean compilationLimitsEnabled) {
             this.compilationLimitsEnabled = compilationLimitsEnabled;
             this.contexts = Set.copyOf(contexts);
             if (SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE)) {
                 this.general = null;
                 Map<String, AtomicReference<ScriptCache>> contextCache = new HashMap<>(this.contexts.size());
-                for (String context : this.contexts) {
-                    contextCache.put(context, new AtomicReference<>(contextFromSettings(settings, context, this.compilationLimitsEnabled)));
+                for (ScriptContext<?> context : this.contexts) {
+                    contextCache.put(context.name,
+                                     new AtomicReference<>(contextFromSettings(settings, context, this.compilationLimitsEnabled)));
                 }
                 this.contextCache = Collections.unmodifiableMap(contextCache);
             } else {
@@ -607,12 +609,24 @@ static class CacheHolder {
         /**
          * Create a ScriptCache for the given context.
          */
-        private static ScriptCache contextFromSettings(Settings settings, String context, boolean compilationLimitsEnabled) {
-            return new ScriptCache(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(settings),
-                SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(settings),
-                compilationLimitsEnabled ?
-                    SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).get(settings) :
-                    SCRIPT_COMPILATION_RATE_ZERO);
+        private static ScriptCache contextFromSettings(Settings settings, ScriptContext<?> context, boolean compilationLimitsEnabled) {
+            String name = context.name;
+            Tuple<Integer, TimeValue> compileRate;
+            Setting<Tuple<Integer, TimeValue>> rateSetting = SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name);
+            if (compilationLimitsEnabled == false) {
+                compileRate = SCRIPT_COMPILATION_RATE_ZERO;
+            } else if (rateSetting.existsOrFallbackExists(settings)) {
+                compileRate = rateSetting.get(settings);
+            } else {
+                compileRate = context.maxCompilationRateDefault;
+            }
+
+            Setting<TimeValue> cacheExpire = SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name);
+            Setting<Integer> cacheSize = SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name);
+
+            return new ScriptCache(cacheSize.existsOrFallbackExists(settings) ? cacheSize.get(settings) : context.cacheSizeDefault,
+                                   cacheExpire.existsOrFallbackExists(settings) ? cacheExpire.get(settings) : context.cacheExpireDefault,
+                                   compileRate);
         }
 
         /**
@@ -666,16 +680,16 @@ ScriptStats stats() {
         /**
          * Update settings for the context cache, if we're in the context cache mode otherwise no-op.
          */
-        void updateContextSettings(Settings settings, String context) {
+        void updateContextSettings(Settings settings, ScriptContext<?> context) {
             if (general != null) {
                 return;
             }
-            AtomicReference<ScriptCache> ref = contextCache.get(context);
-            assert ref != null : "expected script cache to exist for context [" + context + "]";
+            AtomicReference<ScriptCache> ref = contextCache.get(context.name);
+            assert ref != null : "expected script cache to exist for context [" + context.name + "]";
             ScriptCache cache = ref.get();
-            assert cache != null : "expected script cache to be non-null for context [" + context + "]";
+            assert cache != null : "expected script cache to be non-null for context [" + context.name + "]";
             ref.set(contextFromSettings(settings, context, compilationLimitsEnabled));
-            logger.debug("Replaced context [" + context + "] with new settings");
+            logger.debug("Replaced context [" + context.name + "] with new settings");
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 4130c723dbe33..0cc8b9c78fe55 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -36,16 +36,24 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.script.ScriptService.MAX_COMPILATION_RATE_FUNCTION;
+import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_EXPIRE_SETTING;
+import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_SIZE_SETTING;
+import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING;
+import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING;
 import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING;
 import static org.elasticsearch.script.ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING;
+import static org.elasticsearch.script.ScriptService.USE_CONTEXT_RATE_KEY;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -226,7 +234,7 @@ public void testMultipleCompilationsCountedInCompilationStats() throws IOExcepti
 
     public void testCompilationStatsOnCacheHit() throws IOException {
         Settings.Builder builder = Settings.builder();
-        builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
         buildScriptService(builder.build());
         Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
         ScriptContext<?> context = randomFrom(contexts.values());
@@ -243,7 +251,7 @@ public void testIndexedScriptCountedInCompilationStats() throws IOException {
 
     public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
         Settings.Builder builder = Settings.builder();
-        builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
         buildScriptService(builder.build());
         scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values()));
         scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(contexts.values()));
@@ -361,7 +369,7 @@ public void testFallbackContextSettings() {
         TimeValue cacheExpireFooParsed = TimeValue.parseTimeValue(cacheExpireFoo, "");
 
         Settings s = Settings.builder()
-            .put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), cacheSizeBackup)
+            .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), cacheSizeBackup)
             .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheSizeFoo)
             .put(ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), cacheExpireBackup)
             .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheExpireFoo)
@@ -396,7 +404,7 @@ public void testCacheHolderGeneralConstructor() {
         boolean compilationLimitsEnabled = true;
         ScriptService.CacheHolder holder = new ScriptService.CacheHolder(
             Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build(),
-            new HashSet<>(Collections.singleton("foo")),
+            Set.of(newContext("foo")),
             compilationLimitsEnabled
         );
 
@@ -407,7 +415,7 @@ public void testCacheHolderGeneralConstructor() {
         compilationLimitsEnabled = false;
         holder = new ScriptService.CacheHolder(
             Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build(),
-            new HashSet<>(Collections.singleton("foo")),
+            new HashSet<>(Collections.singleton(newContext("foo"))),
             compilationLimitsEnabled
         );
 
@@ -426,13 +434,13 @@ public void testCacheHolderContextConstructor() {
             .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), fooCompilationRate)
             .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), barCompilationRate)
             .build();
-        Set<String> contexts = Set.of("foo", "bar", "baz");
+        Collection<ScriptContext<?>> contexts = Set.of(newContext("foo"), newContext("bar"), newContext("baz"));
         ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, compilationLimitsEnabled);
 
         assertNull(holder.general);
         assertNotNull(holder.contextCache);
         assertEquals(3, holder.contextCache.size());
-        assertEquals(contexts, holder.contextCache.keySet());
+        assertEquals(contexts.stream().map(c -> c.name).collect(Collectors.toSet()), holder.contextCache.keySet());
 
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate);
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate);
@@ -477,7 +485,8 @@ public void testCacheHolderChangeSettings() {
         Settings s = Settings.builder()
             .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate)
             .build();
-        Set<String> contexts = Set.of("foo", "bar", "baz", "qux");
+        Collection<ScriptContext<?>> contexts = Set.of(newContext("foo"), newContext("bar"), newContext("baz"),
+                                                       newContext("qux"));
         ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, true);
 
         assertNotNull(holder.general);
@@ -496,7 +505,7 @@ public void testCacheHolderChangeSettings() {
         assertNull(holder.general);
         assertNotNull(holder.contextCache);
         assertEquals(4, holder.contextCache.size());
-        assertEquals(contexts, holder.contextCache.keySet());
+        assertEquals(contexts.stream().map(c -> c.name).collect(Collectors.toSet()), holder.contextCache.keySet());
 
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate);
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate);
@@ -506,7 +515,7 @@ public void testCacheHolderChangeSettings() {
 
         holder.updateContextSettings(Settings.builder()
                 .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), fooCompilationRate).build(),
-                "bar"
+                newContext("bar")
         );
         assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("bar").get().rate);
 
@@ -529,6 +538,93 @@ public void testCacheHolderChangeSettings() {
         assertSame(holder, update);
     }
 
+    public void testFallbackToContextDefaults() {
+        Tuple<Integer, TimeValue> contextDefaultRate = new Tuple<>(randomIntBetween(10, 1024),
+                                                                   TimeValue.timeValueMinutes(randomIntBetween(10, 200)));
+        String name = "foo";
+        ScriptContext<?> foo = new ScriptContext<>(name,
+                                                   ScriptContextTests.DummyScript.Factory.class,
+                                                   randomIntBetween(1, 1024),
+                                                   TimeValue.timeValueMinutes(randomIntBetween(10, 200)),
+                                                   contextDefaultRate);
+
+
+        int generalCacheSize = randomIntBetween(1, 1024);
+        TimeValue generalExpire = TimeValue.timeValueMinutes(randomIntBetween(10, 200));
+
+        String contextRateStr = randomIntBetween(10, 1024) + "/" +  randomIntBetween(10, 200) + "m";
+        Tuple<Integer, TimeValue>  contextRate = MAX_COMPILATION_RATE_FUNCTION.apply(contextRateStr);
+        int contextCacheSize = randomIntBetween(1, 1024);
+        TimeValue contextExpire = TimeValue.timeValueMinutes(randomIntBetween(10, 200));
+
+        // Use context specific
+        ScriptService.CacheHolder contextCache = new ScriptService.CacheHolder(
+            Settings.builder()
+            .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextCacheSize)
+            .put(SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextExpire)
+            .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextRateStr)
+            .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), generalCacheSize)
+            .put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), generalExpire)
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
+            .build(),
+            List.of(foo),
+            true);
+
+        assertNotNull(contextCache.contextCache);
+        assertNotNull(contextCache.contextCache.get(name));
+        assertNotNull(contextCache.contextCache.get(name).get());
+
+        assertEquals(contextRate, contextCache.contextCache.get(name).get().rate);
+        assertEquals(contextCacheSize, contextCache.contextCache.get(name).get().cacheSize);
+        assertEquals(contextExpire, contextCache.contextCache.get(name).get().cacheExpire);
+
+        // Fallback to general
+        contextCache = new ScriptService.CacheHolder(
+            Settings.builder()
+            .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), generalCacheSize)
+            .put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), generalExpire)
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
+            .build(),
+            List.of(foo),
+            true);
+
+        assertNotNull(contextCache.contextCache);
+        assertNotNull(contextCache.contextCache.get(name));
+        assertNotNull(contextCache.contextCache.get(name).get());
+
+        assertEquals(contextDefaultRate, contextCache.contextCache.get(name).get().rate);
+        assertEquals(generalCacheSize, contextCache.contextCache.get(name).get().cacheSize);
+        assertEquals(generalExpire, contextCache.contextCache.get(name).get().cacheExpire);
+
+        // Fallback to context defaults
+        contextCache = new ScriptService.CacheHolder(
+            Settings.builder()
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
+            .build(),
+            List.of(foo),
+            true);
+
+        assertNotNull(contextCache.contextCache);
+        assertNotNull(contextCache.contextCache.get(name));
+        assertNotNull(contextCache.contextCache.get(name).get());
+
+        assertEquals(contextDefaultRate, contextCache.contextCache.get(name).get().rate);
+        assertEquals(foo.cacheSizeDefault, contextCache.contextCache.get(name).get().cacheSize);
+        assertEquals(foo.cacheExpireDefault, contextCache.contextCache.get(name).get().cacheExpire);
+
+        // Use context specific for ingest
+        contextCache = new ScriptService.CacheHolder(
+            Settings.builder()
+                .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
+                .build(),
+            List.of(foo, IngestScript.CONTEXT, IngestConditionalScript.CONTEXT),
+            true);
+        assertEquals(new Tuple<>(375, TimeValue.timeValueMinutes(5)),
+                     contextCache.contextCache.get("ingest").get().rate);
+        assertEquals(200, contextCache.contextCache.get("ingest").get().cacheSize);
+        assertEquals(TimeValue.timeValueMillis(0), contextCache.contextCache.get("ingest").get().cacheExpire);
+    }
+
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
             scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext);
@@ -545,4 +641,8 @@ private void assertCompileAccepted(String lang, String script, ScriptType script
                 notNullValue()
         );
     }
+
+    ScriptContext<ScriptContextTests.DummyScript.Factory> newContext(String name) {
+        return new ScriptContext<>(name, ScriptContextTests.DummyScript.Factory.class);
+    }
 }

From dc7e2ab91a7ff4a4af3bcf7f3e5a5607ecf17aca Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Sat, 21 Mar 2020 17:34:20 +0100
Subject: [PATCH 188/285] Fix StackOverflowError in CheckStyle for
 CRUDDocumentationIT (#53840)

This file seems to randomly cause a stack overflow like other HLRC ITs that
already have this check suppressed on them.

Closes #37647
---
 buildSrc/src/main/resources/checkstyle_suppressions.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 7bf5087485733..d79d6a95e184e 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -31,6 +31,7 @@
   -->
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]ClusterClientDocumentationIT.java" id="SnippetLength" />
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]GraphDocumentationIT.java" id="SnippetLength" />
+  <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]CRUDDocumentationIT.java" id="SnippetLength" />
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]IndicesClientDocumentationIT.java" id="SnippetLength" />
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]IngestClientDocumentationIT.java" id="SnippetLength" />
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]LicensingDocumentationIT.java" id="SnippetLength" />

From 13f5631eec1175a533e8f68078d280b393afbfa5 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 21 Mar 2020 18:03:20 -0400
Subject: [PATCH 189/285] Introduce aarch64 packaging (#53914)

This commit introduces aarch64 packaging, including bundling an aarch64
JDK distribution. We had to make some interesting choices here:
 - ML binaries are not compiled for aarch64, so for now we disable ML on
   aarch64
 - depending on underlying page sizes, we have to disable class data
   sharing
---
 buildSrc/build.gradle                         |  2 +
 .../elasticsearch/gradle/Architecture.java    | 40 +++++++++++++++
 .../java/org/elasticsearch/gradle/Jdk.java    | 20 ++++++++
 .../gradle/JdkDownloadPlugin.java             | 35 +++++++++----
 .../gradle/test/DistroTestPlugin.java         |  8 +--
 .../gradle/JdkDownloadPluginTests.java        | 51 ++++++++++++++++---
 .../testKit/jdk-download/reuse/build.gradle   |  1 +
 .../testKit/jdk-download/subproj/build.gradle |  3 ++
 distribution/archives/build.gradle            | 42 +++++++++------
 .../archives/linux-aarch64-tar/build.gradle   |  2 +
 .../oss-linux-aarch64-tar/build.gradle        |  2 +
 distribution/build.gradle                     | 10 ++--
 .../packages/aarch64-deb/build.gradle         |  2 +
 .../packages/aarch64-oss-deb/build.gradle     |  2 +
 .../packages/aarch64-oss-rpm/build.gradle     |  2 +
 .../packages/aarch64-rpm/build.gradle         |  2 +
 distribution/packages/build.gradle            | 50 ++++++++++++------
 distribution/src/bin/elasticsearch            |  6 ++-
 distribution/src/bin/elasticsearch-cli        |  1 +
 distribution/src/bin/elasticsearch-env        |  8 ++-
 .../tools/launchers/JvmErgonomics.java        |  1 +
 gradle/runtime-jdk-provision.gradle           |  2 +
 settings.gradle                               |  6 +++
 .../xpack/core/XPackSettings.java             | 11 +++-
 .../sql/src/main/bin/elasticsearch-sql-cli    |  1 +
 25 files changed, 251 insertions(+), 59 deletions(-)
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java
 create mode 100644 distribution/archives/linux-aarch64-tar/build.gradle
 create mode 100644 distribution/archives/oss-linux-aarch64-tar/build.gradle
 create mode 100644 distribution/packages/aarch64-deb/build.gradle
 create mode 100644 distribution/packages/aarch64-oss-deb/build.gradle
 create mode 100644 distribution/packages/aarch64-oss-rpm/build.gradle
 create mode 100644 distribution/packages/aarch64-rpm/build.gradle

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 7ffe1490eed71..9b82bf3c5ce6a 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -187,8 +187,10 @@ if (project != rootProject) {
     distribution project(':distribution:archives:oss-windows-zip')
     distribution project(':distribution:archives:darwin-tar')
     distribution project(':distribution:archives:oss-darwin-tar')
+    distribution project(':distribution:archives:linux-aarch64-tar')
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
+    distribution project(':distribution:archives:oss-linux-aarch64-tar')
   }
 
   // for external projects we want to remove the marker file indicating we are running the Elasticsearch project
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java
new file mode 100644
index 0000000000000..f230d9af86e11
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+public enum Architecture {
+
+    X64,
+    AARCH64;
+
+    public static Architecture current() {
+        final String architecture = System.getProperty("os.arch", "");
+        switch (architecture) {
+            case "amd64":
+            case "x86_64":
+                return X64;
+            case "aarch64":
+                return AARCH64;
+            default:
+                throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
+        }
+    }
+
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java
index 42df062ba3bb0..e4c06560ab2b6 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java
@@ -33,6 +33,7 @@
 
 public class Jdk implements Buildable, Iterable<File> {
 
+    private static final List<String> ALLOWED_ARCHITECTURES = List.of("aarch64", "x64");
     private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
     private static final List<String> ALLOWED_PLATFORMS = List.of("darwin", "linux", "windows", "mac");
     private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
@@ -44,6 +45,7 @@ public class Jdk implements Buildable, Iterable<File> {
     private final Property<String> vendor;
     private final Property<String> version;
     private final Property<String> platform;
+    private final Property<String> architecture;
     private String baseVersion;
     private String major;
     private String build;
@@ -55,6 +57,7 @@ public class Jdk implements Buildable, Iterable<File> {
         this.vendor = objectFactory.property(String.class);
         this.version = objectFactory.property(String.class);
         this.platform = objectFactory.property(String.class);
+        this.architecture = objectFactory.property(String.class);
     }
 
     public String getName() {
@@ -97,6 +100,19 @@ public void setPlatform(String platform) {
         this.platform.set(platform);
     }
 
+    public String getArchitecture() {
+        return architecture.get();
+    }
+
+    public void setArchitecture(final String architecture) {
+        if (ALLOWED_ARCHITECTURES.contains(architecture) == false) {
+            throw new IllegalArgumentException(
+                "unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
+            );
+        }
+        this.architecture.set(architecture);
+    }
+
     public String getBaseVersion() {
         return baseVersion;
     }
@@ -153,9 +169,13 @@ void finalizeValues() {
         if (vendor.isPresent() == false) {
             throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
         }
+        if (architecture.isPresent() == false) {
+            throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
+        }
         version.finalizeValue();
         platform.finalizeValue();
         vendor.finalizeValue();
+        architecture.finalizeValue();
     }
 
     @Override
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index b63d8f65ea04f..daaf1a0fa54e9 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -47,7 +47,6 @@
 import java.util.concurrent.Callable;
 import java.util.stream.StreamSupport;
 
-import static org.elasticsearch.gradle.Util.capitalize;
 import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
 
@@ -72,7 +71,10 @@ public void apply(Project project) {
                 DependencyHandler dependencies = project.getDependencies();
                 Map<String, Object> depConfig = new HashMap<>();
                 depConfig.put("path", ":"); // root project
-                depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
+                depConfig.put(
+                    "configuration",
+                    configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture())
+                );
                 project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
 
                 // ensure a root level jdk download task exists
@@ -87,7 +89,14 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
     }
 
     private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
-        String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
+        String extractTaskName = String.format(
+            Locale.ROOT,
+            "extract-%s-%s-jdk-%s-%s",
+            jdk.getPlatform(),
+            jdk.getArchitecture(),
+            jdk.getVendor(),
+            jdk.getVersion()
+        );
 
         // Skip setup if we've already configured a JDK for this platform, vendor and version
         if (findByName(rootProject.getTasks(), extractTaskName) == null) {
@@ -107,7 +116,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
                 repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
                 artifactPattern = String.format(
                     Locale.ROOT,
-                    "adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
+                    "adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]",
                     jdk.getMajor(),
                     jdk.getBuild()
                 );
@@ -121,14 +130,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
                         + jdk.getHash()
                         + "/"
                         + jdk.getBuild()
-                        + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
                 } else {
                     // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
                     artifactPattern = "java/GA/jdk"
                         + jdk.getMajor()
                         + "/"
                         + jdk.getBuild()
-                        + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
                 }
             } else {
                 throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
@@ -150,14 +159,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
 
             // Declare a configuration and dependency from which to download the remote JDK
             final ConfigurationContainer configurations = rootProject.getConfigurations();
-            String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
+            String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture());
             Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
             rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
 
             // Create JDK extract task
             final Provider<Directory> extractPath = rootProject.getLayout()
                 .getBuildDirectory()
-                .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
+                .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture());
 
             TaskProvider<?> extractTask = createExtractTask(
                 extractTaskName,
@@ -168,7 +177,13 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
             );
 
             // Declare a configuration for the extracted JDK archive
-            String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
+            String artifactConfigName = configName(
+                "extracted_jdk",
+                jdk.getVendor(),
+                jdk.getVersion(),
+                jdk.getPlatform(),
+                jdk.getArchitecture()
+            );
             maybeCreate(configurations, artifactConfigName);
             rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
         }
@@ -254,7 +269,7 @@ private static String dependencyNotation(Jdk jdk) {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
     }
 
     private static String groupName(Jdk jdk) {
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
index 8b62e81e626e8..b09441148558f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
@@ -175,12 +175,14 @@ private static Jdk createJdk(
         String name,
         String vendor,
         String version,
-        String platform
+        String platform,
+        String architecture
     ) {
         Jdk jdk = jdksContainer.create(name);
         jdk.setVendor(vendor);
         jdk.setVersion(version);
         jdk.setPlatform(platform);
+        jdk.setArchitecture(architecture);
         return jdk;
     }
 
@@ -212,8 +214,8 @@ private static List<Object> configureVM(Project project) {
 
         NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
         String platform = box.contains("windows") ? "windows" : "linux";
-        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform);
-        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform);
+        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform, "x64");
+        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform, "x64");
 
         // setup VM used by these tests
         VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java
index ea291858913f8..758aacbf90c8c 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java
@@ -36,7 +36,7 @@ public static void setupRoot() {
     }
 
     public void testMissingVendor() {
-        assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "vendor not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]");
     }
 
     public void testUnknownVendor() {
@@ -46,20 +46,29 @@ public void testUnknownVendor() {
             "unknown",
             "11.0.2+33",
             "linux",
+            "x64",
             "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
         );
     }
 
     public void testMissingVersion() {
-        assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "version not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]");
     }
 
     public void testBadVersionFormat() {
-        assertJdkError(createProject(), "testjdk", "openjdk", "badversion", "linux", "malformed version [badversion] for jdk [testjdk]");
+        assertJdkError(
+            createProject(),
+            "testjdk",
+            "openjdk",
+            "badversion",
+            "linux",
+            "x64",
+            "malformed version [badversion] for jdk [testjdk]"
+        );
     }
 
     public void testMissingPlatform() {
-        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "platform not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]");
     }
 
     public void testUnknownPlatform() {
@@ -69,19 +78,44 @@ public void testUnknownPlatform() {
             "openjdk",
             "11.0.2+33",
             "unknown",
+            "x64",
             "unknown platform [unknown] for jdk [testjdk], must be one of [darwin, linux, windows, mac]"
         );
     }
 
-    private void assertJdkError(Project project, String name, String vendor, String version, String platform, String message) {
+    public void testMissingArchitecture() {
+        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]");
+    }
+
+    public void testUnknownArchitecture() {
+        assertJdkError(
+            createProject(),
+            "testjdk",
+            "openjdk",
+            "11.0.2+33",
+            "linux",
+            "unknown",
+            "unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]"
+        );
+    }
+
+    private void assertJdkError(
+        final Project project,
+        final String name,
+        final String vendor,
+        final String version,
+        final String platform,
+        final String architecture,
+        final String message
+    ) {
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> createJdk(project, name, vendor, version, platform)
+            () -> createJdk(project, name, vendor, version, platform, architecture)
         );
         assertThat(e.getMessage(), equalTo(message));
     }
 
-    private void createJdk(Project project, String name, String vendor, String version, String platform) {
+    private void createJdk(Project project, String name, String vendor, String version, String platform, String architecture) {
         @SuppressWarnings("unchecked")
         NamedDomainObjectContainer<Jdk> jdks = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
         jdks.create(name, jdk -> {
@@ -94,6 +128,9 @@ private void createJdk(Project project, String name, String vendor, String versi
             if (platform != null) {
                 jdk.setPlatform(platform);
             }
+            if (architecture != null) {
+                jdk.setArchitecture(architecture);
+            }
         }).finalizeValues();
     }
 
diff --git a/buildSrc/src/testKit/jdk-download/reuse/build.gradle b/buildSrc/src/testKit/jdk-download/reuse/build.gradle
index 795098c4b5229..c8b08171091b1 100644
--- a/buildSrc/src/testKit/jdk-download/reuse/build.gradle
+++ b/buildSrc/src/testKit/jdk-download/reuse/build.gradle
@@ -7,5 +7,6 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
+    architeecture = "x64"
   }
 }
diff --git a/buildSrc/src/testKit/jdk-download/subproj/build.gradle b/buildSrc/src/testKit/jdk-download/subproj/build.gradle
index a0713bef204d9..a19a6bead4c7b 100644
--- a/buildSrc/src/testKit/jdk-download/subproj/build.gradle
+++ b/buildSrc/src/testKit/jdk-download/subproj/build.gradle
@@ -10,16 +10,19 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
+    architecture = "x64"
   }
   darwin {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "darwin"
+    architecture = "x64"
   }
   windows {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "windows"
+    architecture = "x64"
   }
 }
 
diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 029eb0733023f..17e9cd36d0212 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -50,7 +50,7 @@ task createJvmOptionsDir(type: EmptyDirTask) {
   dirMode = 0750
 }
 
-CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
+CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean oss, boolean jdk) {
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
@@ -70,7 +70,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
       }
       if (jdk) {
         into("darwin".equals(platform) ? 'jdk.app' : 'jdk') {
-          with jdkFiles(project, platform)
+          with jdkFiles(project, platform, architecture)
         }
       }
       into('') {
@@ -116,31 +116,31 @@ Closure commonZipConfig = {
 
 task buildIntegTestZip(type: Zip) {
   configure(commonZipConfig)
-  with archiveFiles(transportModulesFiles, 'zip', null, true, false)
+  with archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
 }
 
 task buildWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, true)
+  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, true)
 }
 
 task buildOssWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, true)
+  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, true)
 }
 
 task buildNoJdkWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, false)
+  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, false)
 }
 
 task buildOssNoJdkWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, false)
+  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, false)
 }
 
 Closure commonTarConfig = {
@@ -153,49 +153,61 @@ Closure commonTarConfig = {
 task buildDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, true)
+  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, true)
 }
 
 task buildOssDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, true)
+  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, true)
 }
 
 task buildNoJdkDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, false)
+  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, false)
 }
 
 task buildOssNoJdkDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, false)
+  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, false)
+}
+
+task buildLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
+  configure(commonTarConfig)
+  archiveClassifier = 'linux-aarch64'
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'aarch64', false, true)
 }
 
 task buildLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, true)
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, true)
+}
+
+task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
+  configure(commonTarConfig)
+  archiveClassifier = 'linux-x86_64'
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
 }
 
 task buildOssLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, true)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
 }
 
 task buildNoJdkLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, false)
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, false)
 }
 
 task buildOssNoJdkLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, false)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, false)
 }
 
 Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }
diff --git a/distribution/archives/linux-aarch64-tar/build.gradle b/distribution/archives/linux-aarch64-tar/build.gradle
new file mode 100644
index 0000000000000..4a6dde5fc0c92
--- /dev/null
+++ b/distribution/archives/linux-aarch64-tar/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.
diff --git a/distribution/archives/oss-linux-aarch64-tar/build.gradle b/distribution/archives/oss-linux-aarch64-tar/build.gradle
new file mode 100644
index 0000000000000..4a6dde5fc0c92
--- /dev/null
+++ b/distribution/archives/oss-linux-aarch64-tar/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 584b80a8c67de..5fc350f5441e1 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -391,16 +391,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    jdkFiles = { Project project, String platform ->
+    jdkFiles = { Project project, String platform, String architecture ->
       project.jdks {
-        "bundled_${platform}" {
+        "bundled_${platform}_${architecture}" {
           it.platform = platform
           it.version = VersionProperties.getBundledJdk(platform)
           it.vendor = VersionProperties.bundledJdkVendor
+          it.architecture = architecture
         }
       }
       return copySpec {
-        from project.jdks."bundled_${platform}"
+        from project.jdks."bundled_${platform}_${architecture}"
         exclude "demo/**"
         eachFile { FileCopyDetails details ->
           if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
@@ -607,10 +608,13 @@ subprojects {
 
 ['archives:windows-zip', 'archives:oss-windows-zip',
   'archives:darwin-tar', 'archives:oss-darwin-tar',
+  'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
   'archives:linux-tar', 'archives:oss-linux-tar',
   'archives:integ-test-zip',
   'packages:rpm', 'packages:deb',
+  'packages:aarch64-rpm', 'packages:aarch64-deb',
   'packages:oss-rpm', 'packages:oss-deb',
+  'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
 ].forEach { subName ->
   Project subproject = project("${project.path}:${subName}")
   Configuration configuration = configurations.create(subproject.name)
diff --git a/distribution/packages/aarch64-deb/build.gradle b/distribution/packages/aarch64-deb/build.gradle
new file mode 100644
index 0000000000000..4a6dde5fc0c92
--- /dev/null
+++ b/distribution/packages/aarch64-deb/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.
diff --git a/distribution/packages/aarch64-oss-deb/build.gradle b/distribution/packages/aarch64-oss-deb/build.gradle
new file mode 100644
index 0000000000000..4a6dde5fc0c92
--- /dev/null
+++ b/distribution/packages/aarch64-oss-deb/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.
diff --git a/distribution/packages/aarch64-oss-rpm/build.gradle b/distribution/packages/aarch64-oss-rpm/build.gradle
new file mode 100644
index 0000000000000..4a6dde5fc0c92
--- /dev/null
+++ b/distribution/packages/aarch64-oss-rpm/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.
diff --git a/distribution/packages/aarch64-rpm/build.gradle b/distribution/packages/aarch64-rpm/build.gradle
new file mode 100644
index 0000000000000..4a6dde5fc0c92
--- /dev/null
+++ b/distribution/packages/aarch64-rpm/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index 9b1efa3e43d7c..105f244e6bc5e 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -98,17 +98,17 @@ addProcessFilesTask('rpm', false, false)
 // Common configuration that is package dependent. This can't go in ospackage
 // since we have different templated files that need to be consumed, but the structure
 // is the same
-Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
+Closure commonPackageConfig(String type, boolean oss, boolean jdk, String architecture) {
   return {
     onlyIf {
       OS.current().equals(OS.WINDOWS) == false
     }
     dependsOn "process${oss ? 'Oss' : ''}${jdk ? '' : 'NoJdk'}${type.capitalize()}Files"
     packageName "elasticsearch${oss ? '-oss' : ''}"
-    arch(type == 'deb' ? 'amd64' : 'X86_64')
+    arch(architecture == 'aarch64' ? 'aarch64' : type == 'deb' ? 'amd64' : 'X86_64')
     // Follow elasticsearch's file naming convention
     String jdkString = jdk ? "" : "no-jdk-"
-    String prefix = "${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
+    String prefix = "${architecture == 'aarch64' ? 'aarch64-' : ''}${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
     destinationDir = file("${prefix}/build/distributions")
 
     // SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself
@@ -143,7 +143,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       }
       if (jdk) {
         into('jdk') {
-          with jdkFiles(project, 'linux')
+          with jdkFiles(project, 'linux', architecture)
         }
       }
       // we need to specify every intermediate directory in these paths so the package managers know they are explicitly
@@ -306,9 +306,9 @@ ospackage {
   into '/usr/share/elasticsearch'
 }
 
-Closure commonDebConfig(boolean oss, boolean jdk) {
+Closure commonDebConfig(boolean oss, boolean jdk, String architecture) {
   return {
-    configure(commonPackageConfig('deb', oss, jdk))
+    configure(commonPackageConfig('deb', oss, jdk, architecture))
 
     // jdeb does not provide a way to set the License control attribute, and ospackage
     // silently ignores setting it. Instead, we set the license as "custom field"
@@ -336,25 +336,33 @@ Closure commonDebConfig(boolean oss, boolean jdk) {
   }
 }
 
+task buildAarch64Deb(type: Deb) {
+  configure(commonDebConfig(false, true, 'aarch64'))
+}
+
 task buildDeb(type: Deb) {
-  configure(commonDebConfig(false, true))
+  configure(commonDebConfig(false, true, 'x64'))
+}
+
+task buildAarch64OssDeb(type: Deb) {
+  configure(commonDebConfig(true, true, 'aarch64'))
 }
 
 task buildOssDeb(type: Deb) {
-  configure(commonDebConfig(true, true))
+  configure(commonDebConfig(true, true, 'x64'))
 }
 
 task buildNoJdkDeb(type: Deb) {
-  configure(commonDebConfig(false, false))
+  configure(commonDebConfig(false, false, 'x64'))
 }
 
 task buildOssNoJdkDeb(type: Deb) {
-  configure(commonDebConfig(true, false))
+  configure(commonDebConfig(true, false, 'x64'))
 }
 
-Closure commonRpmConfig(boolean oss, boolean jdk) {
+Closure commonRpmConfig(boolean oss, boolean jdk, String architecture) {
   return {
-    configure(commonPackageConfig('rpm', oss, jdk))
+    configure(commonPackageConfig('rpm', oss, jdk, architecture))
 
     if (oss) {
       license 'ASL 2.0'
@@ -381,20 +389,28 @@ Closure commonRpmConfig(boolean oss, boolean jdk) {
   }
 }
 
+task buildAarch64Rpm(type: Rpm) {
+  configure(commonRpmConfig(false, true, 'aarch64'))
+}
+
 task buildRpm(type: Rpm) {
-  configure(commonRpmConfig(false, true))
+  configure(commonRpmConfig(false, true, 'x64'))
+}
+
+task buildAarch64OssRpm(type: Rpm) {
+  configure(commonRpmConfig(true, true, 'aarch64'))
 }
 
 task buildOssRpm(type: Rpm) {
-  configure(commonRpmConfig(true, true))
+  configure(commonRpmConfig(true, true, 'x64'))
 }
 
 task buildNoJdkRpm(type: Rpm) {
-  configure(commonRpmConfig(false, false))
+  configure(commonRpmConfig(false, false, 'x64'))
 }
 
 task buildOssNoJdkRpm(type: Rpm) {
-  configure(commonRpmConfig(true, false))
+  configure(commonRpmConfig(true, false, 'x64'))
 }
 
 Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
@@ -446,6 +462,8 @@ subprojects {
         final File rpmDatabase = new File(extractionDir, 'rpm-database')
         commandLine 'rpm',
           '--badreloc',
+          '--ignorearch',
+          '--ignoreos',
           '--nodeps',
           '--noscripts',
           '--notriggers',
diff --git a/distribution/src/bin/elasticsearch b/distribution/src/bin/elasticsearch
index 136aed6755c5e..e2c2288cb664c 100755
--- a/distribution/src/bin/elasticsearch
+++ b/distribution/src/bin/elasticsearch
@@ -29,7 +29,7 @@ for option in "$@"; do
 done
 
 if [ -z "$ES_TMPDIR" ]; then
-  ES_TMPDIR=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
+  ES_TMPDIR=`"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
 fi
 
 # get keystore password before setting java options to avoid
@@ -52,12 +52,13 @@ fi
 #   - second, JVM options are read from jvm.options and jvm.options.d/*.options
 #   - third, JVM options from ES_JAVA_OPTS are applied
 #   - fourth, ergonomic JVM options are applied
-ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
+ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
 
 # manual parsing to find out, if process should be detached
 if [[ $DAEMONIZE = false ]]; then
   exec \
     "$JAVA" \
+    "$XSHARE" \
     $ES_JAVA_OPTS \
     -Des.path.home="$ES_HOME" \
     -Des.path.conf="$ES_PATH_CONF" \
@@ -70,6 +71,7 @@ if [[ $DAEMONIZE = false ]]; then
 else
   exec \
     "$JAVA" \
+    "$XSHARE" \
     $ES_JAVA_OPTS \
     -Des.path.home="$ES_HOME" \
     -Des.path.conf="$ES_PATH_CONF" \
diff --git a/distribution/src/bin/elasticsearch-cli b/distribution/src/bin/elasticsearch-cli
index 4af827b67caf9..6f03456eb0122 100644
--- a/distribution/src/bin/elasticsearch-cli
+++ b/distribution/src/bin/elasticsearch-cli
@@ -22,6 +22,7 @@ ES_JAVA_OPTS="-Xms4m -Xmx64m -XX:+UseSerialGC ${ES_JAVA_OPTS}"
 
 exec \
   "$JAVA" \
+  "$XSHARE" \
   $ES_JAVA_OPTS \
   -Des.path.home="$ES_HOME" \
   -Des.path.conf="$ES_PATH_CONF" \
diff --git a/distribution/src/bin/elasticsearch-env b/distribution/src/bin/elasticsearch-env
index cbdfbf8facb5c..5a54ad58e0abd 100644
--- a/distribution/src/bin/elasticsearch-env
+++ b/distribution/src/bin/elasticsearch-env
@@ -67,8 +67,14 @@ if [ ! -z "$JAVA_OPTS" ]; then
   echo "pass JVM parameters via ES_JAVA_OPTS"
 fi
 
+if [[ "$("$JAVA" -version 2>/dev/null)" =~ "Unable to map CDS archive" ]]; then
+  XSHARE="-Xshare:off"
+else
+  XSHARE="-Xshare:auto"
+fi
+
 # check the Java version
-"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker
+"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker
 
 export HOSTNAME=$HOSTNAME
 
diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java
index 9ed0479c4d15b..0b2e9f9092a74 100644
--- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java
+++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java
@@ -87,6 +87,7 @@ private static List<String> flagsFinal(final List<String> userDefinedJvmOptions)
         final List<String> command = Stream.of(
             Stream.of(java),
             userDefinedJvmOptions.stream(),
+            Stream.of("-Xshare:off"),
             Stream.of("-XX:+PrintFlagsFinal"),
             Stream.of("-version")
         ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList());
diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle
index e05da2d47c5a6..eea8f87822c1a 100644
--- a/gradle/runtime-jdk-provision.gradle
+++ b/gradle/runtime-jdk-provision.gradle
@@ -1,3 +1,4 @@
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
@@ -9,6 +10,7 @@ jdks {
         vendor = VersionProperties.bundledJdkVendor
         version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
         platform = OS.current().name().toLowerCase()
+        architecture = Architecture.current().name().toLowerCase()
     }
 }
 
diff --git a/settings.gradle b/settings.gradle
index e446e67f3ee7a..133c67b205cca 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -27,7 +27,9 @@ List projects = [
   'distribution:archives:darwin-tar',
   'distribution:archives:oss-no-jdk-darwin-tar',
   'distribution:archives:no-jdk-darwin-tar',
+  'distribution:archives:oss-linux-aarch64-tar',
   'distribution:archives:oss-linux-tar',
+  'distribution:archives:linux-aarch64-tar',
   'distribution:archives:linux-tar',
   'distribution:archives:oss-no-jdk-linux-tar',
   'distribution:archives:no-jdk-linux-tar',
@@ -36,11 +38,15 @@ List projects = [
   'distribution:docker:docker-export',
   'distribution:docker:oss-docker-build-context',
   'distribution:docker:oss-docker-export',
+  'distribution:packages:aarch64-oss-deb',
   'distribution:packages:oss-deb',
+  'distribution:packages:aarch64-deb',
   'distribution:packages:deb',
   'distribution:packages:oss-no-jdk-deb',
   'distribution:packages:no-jdk-deb',
+  'distribution:packages:aarch64-oss-rpm',
   'distribution:packages:oss-rpm',
+  'distribution:packages:aarch64-rpm',
   'distribution:packages:rpm',
   'distribution:packages:oss-no-jdk-rpm',
   'distribution:packages:no-jdk-rpm',
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 740d1015a7eb6..85cdf414383eb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -65,8 +65,15 @@ private XPackSettings() {
     public static final Setting<Boolean> GRAPH_ENABLED = Setting.boolSetting("xpack.graph.enabled", true, Setting.Property.NodeScope);
 
     /** Setting for enabling or disabling machine learning. Defaults to true. */
-    public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting("xpack.ml.enabled", true,
-            Setting.Property.NodeScope);
+    public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting(
+        "xpack.ml.enabled",
+        "aarch64".equals(System.getProperty("os.arch")) ? false : true,
+        value -> {
+            if (value && "aarch64".equals(System.getProperty("os.arch"))) {
+                throw new IllegalArgumentException("[xpack.ml.enabled] can not be set to [true] on [aarch64]");
+            }
+        },
+        Setting.Property.NodeScope);
 
     /** Setting for enabling or disabling rollup. Defaults to true. */
     public static final Setting<Boolean> ROLLUP_ENABLED = Setting.boolSetting("xpack.rollup.enabled", true,
diff --git a/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli b/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli
index 47916a7975ee3..4bafba9bda3a8 100755
--- a/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli
+++ b/x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli
@@ -12,5 +12,6 @@ CLI_JAR=$(ls "$ES_HOME"/bin/elasticsearch-sql-cli-*.jar)
 
 exec \
   "$JAVA" \
+  "$XSHARE" \
   -jar "$CLI_JAR" \
   "$@"

From e4042499e22195a80f80fe20941ffcbff9b1d289 Mon Sep 17 00:00:00 2001
From: Peter Schretlen <peter.schretlen@elastic.co>
Date: Mon, 23 Mar 2020 01:10:37 -0400
Subject: [PATCH 190/285] Allow kibana_system to create and invalidate API keys
 on behalf of other users (#53824)

---
 .../xpack/core/security/authz/store/ReservedRolesStore.java | 2 ++
 .../core/security/authz/store/ReservedRolesStoreTests.java  | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index 8040d6ba52f95..4b6c35bb7ce85 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
@@ -113,6 +114,7 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                 .put(KibanaUser.ROLE_NAME, new RoleDescriptor(KibanaUser.ROLE_NAME,
                         new String[] {
                             "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml", "manage_token", "manage_oidc",
+                            InvalidateApiKeyAction.NAME, "grant_api_key",
                             GetBuiltinPrivilegesAction.NAME, "delegate_pki", GetLifecycleAction.NAME,  PutLifecycleAction.NAME,
                             // The symbolic constant for this one is in SecurityActionMapper, so not accessible from X-Pack core
                             "cluster:admin/analyze"
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 0f933dab443e8..d2cdabd3eb212 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -111,6 +111,8 @@
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
 import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationAction;
+import org.elasticsearch.xpack.core.security.action.GrantApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
@@ -326,6 +328,10 @@ public void testKibanaSystemRole() {
         assertThat(kibanaRole.cluster().check(InvalidateTokenAction.NAME, request, authentication), is(true));
         assertThat(kibanaRole.cluster().check(CreateTokenAction.NAME, request, authentication), is(true));
 
+        // API keys
+        assertThat(kibanaRole.cluster().check(InvalidateApiKeyAction.NAME, request, authentication), is(true));
+        assertThat(kibanaRole.cluster().check(GrantApiKeyAction.NAME, request, authentication), is(true));
+
         // Application Privileges
         DeletePrivilegesRequest deleteKibanaPrivileges = new DeletePrivilegesRequest("kibana-.kibana", new String[]{ "all", "read" });
         DeletePrivilegesRequest deleteLogstashPrivileges = new DeletePrivilegesRequest("logstash", new String[]{ "all", "read" });

From 856721c574d37621698e6241b23e570168075a4b Mon Sep 17 00:00:00 2001
From: Ignacio Vera <iverase@gmail.com>
Date: Mon, 23 Mar 2020 07:58:07 +0100
Subject: [PATCH 191/285] Handle properly indexing rectangles that crosses the
 dateline (#53810)

When indexing a rectangle that crosses the dateline, we are currently not handling it properly and we index a polygon that do not cross the dateline. This changes generates two polygons wrapping the dateline.
---
 .../index/mapper/GeoShapeIndexer.java         | 11 +++++++++-
 .../common/geo/GeometryIndexerTests.java      | 20 +++++++++++++++++++
 .../search/geo/GeoShapeQueryTests.java        | 20 +++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
index 9a103471a21a3..4dc368612c9aa 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.geo.GeoPolygonDecomposer;
 import org.elasticsearch.common.geo.GeoShapeUtils;
 import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
@@ -261,7 +262,15 @@ public Void visit(Polygon polygon) {
 
         @Override
         public Void visit(Rectangle r) {
-            addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLucenePolygon(r)));
+            if (r.getMinLon() > r.getMaxLon()) {
+                Rectangle left = new Rectangle(r.getMinLon(), GeoUtils.MAX_LON, r.getMaxLat(), r.getMinLat());
+                addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLucenePolygon(left)));
+                Rectangle right = new Rectangle(GeoUtils.MIN_LON, r.getMaxLon(), r.getMaxLat(), r.getMinLat());
+                addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLucenePolygon(right)));
+
+            } else {
+                addFields(LatLonShape.createIndexableFields(name, GeoShapeUtils.toLucenePolygon(r)));
+            }
             return null;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
index f5828aad13ee1..ddee41639cd3c 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryIndexerTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -33,6 +34,7 @@
 import org.elasticsearch.geometry.MultiPolygon;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.index.mapper.GeoShapeIndexer;
 import org.elasticsearch.test.ESTestCase;
 import java.io.IOException;
@@ -292,6 +294,24 @@ public void testMultiPoint() {
         assertEquals(indexed, indexer.prepareForIndexing(multiPoint));
     }
 
+    public void testRectangle() {
+        Rectangle indexed = new Rectangle(-179, -178, 10, -10);
+        Geometry processed = indexer.prepareForIndexing(indexed);
+        assertEquals(indexed, processed);
+
+        // a rectangle is broken into two triangles
+        List<IndexableField> fields = indexer.indexShape(null, indexed);
+        assertEquals(fields.size(), 2);
+
+        indexed = new Rectangle(179, -179, 10, -10);
+        processed = indexer.prepareForIndexing(indexed);
+        assertEquals(indexed, processed);
+
+        // a rectangle crossing the dateline is broken into 4 triangles
+        fields = indexer.indexShape(null, indexed);
+        assertEquals(fields.size(), 4);
+    }
+
     public void testPolygon() {
         Polygon polygon = new Polygon(new LinearRing(new double[]{160, 200, 200, 160, 160}, new double[]{10, 10, 20, 20, 10}));
         Geometry indexed = new MultiPolygon(Arrays.asList(
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index 032c6d68766be..eca23aba9eab0 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -791,4 +791,24 @@ public void testDistanceQuery() throws Exception {
             .get();
         assertEquals(0, response.getHits().getTotalHits().value);
     }
+
+    public void testIndexRectangleSpanningDateLine() throws Exception {
+        String mapping = Strings.toString(createRandomMapping());
+
+        client().admin().indices().prepareCreate("test").setMapping(mapping).get();
+        ensureGreen();
+
+        EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(new Coordinate(178, 10), new Coordinate(-178, -10));
+
+        XContentBuilder docSource = envelopeBuilder.toXContent(jsonBuilder().startObject().field("geo"), null).endObject();
+        client().prepareIndex("test").setId("1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get();
+
+        ShapeBuilder filterShape = new PointBuilder(179, 0);
+
+        GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("geo", filterShape);
+        geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS);
+        SearchResponse result = client().prepareSearch("test").setQuery(geoShapeQueryBuilder).get();
+        assertSearchResponse(result);
+        assertHitCount(result, 1);
+    }
 }

From 75cb784a20d7207dd67361e0fa8cacea0d11e259 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Mon, 23 Mar 2020 08:36:09 +0000
Subject: [PATCH 192/285] Allow static cluster.max_voting_config_exclusions
 (#53717)

Today we only read `cluster.max_voting_config_exclusions` from the dynamic
settings in the cluster metadata, ignoring any value set in
`elasticsearch.yml`. This commit addresses this.

Closes #53455
---
 ...nsportAddVotingConfigExclusionsAction.java | 32 +++++++++----
 ...tAddVotingConfigExclusionsActionTests.java | 45 ++++++++++++++-----
 2 files changed, 58 insertions(+), 19 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
index 7dcbd863d29e1..a18e5eb6c67cb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
@@ -39,8 +39,10 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -60,11 +62,21 @@ public class TransportAddVotingConfigExclusionsAction extends TransportMasterNod
     public static final Setting<Integer> MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING
         = Setting.intSetting("cluster.max_voting_config_exclusions", 10, 1, Property.Dynamic, Property.NodeScope);
 
+    private volatile int maxVotingConfigExclusions;
+
     @Inject
-    public TransportAddVotingConfigExclusionsAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
-                                                    ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+    public TransportAddVotingConfigExclusionsAction(Settings settings, ClusterSettings clusterSettings, TransportService transportService,
+                                                    ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
+                                                    IndexNameExpressionResolver indexNameExpressionResolver) {
         super(AddVotingConfigExclusionsAction.NAME, transportService, clusterService, threadPool, actionFilters,
             AddVotingConfigExclusionsRequest::new, indexNameExpressionResolver);
+
+        maxVotingConfigExclusions = MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING, this::setMaxVotingConfigExclusions);
+    }
+
+    private void setMaxVotingConfigExclusions(int maxVotingConfigExclusions) {
+        this.maxVotingConfigExclusions = maxVotingConfigExclusions;
     }
 
     @Override
@@ -81,7 +93,8 @@ protected AddVotingConfigExclusionsResponse read(StreamInput in) throws IOExcept
     protected void masterOperation(Task task, AddVotingConfigExclusionsRequest request, ClusterState state,
                                    ActionListener<AddVotingConfigExclusionsResponse> listener) throws Exception {
 
-        resolveVotingConfigExclusionsAndCheckMaximum(request, state); // throws IAE if no nodes matched or maximum exceeded
+        resolveVotingConfigExclusionsAndCheckMaximum(request, state, maxVotingConfigExclusions);
+        // throws IAE if no nodes matched or maximum exceeded
 
         clusterService.submitStateUpdateTask("add-voting-config-exclusions", new ClusterStateUpdateTask(Priority.URGENT) {
 
@@ -90,14 +103,14 @@ protected void masterOperation(Task task, AddVotingConfigExclusionsRequest reque
             @Override
             public ClusterState execute(ClusterState currentState) {
                 assert resolvedExclusions == null : resolvedExclusions;
-                resolvedExclusions = resolveVotingConfigExclusionsAndCheckMaximum(request, currentState);
+                final int finalMaxVotingConfigExclusions = TransportAddVotingConfigExclusionsAction.this.maxVotingConfigExclusions;
+                resolvedExclusions = resolveVotingConfigExclusionsAndCheckMaximum(request, currentState, finalMaxVotingConfigExclusions);
 
                 final CoordinationMetaData.Builder builder = CoordinationMetaData.builder(currentState.coordinationMetaData());
                 resolvedExclusions.forEach(builder::addVotingConfigExclusion);
                 final MetaData newMetaData = MetaData.builder(currentState.metaData()).coordinationMetaData(builder.build()).build();
                 final ClusterState newState = ClusterState.builder(currentState).metaData(newMetaData).build();
-                assert newState.getVotingConfigExclusions().size() <= MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.get(
-                    currentState.metaData().settings());
+                assert newState.getVotingConfigExclusions().size() <= finalMaxVotingConfigExclusions;
                 return newState;
             }
 
@@ -149,9 +162,10 @@ public void onTimeout(TimeValue timeout) {
     }
 
     private static Set<VotingConfigExclusion> resolveVotingConfigExclusionsAndCheckMaximum(AddVotingConfigExclusionsRequest request,
-                                                                                           ClusterState state) {
-        return request.resolveVotingConfigExclusionsAndCheckMaximum(state,
-            MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.get(state.metaData().settings()), MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.getKey());
+                                                                                           ClusterState state,
+                                                                                           int maxVotingConfigExclusions) {
+        return request.resolveVotingConfigExclusionsAndCheckMaximum(state, maxVotingConfigExclusions,
+            MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.getKey());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java
index d4dda501f5395..87dc705ed5281 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
@@ -80,6 +81,8 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
 
     private TransportService transportService;
     private ClusterStateObserver clusterStateObserver;
+    private ClusterSettings clusterSettings;
+    private int staticMaximum;
 
     @BeforeClass
     public static void createThreadPoolAndClusterService() {
@@ -116,8 +119,18 @@ public void setupForTest() {
         transportService = transport.createTransportService(Settings.EMPTY, threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, emptySet());
 
-        new TransportAddVotingConfigExclusionsAction(transportService, clusterService, threadPool, new ActionFilters(emptySet()),
-            new IndexNameExpressionResolver()); // registers action
+        final Settings.Builder nodeSettingsBuilder = Settings.builder();
+        if (randomBoolean()) {
+            staticMaximum = between(5, 15);
+            nodeSettingsBuilder.put(MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.getKey(), staticMaximum);
+        } else {
+            staticMaximum = MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.get(Settings.EMPTY);
+        }
+        final Settings nodeSettings = nodeSettingsBuilder.build();
+        clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        new TransportAddVotingConfigExclusionsAction(nodeSettings, clusterSettings, transportService, clusterService, threadPool,
+            new ActionFilters(emptySet()), new IndexNameExpressionResolver()); // registers action
 
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -307,20 +320,32 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce
     }
 
     public void testReturnsErrorIfMaximumExclusionCountExceeded() throws InterruptedException {
-        final MetaData.Builder metaDataBuilder = MetaData.builder(clusterService.state().metaData()).persistentSettings(
-                Settings.builder().put(clusterService.state().metaData().persistentSettings())
-                        .put(MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.getKey(), 2).build());
+        final MetaData.Builder metaDataBuilder = MetaData.builder(clusterService.state().metaData());
         CoordinationMetaData.Builder coordinationMetaDataBuilder =
-                CoordinationMetaData.builder(clusterService.state().coordinationMetaData())
-                        .addVotingConfigExclusion(localNodeExclusion);
+            CoordinationMetaData.builder(clusterService.state().coordinationMetaData())
+                .addVotingConfigExclusion(localNodeExclusion);
+
+        final int actualMaximum;
+        if (randomBoolean()) {
+            actualMaximum = staticMaximum;
+        } else {
+            actualMaximum = between(2, 15);
+            clusterSettings.applySettings(Settings.builder().put(clusterService.state().metaData().persistentSettings())
+                .put(MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.getKey(), actualMaximum).build());
+        }
+
+        for (int i = 2; i < actualMaximum; i++) {
+            coordinationMetaDataBuilder.addVotingConfigExclusion(
+                new VotingConfigExclusion(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+        }
 
         final int existingCount, newCount;
         if (randomBoolean()) {
             coordinationMetaDataBuilder.addVotingConfigExclusion(otherNode1Exclusion);
-            existingCount = 2;
+            existingCount = actualMaximum;
             newCount = 1;
         } else {
-            existingCount = 1;
+            existingCount = actualMaximum - 1;
             newCount = 2;
         }
 
@@ -345,7 +370,7 @@ public void testReturnsErrorIfMaximumExclusionCountExceeded() throws Interrupted
         assertThat(rootCause, instanceOf(IllegalArgumentException.class));
         assertThat(rootCause.getMessage(), equalTo("add voting config exclusions request for [other*] would add [" + newCount +
             "] exclusions to the existing [" + existingCount +
-            "] which would exceed the maximum of [2] set by [cluster.max_voting_config_exclusions]"));
+            "] which would exceed the maximum of [" + actualMaximum + "] set by [cluster.max_voting_config_exclusions]"));
     }
 
     public void testTimesOut() throws InterruptedException {

From adcf25e0930266b3601ecef0b996426028e972bc Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 23 Mar 2020 10:37:40 +0200
Subject: [PATCH 193/285] [ML] Refactor DFA custom processor to cross
 validation splitter (#53915)

While `CustomProcessor` is generic and allows for flexibility, there
are new requirements that make cross validation a concept it's hard
to abstract behind custom processor. In particular, we would like to
add data_counts to the DFA jobs stats. Counting training VS. test
docs would be a useful statistic. We would also want to add a
different cross validation strategy for multiclass classification.

This commit renames custom processors to cross validation splitters
which allows for those enhancements without cryptically doing
things as a side effect of the abstract custom processing.
---
 .../process/AnalyticsProcessManager.java      |  9 ++++----
 .../CrossValidationSplitter.java}             |  6 ++---
 .../CrossValidationSplitterFactory.java}      | 12 +++++-----
 .../RandomCrossValidationSplitter.java}       | 10 ++++----
 .../RandomCrossValidationSplitterTests.java}  | 23 +++++++++++--------
 5 files changed, 32 insertions(+), 28 deletions(-)
 rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/{customprocessing/CustomProcessor.java => crossvalidation/CrossValidationSplitter.java} (60%)
 rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/{customprocessing/CustomProcessorFactory.java => crossvalidation/CrossValidationSplitterFactory.java} (76%)
 rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/{customprocessing/DatasetSplittingCustomProcessor.java => crossvalidation/RandomCrossValidationSplitter.java} (82%)
 rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/{customprocessing/DatasetSplittingCustomProcessorTests.java => crossvalidation/RandomCrossValidationSplitterTests.java} (85%)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
index 255ecfd77c021..c7baad202d2f6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -31,8 +31,8 @@
 import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsTask;
 import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
 import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory;
-import org.elasticsearch.xpack.ml.dataframe.process.customprocessing.CustomProcessor;
-import org.elasticsearch.xpack.ml.dataframe.process.customprocessing.CustomProcessorFactory;
+import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitter;
+import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory;
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
@@ -208,7 +208,8 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
     private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProcess<AnalyticsResult> process,
                                DataFrameAnalysis analysis, ProgressTracker progressTracker) throws IOException {
 
-        CustomProcessor customProcessor = new CustomProcessorFactory(dataExtractor.getFieldNames()).create(analysis);
+        CrossValidationSplitter crossValidationSplitter = new CrossValidationSplitterFactory(dataExtractor.getFieldNames())
+            .create(analysis);
 
         // The extra fields are for the doc hash and the control field (should be an empty string)
         String[] record = new String[dataExtractor.getFieldNames().size() + 2];
@@ -226,7 +227,7 @@ private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProces
                         String[] rowValues = row.getValues();
                         System.arraycopy(rowValues, 0, record, 0, rowValues.length);
                         record[record.length - 2] = String.valueOf(row.getChecksum());
-                        customProcessor.process(record);
+                        crossValidationSplitter.process(record);
                         process.writeRecord(record);
                     }
                 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/CustomProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
similarity index 60%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/CustomProcessor.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
index 518aee13b8c57..5d12a2a81a607 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/CustomProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
@@ -3,12 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.dataframe.process.customprocessing;
+package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
 
 /**
- * A processor to manipulate rows before writing them to the process
+ * Processes rows in order to split the dataset in training and test subsets
  */
-public interface CustomProcessor {
+public interface CrossValidationSplitter {
 
     void process(String[] row);
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/CustomProcessorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
similarity index 76%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/CustomProcessorFactory.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
index 77f0b127a2638..47c052dd0bf84 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/CustomProcessorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.dataframe.process.customprocessing;
+package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
 
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
@@ -12,23 +12,23 @@
 import java.util.List;
 import java.util.Objects;
 
-public class CustomProcessorFactory {
+public class CrossValidationSplitterFactory {
 
     private final List<String> fieldNames;
 
-    public CustomProcessorFactory(List<String> fieldNames) {
+    public CrossValidationSplitterFactory(List<String> fieldNames) {
         this.fieldNames = Objects.requireNonNull(fieldNames);
     }
 
-    public CustomProcessor create(DataFrameAnalysis analysis) {
+    public CrossValidationSplitter create(DataFrameAnalysis analysis) {
         if (analysis instanceof Regression) {
             Regression regression = (Regression) analysis;
-            return new DatasetSplittingCustomProcessor(
+            return new RandomCrossValidationSplitter(
                 fieldNames, regression.getDependentVariable(), regression.getTrainingPercent(), regression.getRandomizeSeed());
         }
         if (analysis instanceof Classification) {
             Classification classification = (Classification) analysis;
-            return new DatasetSplittingCustomProcessor(
+            return new RandomCrossValidationSplitter(
                 fieldNames, classification.getDependentVariable(), classification.getTrainingPercent(), classification.getRandomizeSeed());
         }
         return row -> {};
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
similarity index 82%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessor.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
index 6e6acfb271e3f..0afc59628e7de 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.dataframe.process.customprocessing;
+package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
 
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
@@ -12,19 +12,19 @@
 import java.util.Random;
 
 /**
- * A processor that randomly clears the dependent variable value
- * in order to split the dataset in training and validation data.
+ * A cross validation splitter that randomly clears the dependent variable value
+ * in order to split the dataset in training and test data.
  * This relies on the fact that when the dependent variable field
  * is empty, then the row is not used for training but only to make predictions.
  */
-class DatasetSplittingCustomProcessor implements CustomProcessor {
+class RandomCrossValidationSplitter implements CrossValidationSplitter {
 
     private final int dependentVariableIndex;
     private final double trainingPercent;
     private final Random random;
     private boolean isFirstRow = true;
 
-    DatasetSplittingCustomProcessor(List<String> fieldNames, String dependentVariable, double trainingPercent, long randomizeSeed) {
+    RandomCrossValidationSplitter(List<String> fieldNames, String dependentVariable, double trainingPercent, long randomizeSeed) {
         this.dependentVariableIndex = findDependentVariableIndex(fieldNames, dependentVariable);
         this.trainingPercent = trainingPercent;
         this.random = new Random(randomizeSeed);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
similarity index 85%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessorTests.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
index ac897413a4eac..eea102e673893 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.dataframe.process.customprocessing;
+package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
@@ -20,7 +20,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 
-public class DatasetSplittingCustomProcessorTests extends ESTestCase {
+public class RandomCrossValidationSplitterTests extends ESTestCase {
 
     private List<String> fields;
     private int dependentVariableIndex;
@@ -40,7 +40,7 @@ public void setUpTests() {
     }
 
     public void testProcess_GivenRowsWithoutDependentVariableValue() {
-        CustomProcessor customProcessor = new DatasetSplittingCustomProcessor(fields, dependentVariable, 50.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(fields, dependentVariable, 50.0, randomizeSeed);
 
         for (int i = 0; i < 100; i++) {
             String[] row = new String[fields.size()];
@@ -50,7 +50,7 @@ public void testProcess_GivenRowsWithoutDependentVariableValue() {
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            customProcessor.process(processedRow);
+            crossValidationSplitter.process(processedRow);
 
             // As all these rows have no dependent variable value, they're not for training and should be unaffected
             assertThat(Arrays.equals(processedRow, row), is(true));
@@ -58,7 +58,8 @@ public void testProcess_GivenRowsWithoutDependentVariableValue() {
     }
 
     public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() {
-        CustomProcessor customProcessor = new DatasetSplittingCustomProcessor(fields, dependentVariable, 100.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
+            fields, dependentVariable, 100.0, randomizeSeed);
 
         for (int i = 0; i < 100; i++) {
             String[] row = new String[fields.size()];
@@ -68,7 +69,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            customProcessor.process(processedRow);
+            crossValidationSplitter.process(processedRow);
 
             // We should pick them all as training percent is 100
             assertThat(Arrays.equals(processedRow, row), is(true));
@@ -78,7 +79,8 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
     public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() {
         double trainingPercent = randomDoubleBetween(1.0, 100.0, true);
         double trainingFraction = trainingPercent / 100;
-        CustomProcessor customProcessor = new DatasetSplittingCustomProcessor(fields, dependentVariable, trainingPercent, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
+            fields, dependentVariable, trainingPercent, randomizeSeed);
 
         int runCount = 20;
         int rowsCount = 1000;
@@ -92,7 +94,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
                 }
 
                 String[] processedRow = Arrays.copyOf(row, row.length);
-                customProcessor.process(processedRow);
+                crossValidationSplitter.process(processedRow);
 
                 for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
                     if (fieldIndex != dependentVariableIndex) {
@@ -124,7 +126,8 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
     }
 
     public void testProcess_ShouldHaveAtLeastOneTrainingRow() {
-        CustomProcessor customProcessor = new DatasetSplittingCustomProcessor(fields, dependentVariable, 1.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
+            fields, dependentVariable, 1.0, randomizeSeed);
 
         // We have some non-training rows and then a training row to check
         // we maintain the first training row and not just the first row
@@ -139,7 +142,7 @@ public void testProcess_ShouldHaveAtLeastOneTrainingRow() {
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            customProcessor.process(processedRow);
+            crossValidationSplitter.process(processedRow);
 
             assertThat(Arrays.equals(processedRow, row), is(true));
         }

From c334b2045a5f3157d8e066b64963f1858342927b Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Mon, 23 Mar 2020 09:42:51 +0100
Subject: [PATCH 194/285] Get Async Search: omit _clusters section when empty
 (#53907)

The _clusters section is omitted by the search API whenever no remote clusters are searched. Async search should do the same, but Get Async Search returns a deserialized response, hence a weird `_clusters` section with all values set to `0` gets returned instead. In fact the recreated Clusters object is not the same object as the EMPTY constant, yet it has the same content.

This commit addresses this by changing the comparison in the `toXContent` method to not print out the section if the number of total clusters is `0`.
---
 .../elasticsearch/action/search/SearchResponse.java    |  6 ++----
 .../action/search/SearchResponseTests.java             | 10 ++++++++++
 .../rest-api-spec/test/async_search/10_basic.yml       |  4 +++-
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 9cabe14e9eed0..80487710729c9 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -413,9 +413,7 @@ public Clusters(int total, int successful, int skipped) {
         }
 
         private Clusters(StreamInput in) throws IOException {
-            this.total = in.readVInt();
-            this.successful = in.readVInt();
-            this.skipped = in.readVInt();
+            this(in.readVInt(), in.readVInt(), in.readVInt());
         }
 
         @Override
@@ -427,7 +425,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            if (this != EMPTY) {
+            if (total > 0) {
                 builder.startObject(_CLUSTERS_FIELD.getPreferredName());
                 builder.field(TOTAL_FIELD.getPreferredName(), total);
                 builder.field(SUCCESSFUL_FIELD.getPreferredName(), successful);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
index 2f437eb60b306..d47213930b162 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -291,4 +292,13 @@ public void testSerialization() throws IOException {
         assertEquals(searchResponse.getSkippedShards(), deserialized.getSkippedShards());
         assertEquals(searchResponse.getClusters(), deserialized.getClusters());
     }
+
+    public void testToXContentEmptyClusters() throws IOException {
+        SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null, 1, 1, 0, 1,
+            ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+        SearchResponse deserialized = copyWriteable(searchResponse, namedWriteableRegistry, SearchResponse::new, Version.CURRENT);
+        XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
+        deserialized.getClusters().toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals(0, Strings.toString(builder).length());
+    }
 }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
index 292e3bd6ccec6..dd78157e79bef 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
@@ -73,7 +73,6 @@
                 field: max
           sort: max
 
-  - set:    { id:                              id }
   - match:  { version:                          6 }
   - match:  { is_partial:                   false }
   - length: { response.hits.hits:               3 }
@@ -98,6 +97,7 @@
   - set:    { id:                                  id }
   - match:  { version:                              6 }
   - match:  { is_partial:                       false }
+  - is_false: response._clusters
   - length: { response.hits.hits:                   3 }
   - match:  { response.hits.hits.0._source.max:     1 }
   - match:  { response.aggregations.max#max.value:  3.0 }
@@ -108,6 +108,7 @@
 
   - match:  { version:                            6 }
   - match:  { is_partial:                     false }
+  - is_false: response._clusters
   - length: { response.hits.hits:                 3 }
   - match:  { response.hits.hits.0._source.max:   1 }
   - match:  { response.aggregations.max.value:    3.0 }
@@ -120,6 +121,7 @@
 
   - match:  { version:                              6 }
   - match:  { is_partial:                       false }
+  - is_false: response._clusters
   - length: { response.hits.hits:                   3 }
   - match:  { response.hits.hits.0._source.max:     1 }
   - match:  { response.aggregations.max#max.value:  3.0 }

From 108f9ca73fff0cbfd95b51e369b7a1a2770051d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Mon, 23 Mar 2020 10:38:36 +0100
Subject: [PATCH 195/285] [DOCS] Adds painless transform examples (#53274)

Co-authored-by: Lisa Cawley <lcawley@elastic.co>
---
 docs/reference/transform/index.asciidoc       |   2 +
 .../transform/painless-examples.asciidoc      | 329 ++++++++++++++++++
 2 files changed, 331 insertions(+)
 create mode 100644 docs/reference/transform/painless-examples.asciidoc

diff --git a/docs/reference/transform/index.asciidoc b/docs/reference/transform/index.asciidoc
index a71d99caac3f3..84d3afd9e9cb7 100644
--- a/docs/reference/transform/index.asciidoc
+++ b/docs/reference/transform/index.asciidoc
@@ -16,6 +16,7 @@ your data.
 * <<transform-api-quickref>>
 * <<ecommerce-transforms>>
 * <<transform-examples>>
+* <<transform-painless-examples>>
 * <<transform-troubleshooting>>
 * <<transform-limitations>>
 
@@ -26,5 +27,6 @@ include::checkpoints.asciidoc[]
 include::api-quickref.asciidoc[]
 include::ecommerce-tutorial.asciidoc[]
 include::examples.asciidoc[]
+include::painless-examples.asciidoc[]
 include::troubleshooting.asciidoc[]
 include::limitations.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/transform/painless-examples.asciidoc b/docs/reference/transform/painless-examples.asciidoc
new file mode 100644
index 0000000000000..8a2a4ec7386e8
--- /dev/null
+++ b/docs/reference/transform/painless-examples.asciidoc
@@ -0,0 +1,329 @@
+[role="xpack"]
+[testenv="basic"]
+[[transform-painless-examples]]
+=== Painless examples for {transforms}
+++++
+<titleabbrev>Painless examples for {transforms}</titleabbrev>
+++++
+
+These examples demonstrate how to use Painless in {transforms}. You can learn 
+more about the Painless scripting language in the 
+{painless}/painless-guide.html[Painless guide].
+
+* <<painless-top-hits>>
+* <<painless-time-features>>
+* <<painless-group-by>>
+* <<painless-bucket-script>>
+
+
+[discrete]
+[[painless-top-hits]]
+==== Getting top hits by using scripted metric
+
+This snippet shows how to find the latest document, in other words the document 
+with the earliest timestamp. From a technical perspective, it helps to achieve 
+the function of a <<search-aggregations-metrics-top-hits-aggregation>> by using 
+scripted metric aggregation which provides a metric output.
+
+[source,js]
+--------------------------------------------------
+"latest_doc": { 
+  "scripted_metric": {
+    "init_script": "state.timestamp_latest = 0L; state.last_doc = ''", <1>
+    "map_script": """ <2>
+      def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli(); 
+      if (current_date > state.timestamp_latest) 
+      {state.timestamp_latest = current_date;
+      state.last_doc = new HashMap(params['_source']);}
+    """,
+    "combine_script": "return state", <3>
+    "reduce_script": """ <4>
+      def last_doc = '';
+      def timestamp_latest = 0L;
+      for (s in states) {if (s.timestamp_latest > (timestamp_latest))
+      {timestamp_latest = s.timestamp_latest; last_doc = s.last_doc;}} 
+      return last_doc
+    """
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+<1> The `init_script` creates a long type `timestamp_latest` and a string type 
+`last_doc` in the `state` object.
+<2> The `map_script` defines `current_date` based on the timestamp of the 
+document, then compares `current_date` with `state.timestamp_latest`, finally 
+returns `state.last_doc` from the shard. By using `new HashMap(...)` we copy the 
+source document, this is important whenever you want to pass the full source 
+object from one phase to the next.
+<3> The `combine_script` returns `state` from each shard.
+<4> The `reduce_script` iterates through the value of `s.timestamp_latest` 
+returned by each shard and returns the document with the latest timestamp 
+(`last_doc`). In the response, the top hit (in other words, the `latest_doc`) is 
+nested below the `latest_doc` field.
+
+Check the
+<<scripted-metric-aggregation-scope,scope of scripts>>
+for detailed explanation on the respective scripts.
+
+You can retrieve the last value in a similar way: 
+
+[source,js]
+--------------------------------------------------
+"latest_value": {
+  "scripted_metric": {
+    "init_script": "state.timestamp_latest = 0L; state.last_value = ''",
+    "map_script": """
+      def current_date = doc['date'].getValue().toInstant().toEpochMilli(); 
+      if (current_date > state.timestamp_latest) 
+      {state.timestamp_latest = current_date;
+      state.last_value = params['_source']['value'];}
+    """,
+    "combine_script": "return state",
+    "reduce_script": """
+      def last_value = '';
+      def timestamp_latest = 0L; 
+      for (s in states) {if (s.timestamp_latest > (timestamp_latest)) 
+      {timestamp_latest = s.timestamp_latest; last_value = s.last_value;}} 
+      return last_value
+    """
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+
+[discrete]
+[[painless-time-features]]
+==== Getting time features as scripted fields
+
+This snippet shows how to extract time based features by using Painless. The 
+snippet uses an index where `@timestamp` is defined as a `date` type field.
+
+[source,js]
+--------------------------------------------------
+"script_fields": {
+    "hour_of_day": { <1>
+      "script": {
+        "lang": "painless",
+        "source": """
+          ZonedDateTime date =  doc['@timestamp'].value; <2>
+          return date.getHour(); <3>
+        """
+      }
+    },
+    "month_of_year": { <4>
+      "script": {
+        "lang": "painless",
+        "source": """
+          ZonedDateTime date =  doc['@timestamp'].value; <5>
+          return date.getMonthValue(); <6>
+        """
+      }
+    }
+  }
+--------------------------------------------------
+// NOTCONSOLE
+
+<1> Contains the Painless script that returns the hour of the day.
+<2> Sets `date` based on the timestamp of the document.
+<3> Returns the hour value from `date`.
+<4> Contains the Painless script that returns the month of the year.
+<5> Sets `date` based on the timestamp of the document.
+<6> Returns the month value from `date`.
+
+
+[discrete]
+[[painless-group-by]]
+==== Using Painless in `group_by`
+
+It is possible to base the `group_by` property of a {transform} on the output of 
+a script. The following example uses the {kib} sample web logs dataset. The goal 
+here is to make the {transform} output easier to understand through normalizing 
+the value of the fields that the data is grouped by.
+
+[source,console]
+--------------------------------------------------
+POST _transform/_preview
+{
+  "source": {
+    "index": [ <1>
+      "kibana_sample_data_logs"
+    ]
+  },
+  "pivot": {
+    "group_by": {
+      "agent": {
+        "terms": {
+          "script": { <2>
+            "source": """String agent = doc['agent.keyword'].value; 
+            if (agent.contains("MSIE")) { 
+              return "internet explorer";
+            } else if (agent.contains("AppleWebKit")) { 
+              return "safari"; 
+            } else if (agent.contains('Firefox')) { 
+              return "firefox";
+            } else { return agent }""",
+            "lang": "painless"
+          }
+        }
+      }
+    },
+    "aggregations": { <3>
+      "200": {
+        "filter": {
+          "term": {
+            "response": "200"
+          }
+        }
+      },
+      "404": {
+        "filter": {
+          "term": {
+            "response": "404"
+          }
+        }
+      },
+      "503": {
+        "filter": {
+          "term": {
+            "response": "503"
+          }
+        }
+      }
+    }
+  },
+  "dest": { <4>
+    "index": "pivot_logs"
+  }
+} 
+--------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+<1> Specifies the source index or indices.
+<2> The script defines an `agent` string based on the `agent` field of the 
+documents, then iterates through the values. If an `agent` field contains 
+"MSIE", than the script returns "Internet Explorer". If it contains 
+`AppleWebKit`, it returns "safari". It returns "firefox" if the field value 
+contains "Firefox". Finally, in every other case, the value of the field is 
+returned.
+<3> The aggregations object contains filters that narrow down the results to 
+documents that contains `200`, `404`, or `503` values in the `response` field.
+<4> Specifies the destination index of the {transform}.
+
+The API returns the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "preview" : [
+    {
+      "agent" : "firefox",
+      "200" : 4931,
+      "404" : 259,
+      "503" : 172
+    },
+    {
+      "agent" : "internet explorer",
+      "200" : 3674,
+      "404" : 210,
+      "503" : 126
+    },
+    {
+      "agent" : "safari",
+      "200" : 4227,
+      "404" : 332,
+      "503" : 143
+    }
+  ],
+  "mappings" : {
+    "properties" : {
+      "200" : {
+        "type" : "long"
+      },
+      "agent" : {
+        "type" : "keyword"
+      },
+      "404" : {
+        "type" : "long"
+      },
+      "503" : {
+        "type" : "long"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+You can see that the `agent` values are simplified so it is easier to interpret 
+them. The table below shows how normalization modifies the output of the 
+{transform} in our example compared to the non-normalized values.
+
+[width="50%"]
+
+|===
+| Non-normalized `agent` value                                                 | Normalized `agent` value 
+
+| "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" | "internet explorer"
+| "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24" | "safari"
+| "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1" | "firefox"
+|===
+
+
+[discrete]
+[[painless-bucket-script]]
+==== Getting duration by using bucket script
+
+This example shows you how to get the duration of a session by client IP from a 
+data log by using 
+{ref}/search-aggregations-pipeline-bucket-script-aggregation.html[bucket script]. 
+The example uses the {kib} sample web logs dataset.
+
+[source,console]
+--------------------------------------------------
+PUT _data_frame/transforms/data_log
+{
+  "source": {
+    "index": "kibana_sample_data_logs"
+  },
+  "dest": {
+    "index": "data-logs-by-client"
+  },
+  "pivot": {
+    "group_by": {
+      "machine.os": {"terms": {"field": "machine.os.keyword"}},
+      "machine.ip": {"terms": {"field": "clientip"}}
+    },
+    "aggregations": {
+      "time_frame.lte": {
+        "max": {
+          "field": "timestamp"
+        }
+      },
+      "time_frame.gte": {
+        "min": {
+          "field": "timestamp"
+        }
+      },
+      "time_length": { <1>
+        "bucket_script": {
+          "buckets_path": { <2>
+            "min": "time_frame.gte.value",
+            "max": "time_frame.lte.value"
+          },
+          "script": "params.max - params.min" <3>
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+<1> To define the length of the sessions, we use a bucket script.
+<2> The bucket path is a map of script variables and their associated path to 
+the buckets you want to use for the variable. In this particular case, `min` and 
+`max` are variables mapped to `time_frame.gte.value` and `time_frame.lte.value`.
+<3> Finally, the script substracts the start date of the session from the end 
+date which results in the duration of the session.

From c8290793303d3f7f9cef08b924615fd22be9560c Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Mon, 23 Mar 2020 10:50:37 +0100
Subject: [PATCH 196/285] Only link fd* files during source-only snapshot
 (#53463)

Source-only snapshots currently create a second full source-only copy of the shard on disk to
support incrementality during upload. Given that stored fields are occupying a substantial part
of a shard's storage, this means that clusters with source-only snapshots can require up to
50% more local storage. Ideally we would only generate source-only parts of the shard for the
things that need to be uploaded (i.e. do incrementality checks on original file instead of
trimmed-down source-only versions), but that requires much bigger changes to the snapshot
infrastructure. This here is an attempt to dramatically cut down on the storage used by the
source-only copy of the shard by soft-linking the stored-fields files (fd*) instead of copying
them.

Relates #50231
---
 .../snapshots/SourceOnlySnapshot.java         | 293 ++++++++++++++----
 .../SourceOnlySnapshotRepository.java         |   9 +-
 .../snapshots/SourceOnlySnapshotTests.java    |  57 ++--
 3 files changed, 281 insertions(+), 78 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
index ef9a8e8d5f063..ba47939481bc2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
@@ -30,25 +30,34 @@
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.TrackingDirectoryWrapper;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.core.internal.io.IOUtils;
 
 import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
 import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION;
@@ -60,23 +69,23 @@ public class SourceOnlySnapshot {
 
     private static final String FIELDS_INDEX_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_INDEX_EXTENSION_SUFFIX;
     private static final String FIELDS_META_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_META_EXTENSION_SUFFIX;
-    private final Directory targetDirectory;
+    private final LinkedFilesDirectory targetDirectory;
     private final Supplier<Query> deleteByQuerySupplier;
 
-    public SourceOnlySnapshot(Directory targetDirectory, Supplier<Query> deleteByQuerySupplier) {
+    public SourceOnlySnapshot(LinkedFilesDirectory targetDirectory, Supplier<Query> deleteByQuerySupplier) {
         this.targetDirectory = targetDirectory;
         this.deleteByQuerySupplier = deleteByQuerySupplier;
     }
 
-    public SourceOnlySnapshot(Directory targetDirectory) {
+    public SourceOnlySnapshot(LinkedFilesDirectory targetDirectory) {
         this(targetDirectory, null);
     }
 
     public synchronized List<String> syncSnapshot(IndexCommit commit) throws IOException {
         long generation;
         Map<BytesRef, SegmentCommitInfo> existingSegments = new HashMap<>();
-        if (Lucene.indexExists(targetDirectory)) {
-            SegmentInfos existingsSegmentInfos = Lucene.readSegmentInfos(targetDirectory);
+        if (Lucene.indexExists(targetDirectory.getWrapped())) {
+            SegmentInfos existingsSegmentInfos = Lucene.readSegmentInfos(targetDirectory.getWrapped());
             for (SegmentCommitInfo info : existingsSegmentInfos) {
                 existingSegments.put(new BytesRef(info.info.getId()), info);
             }
@@ -191,63 +200,78 @@ DirectoryReader wrapReader(DirectoryReader reader) throws IOException {
 
     private SegmentCommitInfo syncSegment(SegmentCommitInfo segmentCommitInfo, LiveDocs liveDocs, FieldInfos fieldInfos,
                                           Map<BytesRef, SegmentCommitInfo> existingSegments, List<String> createdFiles) throws IOException {
-        SegmentInfo si = segmentCommitInfo.info;
-        Codec codec = si.getCodec();
-        final String segmentSuffix = "";
-        SegmentCommitInfo newInfo;
-        final TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(targetDirectory);
-        BytesRef segmentId = new BytesRef(si.getId());
-        boolean exists = existingSegments.containsKey(segmentId);
-        if (exists == false) {
-            SegmentInfo newSegmentInfo = new SegmentInfo(si.dir, si.getVersion(), si.getMinVersion(), si.name, si.maxDoc(), false,
-                si.getCodec(), si.getDiagnostics(), si.getId(), si.getAttributes(), null);
-            // we drop the sort on purpose since the field we sorted on doesn't exist in the target index anymore.
-            newInfo = new SegmentCommitInfo(newSegmentInfo, 0, 0, -1, -1, -1);
-            List<FieldInfo> fieldInfoCopy = new ArrayList<>(fieldInfos.size());
-            for (FieldInfo fieldInfo : fieldInfos) {
-                fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number,
-                    false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0, 0,
-                    fieldInfo.isSoftDeletesField()));
-            }
-            FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0]));
-            codec.fieldInfosFormat().write(trackingDir, newSegmentInfo, segmentSuffix, newFieldInfos, IOContext.DEFAULT);
-            newInfo.setFieldInfosFiles(trackingDir.getCreatedFiles());
-            String idxFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_INDEX_EXTENSION);
-            String dataFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_EXTENSION);
-            String metaFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_META_EXTENSION);
-            Directory sourceDir = newSegmentInfo.dir;
+        Directory toClose = null;
+        try {
+            SegmentInfo si = segmentCommitInfo.info;
+            Codec codec = si.getCodec();
+            Directory sourceDir = si.dir;
             if (si.getUseCompoundFile()) {
-                sourceDir = codec.compoundFormat().getCompoundReader(sourceDir, si, IOContext.DEFAULT);
+                sourceDir = new LinkedFilesDirectory.CloseMePleaseWrapper(
+                    codec.compoundFormat().getCompoundReader(sourceDir, si, IOContext.DEFAULT));
+                toClose = sourceDir;
+            }
+            final String segmentSuffix = "";
+            SegmentCommitInfo newInfo;
+            final TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(targetDirectory);
+            BytesRef segmentId = new BytesRef(si.getId());
+            boolean exists = existingSegments.containsKey(segmentId);
+            if (exists == false) {
+                SegmentInfo newSegmentInfo = new SegmentInfo(targetDirectory, si.getVersion(), si.getMinVersion(), si.name, si.maxDoc(),
+                    false, si.getCodec(), si.getDiagnostics(), si.getId(), si.getAttributes(), null);
+                // we drop the sort on purpose since the field we sorted on doesn't exist in the target index anymore.
+                newInfo = new SegmentCommitInfo(newSegmentInfo, 0, 0, -1, -1, -1);
+                List<FieldInfo> fieldInfoCopy = new ArrayList<>(fieldInfos.size());
+                for (FieldInfo fieldInfo : fieldInfos) {
+                    fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number,
+                        false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0, 0,
+                        fieldInfo.isSoftDeletesField()));
+                }
+                FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0]));
+                codec.fieldInfosFormat().write(trackingDir, newSegmentInfo, segmentSuffix, newFieldInfos, IOContext.DEFAULT);
+                newInfo.setFieldInfosFiles(trackingDir.getCreatedFiles());
+            } else {
+                newInfo = existingSegments.get(segmentId);
+                assert newInfo.info.getUseCompoundFile() == false;
             }
+
+            // link files for stored fields to target directory
+            final String idxFile = IndexFileNames.segmentFileName(newInfo.info.name, segmentSuffix, FIELDS_INDEX_EXTENSION);
+            final String dataFile = IndexFileNames.segmentFileName(newInfo.info.name, segmentSuffix, FIELDS_EXTENSION);
+            final String metaFile = IndexFileNames.segmentFileName(newInfo.info.name, segmentSuffix, FIELDS_META_EXTENSION);
             trackingDir.copyFrom(sourceDir, idxFile, idxFile, IOContext.DEFAULT);
+            assert targetDirectory.linkedFiles.containsKey(idxFile);
+            assert trackingDir.getCreatedFiles().contains(idxFile);
             trackingDir.copyFrom(sourceDir, dataFile, dataFile, IOContext.DEFAULT);
-             if (Arrays.asList(sourceDir.listAll()).contains(metaFile)) { // only exists for Lucene 8.5+ indices
-                 trackingDir.copyFrom(sourceDir, metaFile, metaFile, IOContext.DEFAULT);
-             }
-            if (sourceDir != newSegmentInfo.dir) {
-                sourceDir.close();
+            assert targetDirectory.linkedFiles.containsKey(dataFile);
+            assert trackingDir.getCreatedFiles().contains(dataFile);
+            if (Arrays.asList(sourceDir.listAll()).contains(metaFile)) { // only exists for Lucene 8.5+ indices
+                trackingDir.copyFrom(sourceDir, metaFile, metaFile, IOContext.DEFAULT);
+                assert targetDirectory.linkedFiles.containsKey(metaFile);
+                assert trackingDir.getCreatedFiles().contains(metaFile);
             }
-        } else {
-            newInfo = existingSegments.get(segmentId);
-            assert newInfo.info.getUseCompoundFile() == false;
-        }
-        if (liveDocs.bits != null && liveDocs.numDeletes != 0 && liveDocs.numDeletes != newInfo.getDelCount()) {
-            if (newInfo.getDelCount() != 0) {
-                assert assertLiveDocs(liveDocs.bits, liveDocs.numDeletes);
+
+            if (liveDocs.bits != null && liveDocs.numDeletes != 0 && liveDocs.numDeletes != newInfo.getDelCount()) {
+                assert newInfo.getDelCount() == 0 || assertLiveDocs(liveDocs.bits, liveDocs.numDeletes);
+                codec.liveDocsFormat().writeLiveDocs(liveDocs.bits, trackingDir, newInfo, liveDocs.numDeletes - newInfo.getDelCount(),
+                    IOContext.DEFAULT);
+                SegmentCommitInfo info = new SegmentCommitInfo(newInfo.info, liveDocs.numDeletes, 0, newInfo.getNextDelGen(), -1, -1);
+                info.setFieldInfosFiles(newInfo.getFieldInfosFiles());
+                info.info.setFiles(trackingDir.getCreatedFiles());
+                newInfo = info;
             }
-            codec.liveDocsFormat().writeLiveDocs(liveDocs.bits, trackingDir, newInfo, liveDocs.numDeletes - newInfo.getDelCount(),
-                IOContext.DEFAULT);
-            SegmentCommitInfo info = new SegmentCommitInfo(newInfo.info, liveDocs.numDeletes, 0, newInfo.getNextDelGen(), -1, -1);
-            info.setFieldInfosFiles(newInfo.getFieldInfosFiles());
-            info.info.setFiles(trackingDir.getCreatedFiles());
-            newInfo = info;
-        }
-        if (exists == false) {
-            newInfo.info.setFiles(trackingDir.getCreatedFiles());
-            codec.segmentInfoFormat().write(trackingDir, newInfo.info, IOContext.DEFAULT);
+            if (exists == false) {
+                newInfo.info.setFiles(trackingDir.getCreatedFiles());
+                codec.segmentInfoFormat().write(trackingDir, newInfo.info, IOContext.DEFAULT);
+            }
+            final Set<String> createdFilesForThisSegment = trackingDir.getCreatedFiles();
+            createdFilesForThisSegment.remove(idxFile);
+            createdFilesForThisSegment.remove(dataFile);
+            createdFilesForThisSegment.remove(metaFile);
+            createdFiles.addAll(createdFilesForThisSegment);
+            return newInfo;
+        } finally {
+            IOUtils.close(toClose);
         }
-        createdFiles.addAll(trackingDir.getCreatedFiles());
-        return newInfo;
     }
 
     private boolean assertLiveDocs(Bits liveDocs, int deletes) {
@@ -270,4 +294,163 @@ private static class LiveDocs {
             this.bits = bits;
         }
     }
+
+    public static class LinkedFilesDirectory extends Directory {
+
+        private final Directory wrapped;
+        private final Map<String, Directory> linkedFiles = new HashMap<>();
+
+        public LinkedFilesDirectory(Directory wrapped) {
+            this.wrapped = wrapped;
+        }
+
+        public Directory getWrapped() {
+            return wrapped;
+        }
+
+        @Override
+        public String[] listAll() throws IOException {
+            Set<String> files = new HashSet<>();
+            Collections.addAll(files, wrapped.listAll());
+            files.addAll(linkedFiles.keySet());
+            String[] result = files.toArray(Strings.EMPTY_ARRAY);
+            Arrays.sort(result);
+            return result;
+        }
+
+        @Override
+        public void deleteFile(String name) throws IOException {
+            final Directory directory = linkedFiles.remove(name);
+            if (directory == null) {
+                wrapped.deleteFile(name);
+            } else {
+                try (directory) {
+                    wrapped.deleteFile(name);
+                } catch (NoSuchFileException | FileNotFoundException e) {
+                    // ignore
+                }
+            }
+        }
+
+        @Override
+        public long fileLength(String name) throws IOException {
+            final Directory linkedDir = linkedFiles.get(name);
+            if (linkedDir != null) {
+                return linkedDir.fileLength(name);
+            } else {
+                return wrapped.fileLength(name);
+            }
+        }
+
+        @Override
+        public IndexOutput createOutput(String name, IOContext context) throws IOException {
+            if (linkedFiles.containsKey(name)) {
+                throw new IllegalArgumentException("file cannot be created as linked file with name " + name + " already exists");
+            } else {
+                return wrapped.createOutput(name, context);
+            }
+        }
+
+        @Override
+        public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+            return wrapped.createTempOutput(prefix, suffix, context);
+        }
+
+        @Override
+        public void sync(Collection<String> names) throws IOException {
+            final List<String> primaryNames = new ArrayList<>();
+
+            for (String name : names) {
+                if (linkedFiles.containsKey(name) == false) {
+                    primaryNames.add(name);
+                }
+            }
+
+            if (primaryNames.isEmpty() == false) {
+                wrapped.sync(primaryNames);
+            }
+        }
+
+        @Override
+        public void syncMetaData() throws IOException {
+            wrapped.syncMetaData();
+        }
+
+        @Override
+        public void rename(String source, String dest) throws IOException {
+            if (linkedFiles.containsKey(source) || linkedFiles.containsKey(dest)) {
+                throw new IllegalArgumentException("file cannot be renamed as linked file with name " + source + " or " + dest +
+                    " already exists");
+            } else {
+                wrapped.rename(source, dest);
+            }
+        }
+
+        @Override
+        public IndexInput openInput(String name, IOContext context) throws IOException {
+            final Directory linkedDir = linkedFiles.get(name);
+            if (linkedDir != null) {
+                return linkedDir.openInput(name, context);
+            } else {
+                return wrapped.openInput(name, context);
+            }
+        }
+
+        @Override
+        public Lock obtainLock(String name) throws IOException {
+            return wrapped.obtainLock(name);
+        }
+
+        @Override
+        public void close() throws IOException {
+            IOUtils.close(() -> IOUtils.close(linkedFiles.values()), linkedFiles::clear, wrapped);
+        }
+
+        @Override
+        public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
+            if (src.equals(dest) == false) {
+                throw new IllegalArgumentException();
+            } else {
+                final Directory previous;
+                if (from instanceof CloseMePleaseWrapper) {
+                    ((CloseMePleaseWrapper) from).incRef();
+                    previous = linkedFiles.put(src, from);
+                } else {
+                    previous = linkedFiles.put(src, new FilterDirectory(from) {
+                        @Override
+                        public void close() {
+                            // ignore
+                        }
+                    });
+                }
+                IOUtils.close(previous);
+            }
+        }
+
+        static class CloseMePleaseWrapper extends FilterDirectory {
+
+            private final AtomicInteger refCount = new AtomicInteger(1);
+
+            CloseMePleaseWrapper(Directory in) {
+                super(in);
+            }
+
+            public void incRef() {
+                int ref = refCount.incrementAndGet();
+                assert ref > 1;
+            }
+
+            @Override
+            public void close() throws IOException {
+                if (refCount.decrementAndGet() == 0) {
+                    in.close();
+                }
+            }
+        }
+
+        @Override
+        public Set<String> getPendingDeletions() throws IOException {
+            return wrapped.getPendingDeletions();
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
index 2cd00bc3301f0..e65fbdace62d6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
@@ -138,9 +138,10 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s
         Path snapPath = dataPath.resolve(SNAPSHOT_DIR_NAME);
         final List<Closeable> toClose = new ArrayList<>(3);
         try {
-            FSDirectory directory = new SimpleFSDirectory(snapPath);
-            toClose.add(directory);
-            Store tempStore = new Store(store.shardId(), store.indexSettings(), directory, new ShardLock(store.shardId()) {
+            SourceOnlySnapshot.LinkedFilesDirectory overlayDir = new SourceOnlySnapshot.LinkedFilesDirectory(
+                new SimpleFSDirectory(snapPath));
+            toClose.add(overlayDir);
+            Store tempStore = new Store(store.shardId(), store.indexSettings(), overlayDir, new ShardLock(store.shardId()) {
                 @Override
                 protected void closeInternal() {
                     // do nothing;
@@ -148,7 +149,7 @@ protected void closeInternal() {
             }, Store.OnClose.EMPTY);
             Supplier<Query> querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null;
             // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here
-            SourceOnlySnapshot snapshot = new SourceOnlySnapshot(tempStore.directory(), querySupplier);
+            SourceOnlySnapshot snapshot = new SourceOnlySnapshot(overlayDir, querySupplier);
             snapshot.syncSnapshot(snapshotIndexCommit);
             // we will use the lucene doc ID as the seq ID so we set the local checkpoint to maxDoc with a new index UUID
             SegmentInfos segmentInfos = tempStore.readLastCommittedSegmentsInfo();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java
index 29854002a68b9..3e51131ba1343 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOSupplier;
 import org.elasticsearch.common.lucene.Lucene;
@@ -48,7 +49,7 @@
 
 public class SourceOnlySnapshotTests extends ESTestCase {
     public void testSourceOnlyRandom() throws IOException {
-        try (Directory dir = newDirectory(); Directory targetDir = newDirectory()) {
+        try (Directory dir = newDirectory(); BaseDirectoryWrapper targetDir = newDirectory()) {
             SnapshotDeletionPolicy deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
             IndexWriterConfig indexWriterConfig = newIndexWriterConfig().setIndexDeletionPolicy
                 (deletionPolicy).setSoftDeletesField(random().nextBoolean() ? null : Lucene.SOFT_DELETES_FIELD);
@@ -56,7 +57,9 @@ public void testSourceOnlyRandom() throws IOException {
                 final String softDeletesField = writer.w.getConfig().getSoftDeletesField();
                 // we either use the soft deletes directly or manually delete them to test the additional delete functionality
                 boolean modifyDeletedDocs = softDeletesField != null && randomBoolean();
-                SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir,
+                targetDir.setCheckIndexOnClose(false);
+                final SourceOnlySnapshot.LinkedFilesDirectory wrappedDir = new SourceOnlySnapshot.LinkedFilesDirectory(targetDir);
+                SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(wrappedDir,
                     modifyDeletedDocs ? () -> new DocValuesFieldExistsQuery(softDeletesField) : null) {
                     @Override
                     DirectoryReader wrapReader(DirectoryReader reader) throws IOException {
@@ -92,7 +95,7 @@ DirectoryReader wrapReader(DirectoryReader reader) throws IOException {
                 IndexCommit snapshot = deletionPolicy.snapshot();
                 try {
                     snapshoter.syncSnapshot(snapshot);
-                    try (DirectoryReader snapReader = snapshoter.wrapReader(DirectoryReader.open(targetDir));
+                    try (DirectoryReader snapReader = snapshoter.wrapReader(DirectoryReader.open(wrappedDir));
                          DirectoryReader wrappedReader = snapshoter.wrapReader(DirectoryReader.open(snapshot))) {
                          DirectoryReader reader = modifyDeletedDocs
                              ? new SoftDeletesDirectoryReaderWrapper(wrappedReader, softDeletesField) :
@@ -111,6 +114,7 @@ DirectoryReader wrapReader(DirectoryReader reader) throws IOException {
                     }
                 } finally {
                     deletionPolicy.release(snapshot);
+                    wrappedDir.close();
                 }
             }
         }
@@ -165,13 +169,15 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
             doc.add(new StoredField("src", "the quick brown fox"));
             writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField(Lucene.SOFT_DELETES_FIELD, 1));
             writer.commit();
-            Directory targetDir = newDirectory();
+            BaseDirectoryWrapper targetDir = newDirectory();
+            targetDir.setCheckIndexOnClose(false);
             IndexCommit snapshot = deletionPolicy.snapshot();
-            SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir);
+            SourceOnlySnapshot.LinkedFilesDirectory wrappedDir = new SourceOnlySnapshot.LinkedFilesDirectory(targetDir);
+            SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(wrappedDir);
             snapshoter.syncSnapshot(snapshot);
 
             StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(snapshot);
-            try (DirectoryReader snapReader = DirectoryReader.open(targetDir)) {
+            try (DirectoryReader snapReader = DirectoryReader.open(wrappedDir)) {
                 assertEquals(snapReader.maxDoc(), 3);
                 assertEquals(snapReader.numDocs(), 2);
                 for (int i = 0; i < 3; i++) {
@@ -182,7 +188,12 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
                 assertEquals(0, id.totalHits.value);
             }
 
-            snapshoter = new SourceOnlySnapshot(targetDir);
+            targetDir = newDirectory(targetDir);
+            targetDir.setCheckIndexOnClose(false);
+            wrappedDir.close();
+            wrappedDir = new SourceOnlySnapshot.LinkedFilesDirectory(targetDir);
+
+            snapshoter = new SourceOnlySnapshot(wrappedDir);
             List<String> createdFiles = snapshoter.syncSnapshot(snapshot);
             assertEquals(0, createdFiles.size());
             deletionPolicy.release(snapshot);
@@ -200,17 +211,18 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
             doc.add(new StoredField("src", "the quick blue fox"));
             writer.addDocument(doc);
             writer.commit();
+            targetDir = newDirectory(targetDir);
+            targetDir.setCheckIndexOnClose(false);
+            wrappedDir.close();
+            wrappedDir = new SourceOnlySnapshot.LinkedFilesDirectory(targetDir);
             {
                 snapshot = deletionPolicy.snapshot();
-                snapshoter = new SourceOnlySnapshot(targetDir);
+                snapshoter = new SourceOnlySnapshot(wrappedDir);
                 createdFiles = snapshoter.syncSnapshot(snapshot);
-                assertEquals(5, createdFiles.size());
+                assertEquals(2, createdFiles.size());
                 for (String file : createdFiles) {
                     String extension = IndexFileNames.getExtension(file);
                     switch (extension) {
-                        case "fdt":
-                        case "fdx":
-                        case "fdm":
                         case "fnm":
                         case "si":
                             break;
@@ -218,7 +230,7 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
                             fail("unexpected extension: " + extension);
                     }
                 }
-                try(DirectoryReader snapReader = DirectoryReader.open(targetDir)) {
+                try(DirectoryReader snapReader = DirectoryReader.open(wrappedDir)) {
                     assertEquals(snapReader.maxDoc(), 5);
                     assertEquals(snapReader.numDocs(), 4);
                 }
@@ -226,9 +238,13 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
             }
             writer.deleteDocuments(new Term("id", "5"));
             writer.commit();
+            targetDir = newDirectory(targetDir);
+            targetDir.setCheckIndexOnClose(false);
+            wrappedDir.close();
+            wrappedDir = new SourceOnlySnapshot.LinkedFilesDirectory(targetDir);
             {
                 snapshot = deletionPolicy.snapshot();
-                snapshoter = new SourceOnlySnapshot(targetDir);
+                snapshoter = new SourceOnlySnapshot(wrappedDir);
                 createdFiles = snapshoter.syncSnapshot(snapshot);
                 assertEquals(1, createdFiles.size());
                 for (String file : createdFiles) {
@@ -240,15 +256,15 @@ public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo,
                             fail("unexpected extension: " + extension);
                     }
                 }
-                try(DirectoryReader snapReader = DirectoryReader.open(targetDir)) {
+                try(DirectoryReader snapReader = DirectoryReader.open(wrappedDir)) {
                     assertEquals(snapReader.maxDoc(), 5);
                     assertEquals(snapReader.numDocs(), 3);
                 }
                 deletionPolicy.release(snapshot);
             }
             writer.close();
-            targetDir.close();
             reader.close();
+            wrappedDir.close();
         }
     }
 
@@ -284,12 +300,14 @@ public boolean keepFullyDeletedSegment(IOSupplier<CodecReader> readerIOSupplier)
             doc.add(new StoredField("src", "the quick brown fox"));
             writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField(Lucene.SOFT_DELETES_FIELD, 1));
             writer.commit();
-            try (Directory targetDir = newDirectory()) {
+            try (BaseDirectoryWrapper targetDir = newDirectory()) {
+                targetDir.setCheckIndexOnClose(false);
                 IndexCommit snapshot = deletionPolicy.snapshot();
-                SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(targetDir);
+                SourceOnlySnapshot.LinkedFilesDirectory wrappedDir = new SourceOnlySnapshot.LinkedFilesDirectory(targetDir);
+                SourceOnlySnapshot snapshoter = new SourceOnlySnapshot(wrappedDir);
                 snapshoter.syncSnapshot(snapshot);
 
-                try (DirectoryReader snapReader = DirectoryReader.open(targetDir)) {
+                try (DirectoryReader snapReader = DirectoryReader.open(wrappedDir)) {
                     assertEquals(snapReader.maxDoc(), 1);
                     assertEquals(snapReader.numDocs(), 1);
                     assertEquals("3", snapReader.document(0).getField("rank").stringValue());
@@ -298,6 +316,7 @@ public boolean keepFullyDeletedSegment(IOSupplier<CodecReader> readerIOSupplier)
                     assertEquals(writerReader.maxDoc(), 2);
                     assertEquals(writerReader.numDocs(), 1);
                 }
+                wrappedDir.close();
             }
             writer.close();
         }

From 25d3912ede0e5a7d5bd094d5fe1c952348491299 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 23 Mar 2020 11:12:42 +0100
Subject: [PATCH 197/285] Fix Azure Repository with HTTPs Endpoint (#53903)

Upgrading to 8.6.2 in #53865 broke running against HTTPs endpoints (and hence real azure)
because the https url connection needs the newly added permission to work.
---
 .../src/main/plugin-metadata/plugin-security.policy              | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
index 19a35f8405903..0e2572a63156d 100644
--- a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
@@ -20,4 +20,5 @@
 grant {
   // azure client opens socket connections for to access repository
   permission java.net.SocketPermission "*", "connect";
+  permission java.lang.RuntimePermission "setFactory";
 };

From 23cccf088810b8416ed278571352393cc2de9523 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 23 Mar 2020 12:22:37 +0100
Subject: [PATCH 198/285] Use Azure Bulk Deletes in Azure Repository (#53919)

Now that we upgraded the Azure SDK to 8.6.2 in #53865 we can make use of
bulk deletes.
---
 .../qa/microsoft-azure-storage/build.gradle   |  2 -
 .../azure/AzureBlobContainer.java             | 41 +-------
 .../repositories/azure/AzureBlobStore.java    | 20 ++--
 .../repositories/azure/AzureRepository.java   |  2 +-
 .../azure/AzureRepositoryPlugin.java          | 15 ---
 .../azure/AzureStorageService.java            | 98 +++++++++----------
 .../azure/AzureBlobContainerRetriesTests.java |  8 +-
 .../azure/AzureBlobStoreRepositoryTests.java  |  2 +-
 .../java/fixture/azure/AzureHttpFixture.java  |  2 +-
 .../java/fixture/azure/AzureHttpHandler.java  | 55 ++++++++++-
 10 files changed, 112 insertions(+), 133 deletions(-)

diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
index b594ff3b172f2..3e5cd7758d22c 100644
--- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
+++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
@@ -88,7 +88,5 @@ testClusters.integTest {
     // in a hacky way to change the protocol and endpoint. We must fix that.
     setting 'azure.client.integration_test.endpoint_suffix',
       { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=${-> azureAddress()}" }, IGNORE_VALUE
-    String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
-    setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
   }
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
index 2093139e115a3..6bd480f7923a9 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
@@ -23,17 +23,12 @@
 import com.microsoft.azure.storage.StorageException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.action.support.GroupedActionListener;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.DeleteResult;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,20 +37,18 @@
 import java.nio.file.NoSuchFileException;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
 
 public class AzureBlobContainer extends AbstractBlobContainer {
 
     private final Logger logger = LogManager.getLogger(AzureBlobContainer.class);
     private final AzureBlobStore blobStore;
-    private final ThreadPool threadPool;
     private final String keyPath;
 
-    AzureBlobContainer(BlobPath path, AzureBlobStore blobStore, ThreadPool threadPool) {
+    AzureBlobContainer(BlobPath path, AzureBlobStore blobStore) {
         super(path);
         this.blobStore = blobStore;
         this.keyPath = path.buildAsString();
-        this.threadPool = threadPool;
     }
 
     private boolean blobExists(String blobName) {
@@ -112,7 +105,7 @@ public void writeBlobAtomic(String blobName, InputStream inputStream, long blobS
     @Override
     public DeleteResult delete() throws IOException {
         try {
-            return blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
+            return blobStore.deleteBlobDirectory(keyPath);
         } catch (URISyntaxException | StorageException e) {
             throw new IOException(e);
         }
@@ -120,33 +113,9 @@ public DeleteResult delete() throws IOException {
 
     @Override
     public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
-        final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
-        if (blobNames.isEmpty()) {
-            result.onResponse(null);
-        } else {
-            final GroupedActionListener<Void> listener =
-                new GroupedActionListener<>(ActionListener.map(result, v -> null), blobNames.size());
-            final ExecutorService executor = threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME);
-            // Executing deletes in parallel since Azure SDK 8 is using blocking IO while Azure does not provide a bulk delete API endpoint
-            // TODO: Upgrade to newer non-blocking Azure SDK 11 and execute delete requests in parallel that way.
-            for (String blobName : blobNames) {
-                executor.execute(ActionRunnable.run(listener, () -> {
-                    logger.trace("deleteBlob({})", blobName);
-                    try {
-                        blobStore.deleteBlob(buildKey(blobName));
-                    } catch (StorageException e) {
-                        if (e.getHttpStatusCode() != HttpURLConnection.HTTP_NOT_FOUND) {
-                            throw new IOException(e);
-                        }
-                    } catch (URISyntaxException e) {
-                        throw new IOException(e);
-                    }
-                }));
-            }
-        }
         try {
-            result.actionGet();
-        } catch (Exception e) {
+            blobStore.deleteBlobsIgnoringIfNotExists(blobNames.stream().map(this::buildKey).collect(Collectors.toList()));
+        } catch (URISyntaxException | StorageException e) {
             throw new IOException("Exception during bulk delete", e);
         }
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
index 714e29edea29d..173f13d801ff3 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
@@ -28,14 +28,13 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.DeleteResult;
 import org.elasticsearch.repositories.azure.AzureRepository.Repository;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
-import java.util.concurrent.Executor;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -44,17 +43,15 @@
 public class AzureBlobStore implements BlobStore {
 
     private final AzureStorageService service;
-    private final ThreadPool threadPool;
 
     private final String clientName;
     private final String container;
     private final LocationMode locationMode;
 
-    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service, ThreadPool threadPool) {
+    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service) {
         this.container = Repository.CONTAINER_SETTING.get(metadata.settings());
         this.clientName = Repository.CLIENT_NAME.get(metadata.settings());
         this.service = service;
-        this.threadPool = threadPool;
         // locationMode is set per repository, not per client
         this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
         final Map<String, AzureStorageSettings> prevSettings = this.service.refreshAndClearCache(emptyMap());
@@ -80,7 +77,7 @@ public LocationMode getLocationMode() {
 
     @Override
     public BlobContainer blobContainer(BlobPath path) {
-        return new AzureBlobContainer(path, this, threadPool);
+        return new AzureBlobContainer(path, this);
     }
 
     @Override
@@ -91,13 +88,12 @@ public boolean blobExists(String blob) throws URISyntaxException, StorageExcepti
         return service.blobExists(clientName, container, blob);
     }
 
-    public void deleteBlob(String blob) throws URISyntaxException, StorageException, IOException {
-        service.deleteBlob(clientName, container, blob);
+    public void deleteBlobsIgnoringIfNotExists(Collection<String> blobs) throws URISyntaxException, StorageException {
+        service.deleteBlobsIgnoringIfNotExists(clientName, container, blobs);
     }
 
-    public DeleteResult deleteBlobDirectory(String path, Executor executor)
-            throws URISyntaxException, StorageException, IOException {
-        return service.deleteBlobDirectory(clientName, container, path, executor);
+    public DeleteResult deleteBlobDirectory(String path) throws URISyntaxException, StorageException, IOException {
+        return service.deleteBlobDirectory(clientName, container, path);
     }
 
     public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException {
@@ -111,7 +107,7 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String keyPath, String prefix
 
     public Map<String, BlobContainer> children(BlobPath path) throws URISyntaxException, StorageException, IOException {
         return Collections.unmodifiableMap(service.children(clientName, container, path).stream().collect(
-            Collectors.toMap(Function.identity(), name -> new AzureBlobContainer(path.add(name), this, threadPool))));
+            Collectors.toMap(Function.identity(), name -> new AzureBlobContainer(path.add(name), this))));
     }
 
     public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists)
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 7b7a9108c1ef5..a6a393bbb47c6 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -115,7 +115,7 @@ protected BlobStore getBlobStore() {
 
     @Override
     protected AzureBlobStore createBlobStore() {
-        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, threadPool);
+        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService);
 
         logger.debug(() -> new ParameterizedMessage(
             "using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
index d98a7c3cbd717..ae1258a73b4f5 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
@@ -23,16 +23,12 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.Repository;
-import org.elasticsearch.threadpool.ExecutorBuilder;
-import org.elasticsearch.threadpool.ScalingExecutorBuilder;
-
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -43,8 +39,6 @@
  */
 public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin {
 
-    public static final String REPOSITORY_THREAD_POOL_NAME = "repository_azure";
-
     // protected for testing
     final AzureStorageService azureStoreService;
 
@@ -80,15 +74,6 @@ public List<Setting<?>> getSettings() {
         );
     }
 
-    @Override
-    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
-        return Collections.singletonList(executorBuilder());
-    }
-
-    public static ExecutorBuilder<?> executorBuilder() {
-        return new ScalingExecutorBuilder(REPOSITORY_THREAD_POOL_NAME, 0, 32, TimeValue.timeValueSeconds(30L));
-    }
-
     @Override
     public void reload(Settings settings) {
         // secure settings should be readable
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 26ade5bdec624..c15af49578386 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -20,13 +20,16 @@
 package org.elasticsearch.repositories.azure;
 
 import com.microsoft.azure.storage.AccessCondition;
+import com.microsoft.azure.storage.BatchException;
 import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.Constants;
 import com.microsoft.azure.storage.OperationContext;
 import com.microsoft.azure.storage.RetryExponentialRetry;
 import com.microsoft.azure.storage.RetryPolicy;
 import com.microsoft.azure.storage.RetryPolicyFactory;
 import com.microsoft.azure.storage.StorageErrorCodeStrings;
 import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobDeleteBatchOperation;
 import com.microsoft.azure.storage.blob.BlobInputStream;
 import com.microsoft.azure.storage.blob.BlobListingDetails;
 import com.microsoft.azure.storage.blob.BlobProperties;
@@ -41,7 +44,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.DeleteResult;
@@ -51,7 +53,6 @@
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -62,13 +63,13 @@
 import java.security.InvalidKeyException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 
@@ -187,72 +188,61 @@ public boolean blobExists(String account, String container, String blob) throws
         });
     }
 
-    public void deleteBlob(String account, String container, String blob) throws URISyntaxException, StorageException {
+    public void deleteBlobsIgnoringIfNotExists(String account, String container, Collection<String> blobs)
+            throws URISyntaxException, StorageException {
+        logger.trace(() -> new ParameterizedMessage("delete blobs for container [{}], blob [{}]", container, blobs));
         final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
         // Container name must be lower case.
         final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
-        logger.trace(() -> new ParameterizedMessage("delete blob for container [{}], blob [{}]", container, blob));
-        SocketAccess.doPrivilegedVoidException(() -> {
-            final CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blob);
-            logger.trace(() -> new ParameterizedMessage("container [{}]: blob [{}] found. removing.", container, blob));
-            azureBlob.delete(DeleteSnapshotsOption.NONE, null, null, client.v2().get());
-        });
+        final Iterator<String> blobIterator = blobs.iterator();
+        int currentBatchSize = 0;
+        while (blobIterator.hasNext()) {
+            final BlobDeleteBatchOperation batchDeleteOp = new BlobDeleteBatchOperation();
+            do {
+                batchDeleteOp.addSubOperation(blobContainer.getBlockBlobReference(blobIterator.next()),
+                    DeleteSnapshotsOption.NONE, null, null);
+                ++currentBatchSize;
+            } while (blobIterator.hasNext() && currentBatchSize < Constants.BATCH_MAX_REQUESTS);
+            currentBatchSize = 0;
+            try {
+                SocketAccess.doPrivilegedVoidException(() -> blobContainer.getServiceClient().executeBatch(batchDeleteOp));
+            } catch (BatchException e) {
+                for (StorageException ex : e.getExceptions().values()) {
+                    if (ex.getHttpStatusCode() != HttpURLConnection.HTTP_NOT_FOUND) {
+                        logger.error("Batch exceptions [{}]", e.getExceptions());
+                        throw e;
+                    }
+                }
+            }
+        }
     }
 
-    DeleteResult deleteBlobDirectory(String account, String container, String path, Executor executor)
+    DeleteResult deleteBlobDirectory(String account, String container, String path)
             throws URISyntaxException, StorageException, IOException {
         final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
         final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
-        final Collection<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
-        final AtomicLong outstanding = new AtomicLong(1L);
-        final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
         final AtomicLong blobsDeleted = new AtomicLong();
         final AtomicLong bytesDeleted = new AtomicLong();
+        final List<String> blobsToDelete = new ArrayList<>();
         SocketAccess.doPrivilegedVoidException(() -> {
-            for (final ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
+            for (ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
                 // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
                 // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
                 final String blobPath = blobItem.getUri().getPath().substring(1 + container.length() + 1);
-                outstanding.incrementAndGet();
-                executor.execute(new AbstractRunnable() {
-                    @Override
-                    protected void doRun() throws Exception {
-                        final long len;
-                        if (blobItem instanceof CloudBlob) {
-                            len = ((CloudBlob) blobItem).getProperties().getLength();
-                        } else {
-                            len = -1L;
-                        }
-                        deleteBlob(account, container, blobPath);
-                        blobsDeleted.incrementAndGet();
-                        if (len >= 0) {
-                            bytesDeleted.addAndGet(len);
-                        }
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        exceptions.add(e);
-                    }
-
-                    @Override
-                    public void onAfter() {
-                        if (outstanding.decrementAndGet() == 0) {
-                            result.onResponse(null);
-                        }
-                    }
-                });
+                final long len;
+                if (blobItem instanceof CloudBlob) {
+                    len = ((CloudBlob) blobItem).getProperties().getLength();
+                } else {
+                    len = -1L;
+                }
+                blobsToDelete.add(blobPath);
+                blobsDeleted.incrementAndGet();
+                if (len >= 0) {
+                    bytesDeleted.addAndGet(len);
+                }
             }
         });
-        if (outstanding.decrementAndGet() == 0) {
-            result.onResponse(null);
-        }
-        result.actionGet();
-        if (exceptions.isEmpty() == false) {
-            final IOException ex = new IOException("Deleting directory [" + path + "] failed");
-            exceptions.forEach(ex::addSuppressed);
-            throw ex;
-        }
+        deleteBlobsIgnoringIfNotExists(account, container, blobsToDelete);
         return new DeleteResult(blobsDeleted.get(), bytesDeleted.get());
     }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
index ce3cba065c35b..b8ec6115b4d17 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
@@ -44,8 +44,6 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
 
@@ -64,7 +62,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
@@ -91,11 +88,9 @@
 public class AzureBlobContainerRetriesTests extends ESTestCase {
 
     private HttpServer httpServer;
-    private ThreadPool threadPool;
 
     @Before
     public void setUp() throws Exception {
-        threadPool = new TestThreadPool(getTestClass().getName(), AzureRepositoryPlugin.executorBuilder());
         httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.start();
         super.setUp();
@@ -105,7 +100,6 @@ public void setUp() throws Exception {
     public void tearDown() throws Exception {
         httpServer.stop(0);
         super.tearDown();
-        ThreadPool.terminate(threadPool, 10L, TimeUnit.SECONDS);
     }
 
     private BlobContainer createBlobContainer(final int maxRetries) {
@@ -145,7 +139,7 @@ BlobRequestOptions getBlobRequestOptionsForWriteBlob() {
                 .put(ACCOUNT_SETTING.getKey(), clientName)
                 .build());
 
-        return new AzureBlobContainer(BlobPath.cleanPath(), new AzureBlobStore(repositoryMetaData, service, threadPool), threadPool);
+        return new AzureBlobContainer(BlobPath.cleanPath(), new AzureBlobStore(repositoryMetaData, service));
     }
 
     public void testReadNonexistentBlobThrowsNoSuchFileException() {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
index b23693fd268d4..47703b90e43cc 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -64,7 +64,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     @Override
     protected Map<String, HttpHandler> createHttpHandlers() {
-        return Collections.singletonMap("/container", new AzureBlobStoreHttpHandler("container"));
+        return Collections.singletonMap("/", new AzureBlobStoreHttpHandler("container"));
     }
 
     @Override
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
index 1def1439429e6..5498c4ebc033f 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
@@ -30,7 +30,7 @@ public class AzureHttpFixture {
 
     private AzureHttpFixture(final String address, final int port, final String container) throws IOException {
         this.server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(address), port), 0);
-        server.createContext("/" + container, new AzureHttpHandler(container));
+        server.createContext("/", new AzureHttpHandler(container));
     }
 
     private void start() throws Exception {
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
index 7a94a8c9f2e57..6cd93fc9347fa 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
@@ -22,16 +22,21 @@
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
 
+import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -67,18 +72,21 @@ public void handle(final HttpExchange exchange) throws IOException {
             assert read == -1 : "Request body should have been empty but saw [" + read + "]";
         }
         try {
+            // Request body is closed in the finally block
+            final BytesReference requestBody = Streams.readFully(Streams.noCloseStream(exchange.getRequestBody()));
             if (Regex.simpleMatch("PUT /" + container + "/*blockid=*", request)) {
                 // Put Block (https://docs.microsoft.com/en-us/rest/api/storageservices/put-block)
                 final Map<String, String> params = new HashMap<>();
                 RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params);
 
                 final String blockId = params.get("blockid");
-                blobs.put(blockId, Streams.readFully(exchange.getRequestBody()));
+                blobs.put(blockId, requestBody);
                 exchange.sendResponseHeaders(RestStatus.CREATED.getStatus(), -1);
 
             } else if (Regex.simpleMatch("PUT /" + container + "/*comp=blocklist*", request)) {
                 // Put Block List (https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list)
-                final String blockList = Streams.copyToString(new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8));
+                final String blockList =
+                    Streams.copyToString(new InputStreamReader(requestBody.streamInput(), StandardCharsets.UTF_8));
                 final List<String> blockIds = Arrays.stream(blockList.split("<Latest>"))
                     .filter(line -> line.contains("</Latest>"))
                     .map(line -> line.substring(0, line.indexOf("</Latest>")))
@@ -97,12 +105,12 @@ public void handle(final HttpExchange exchange) throws IOException {
                 // PUT Blob (see https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob)
                 final String ifNoneMatch = exchange.getRequestHeaders().getFirst("If-None-Match");
                 if ("*".equals(ifNoneMatch)) {
-                    if (blobs.putIfAbsent(exchange.getRequestURI().getPath(), Streams.readFully(exchange.getRequestBody())) != null) {
+                    if (blobs.putIfAbsent(exchange.getRequestURI().getPath(), requestBody) != null) {
                         sendError(exchange, RestStatus.CONFLICT);
                         return;
                     }
                 } else {
-                    blobs.put(exchange.getRequestURI().getPath(), Streams.readFully(exchange.getRequestBody()));
+                    blobs.put(exchange.getRequestURI().getPath(), requestBody);
                 }
                 exchange.sendResponseHeaders(RestStatus.CREATED.getStatus(), -1);
 
@@ -190,6 +198,45 @@ public void handle(final HttpExchange exchange) throws IOException {
                 exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
                 exchange.getResponseBody().write(response);
 
+            } else if (Regex.simpleMatch("POST /?comp=batch", request)) {
+                // Batch Delete (https://docs.microsoft.com/en-us/rest/api/storageservices/blob-batch)
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(requestBody.streamInput()))) {
+                    final Set<String> toDelete = reader.lines().filter(l -> l.startsWith("DELETE"))
+                        .map(l -> l.split(" ")[1]).collect(Collectors.toSet());
+                    final BytesStreamOutput baos = new BytesStreamOutput();
+                    final String batchSeparator = "batchresponse_" + UUIDs.randomBase64UUID();
+                    try (Writer writer = new OutputStreamWriter(baos)) {
+                        int contentId = 0;
+                        for (String b : toDelete) {
+                            writer.write("\r\n--" + batchSeparator + "\r\n" +
+                                "Content-Type: application/http \r\n" +
+                                "Content-ID: " + contentId++ + " \r\n");
+                            if (blobs.remove(b) == null) {
+                                writer.write("\r\nHTTP/1.1 404 The specified blob does not exist. \r\n" +
+                                    "x-ms-error-code: BlobNotFound \r\n" +
+                                    "x-ms-request-id: " + UUIDs.randomBase64UUID() + " \r\n" +
+                                    "x-ms-version: 2018-11-09\r\n" +
+                                    "Content-Length: 216 \r\n" +
+                                    "Content-Type: application/xml\r\n\r\n" +
+                                    "<?xml version=\"1.0\" encoding=\"utf-8\"?> \r\n" +
+                                    "<Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.\r\n" +
+                                    "RequestId:" + UUIDs.randomBase64UUID() + "\r\n" +
+                                    "Time:2020-01-01T01:01:01.0000000Z</Message></Error>\r\n");
+                            } else {
+                                writer.write(
+                                    "\r\nHTTP/1.1 202 Accepted \r\n" +
+                                        "x-ms-request-id: " + UUIDs.randomBase64UUID() + " \r\n" +
+                                        "x-ms-version: 2018-11-09\r\n\r\n");
+                            }
+                        }
+                        writer.write("--" + batchSeparator + "--");
+                    }
+                    final Headers headers = exchange.getResponseHeaders();
+                    headers.add("Content-Type",
+                        "multipart/mixed; boundary=" + batchSeparator);
+                    exchange.sendResponseHeaders(RestStatus.ACCEPTED.getStatus(), baos.size());
+                    baos.bytes().writeTo(exchange.getResponseBody());
+                }
             } else {
                 sendError(exchange, RestStatus.BAD_REQUEST);
             }

From 1df73828ededae3f588f4b64db8ac5fed65d3b52 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Mon, 23 Mar 2020 07:36:37 -0400
Subject: [PATCH 199/285] [DOCS] Note doc links should be live in REST API JSON
 specs (#53871)

Downstream Elasticsearch clients, such as the Elaticsearch-JS client,
use the documentation links in our REST API JSON specifications to
create their docs.

Using a broken link or linking to yet-to-be-created doc pages can
break the docs build for these clients.

This PR adds a related note to the README for the REST API JSON Specs.
---
 rest-api-spec/README.markdown | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/rest-api-spec/README.markdown b/rest-api-spec/README.markdown
index 15b3c14941909..e2c3d37f11da3 100644
--- a/rest-api-spec/README.markdown
+++ b/rest-api-spec/README.markdown
@@ -43,7 +43,12 @@ Example for the ["Create Index"](https://www.elastic.co/guide/en/elasticsearch/r
 The specification contains:
 
 * The _name_ of the API (`indices.create`), which usually corresponds to the client calls
-* Link to the documentation at the <http://elastic.co> website
+* Link to the documentation at the <http://elastic.co> website.
+
+  **IMPORANT:** This should be a _live_ link. Several downstream ES clients use
+  this link to generate their documentation. Using a broken link or linking to
+  yet-to-be-created doc pages can break the [Elastic docs
+  build](https://github.com/elastic/docs#building-documentation).
 * `stability` indicating the state of the API, has to be declared explicitly or YAML tests will fail
     * `experimental` highly likely to break in the near future (minor/patch), no bwc guarantees.
     Possibly removed in the future.

From 2dc79505825fa75e0711dcfa8e9c69e8028fc979 Mon Sep 17 00:00:00 2001
From: musteaf <gs_mustea@hotmail.com>
Date: Mon, 23 Mar 2020 14:52:25 +0300
Subject: [PATCH 200/285] SQL: Extend DATE_TRUNC to also operate on
 intervals(elastic - #46632 ) (#47720)

The function is extended to operate on intervals according to the PostgreSQL: https://www.postgresql.org/docs/9.1/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC

Closes : #46632
---
 .../sql/functions/date-time.asciidoc          |  26 +-
 .../xpack/ql/util/DateUtils.java              |  10 +-
 .../qa/src/main/resources/datetime.csv-spec   |  71 +++++
 .../qa/src/main/resources/docs/docs.csv-spec  |  30 ++
 .../sql/expression/SqlTypeResolutions.java    |   6 +-
 .../datetime/BinaryDateTimeFunction.java      |   6 +-
 .../function/scalar/datetime/DatePart.java    |  16 ++
 .../function/scalar/datetime/DateTrunc.java   | 262 +++++++++++++-----
 .../scalar/datetime/DateTruncProcessor.java   |  24 +-
 .../whitelist/InternalSqlScriptUtils.java     |   7 +-
 .../xpack/sql/type/SqlDataTypes.java          |   5 +-
 .../xpack/sql/plugin/sql_whitelist.txt        |   2 +-
 .../analyzer/VerifierErrorMessagesTests.java  |   8 +-
 .../datetime/DateTruncProcessorTests.java     | 161 ++++++++++-
 14 files changed, 536 insertions(+), 98 deletions(-)

diff --git a/docs/reference/sql/functions/date-time.asciidoc b/docs/reference/sql/functions/date-time.asciidoc
index 6a9b0c6f8a8e3..b0be286f39c0a 100644
--- a/docs/reference/sql/functions/date-time.asciidoc
+++ b/docs/reference/sql/functions/date-time.asciidoc
@@ -500,18 +500,19 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[datePartDateTimeTzOffsetMinus]
 --------------------------------------------------
 DATE_TRUNC(
     string_exp, <1>
-    datetime_exp) <2>
+    datetime_exp/interval_exp) <2>
 --------------------------------------------------
 
 *Input*:
 
-<1> string expression denoting the unit to which the date/datetime should be truncated to
-<2> date/datetime expression
+<1> string expression denoting the unit to which the date/datetime/interval should be truncated to
+<2> date/datetime/interval expression
 
-*Output*: datetime
+*Output*: datetime/interval
 
-*Description*: Truncate the date/datetime to the specified unit by setting all fields that are less significant than the specified
+*Description*: Truncate the date/datetime/interval to the specified unit by setting all fields that are less significant than the specified
 one to zero (or one, for day, day of week and month). If any of the two arguments is `null` a `null` is returned.
+If the first argument is `week` and the second argument is of `interval` type, an error is thrown since the `interval` data type doesn't support a `week` time unit.
 
 [cols="^,^"]
 |===
@@ -563,6 +564,21 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[truncateDateDecades]
 include-tagged::{sql-specs}/docs/docs.csv-spec[truncateDateQuarter]
 --------------------------------------------------
 
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[truncateIntervalCenturies]
+--------------------------------------------------
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[truncateIntervalHour]
+--------------------------------------------------
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[truncateIntervalDay]
+--------------------------------------------------
+
 [[sql-functions-datetime-day]]
 ==== `DAY_OF_MONTH/DOM/DAY`
 
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/DateUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/DateUtils.java
index 583d8de43f32d..a568e4039755f 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/DateUtils.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/DateUtils.java
@@ -34,7 +34,7 @@ public class DateUtils {
     public static final ZoneId UTC = ZoneId.of("Z");
 
     public static final String EMPTY = "";
-    
+
     public static final DateTimeFormatter ISO_DATE_WITH_MILLIS = new DateTimeFormatterBuilder()
             .parseCaseInsensitive()
             .append(ISO_LOCAL_DATE)
@@ -72,9 +72,9 @@ public class DateUtils {
             .appendOffsetId()
             .toFormatter(Locale.ROOT);
 
-    private static final int SECONDS_PER_MINUTE = 60;
-    private static final int SECONDS_PER_HOUR = SECONDS_PER_MINUTE * 60;
-    private static final int SECONDS_PER_DAY = SECONDS_PER_HOUR * 24;
+    public static final int SECONDS_PER_MINUTE = 60;
+    public static final int SECONDS_PER_HOUR = SECONDS_PER_MINUTE * 60;
+    public static final int SECONDS_PER_DAY = SECONDS_PER_HOUR * 24;
 
     private DateUtils() {}
 
@@ -82,7 +82,7 @@ public static String toString(Object value) {
         if (value == null) {
             return "null";
         }
-        
+
         if (value instanceof ZonedDateTime) {
             return ((ZonedDateTime) value).format(ISO_DATE_WITH_MILLIS);
         }
diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
index 16550c3e9144e..1dfc1497ebc76 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
@@ -499,6 +499,16 @@ DATE_TRUNC('week', '2019-09-04'::date) as dt_week,  DATE_TRUNC('day', '2019-09-0
 2000-01-01T00:00:00.000Z | 2000-01-01T00:00:00.000Z | 2010-01-01T00:00:00.000Z | 2019-01-01T00:00:00.000Z | 2019-07-01T00:00:00.000Z | 2019-09-01T00:00:00.000Z | 2019-09-02T00:00:00.000Z | 2019-09-04T00:00:00.000Z
 ;
 
+selectDateTruncWithInterval
+SELECT DATE_TRUNC('hour', INTERVAL '1 12:43:21' DAY TO SECONDS) as dt_hour, DATE_TRUNC('minute', INTERVAL '1 12:43:21' DAY TO SECONDS) as dt_min,
+DATE_TRUNC('seconds', INTERVAL '1 12:43:21' DAY TO SECONDS) as dt_sec, DATE_TRUNC('ms', INTERVAL '1 12:43:21' DAY TO SECONDS)::string as dt_millis,
+DATE_TRUNC('mcs', INTERVAL '1 12:43:21' DAY TO SECONDS)::string as dt_micro, DATE_TRUNC('nanoseconds', INTERVAL '1 12:43:21' DAY TO SECONDS)::string as dt_nano;
+
+    dt_hour    |    dt_min     |    dt_sec     |   dt_millis   |   dt_micro    |    dt_nano
+---------------+---------------+---------------+---------------+---------------+---------------
++1 12:00:00    |+1 12:43:00    |+1 12:43:21    |+1 12:43:21    |+1 12:43:21    |+1 12:43:21
+;
+
 selectDateTruncWithField
 schema::emp_no:i|birth_date:ts|dt_mil:ts|dt_cent:ts|dt_dec:ts|dt_year:ts|dt_quarter:ts|dt_month:ts|dt_week:ts|dt_day:ts
 SELECT emp_no, birth_date, DATE_TRUNC('millennium', birth_date) as dt_mil, DATE_TRUNC('centuries', birth_date) as dt_cent,
@@ -585,6 +595,21 @@ SELECT emp_no, hire_date, DATE_TRUNC('quarter', hire_date) as dt FROM test_emp O
 10076   | 1985-07-09 00:00:00.000Z | 1985-07-01 00:00:00.000Z
 ;
 
+dateTruncOrderByWithInterval
+schema::first_name:s|dt:ts|hire_date:ts|languages:byte
+SELECT first_name, hire_date + DATE_TRUNC('centuries', CASE WHEN languages = 5 THEN INTERVAL '18-3' YEAR TO MONTH
+WHEN languages = 4 THEN INTERVAL '108-4' YEAR TO MONTH WHEN languages = 3 THEN INTERVAL '212-3' YEAR TO MONTH
+ELSE INTERVAL '318-6' YEAR TO MONTH END) as dt, hire_date, languages FROM test_emp WHERE emp_no <= 10006 ORDER BY dt NULLS LAST LIMIT 5;
+
+   first_name |            dt            |        hire_date         | languages
+--------------+--------------------------+--------------------------+-----------
+Bezalel       | 1985-11-21 00:00:00.000Z | 1985-11-21T00:00:00.000Z | 5
+Chirstian     | 1986-12-01 00:00:00.000Z | 1986-12-01T00:00:00.000Z | 5
+Parto         | 2086-08-28 00:00:00.000Z | 1986-08-28T00:00:00.000Z | 4
+Anneke        | 2189-06-02 00:00:00.000Z | 1989-06-02T00:00:00.000Z | 3
+Georgi        | 2286-06-26 00:00:00.000Z | 1986-06-26T00:00:00.000Z | 2
+;
+
 dateTruncFilter
 schema::emp_no:i|hire_date:ts|dt:ts
 SELECT emp_no, hire_date, DATE_TRUNC('quarter', hire_date) as dt FROM test_emp WHERE DATE_TRUNC('quarter', hire_date) > '1994-07-01T00:00:00.000Z'::timestamp ORDER BY emp_no;
@@ -601,6 +626,24 @@ SELECT emp_no, hire_date, DATE_TRUNC('quarter', hire_date) as dt FROM test_emp W
 10093   | 1996-11-05 00:00:00.000Z | 1996-10-01 00:00:00.000Z
 ;
 
+dateTruncFilterWithInterval
+schema::first_name:s|hire_date:ts
+SELECT first_name, hire_date FROM test_emp WHERE hire_date > '2090-03-05T10:11:22.123Z'::datetime - DATE_TRUNC('centuries', INTERVAL 190 YEARS) ORDER BY first_name DESC, hire_date ASC LIMIT 10;
+
+  first_name   |          hire_date
+---------------+-------------------------
+null           | 1990-06-20 00:00:00.000Z
+null           | 1990-12-05 00:00:00.000Z
+null           | 1991-09-01 00:00:00.000Z
+null           | 1992-01-03 00:00:00.000Z
+null           | 1994-02-17 00:00:00.000Z
+Yongqiao       | 1995-03-20 00:00:00.000Z
+Yishay         | 1990-10-20 00:00:00.000Z
+Yinghua        | 1990-12-25 00:00:00.000Z
+Weiyi          | 1993-02-14 00:00:00.000Z
+Tuval          | 1995-12-15 00:00:00.000Z
+;
+
 dateTruncGroupBy
 schema::count:l|dt:ts
 SELECT count(*) as count, DATE_TRUNC('decade', hire_date) dt FROM test_emp GROUP BY dt ORDER BY 2;
@@ -611,6 +654,24 @@ SELECT count(*) as count, DATE_TRUNC('decade', hire_date) dt FROM test_emp GROUP
 41      | 1990-01-01 00:00:00.000Z
 ;
 
+dateTruncGroupByWithInterval
+schema::count:l|dt:ts
+SELECT count(*) as count, birth_date + DATE_TRUNC('hour', INTERVAL '1 12:43:21' DAY TO SECONDS) dt FROM test_emp GROUP BY dt ORDER BY 2 LIMIT 10;
+
+ count  |          dt
+--------+-------------------------
+10      | null
+1       | 1952-02-28 12:00:00.000Z
+1       | 1952-04-20 12:00:00.000Z
+1       | 1952-05-16 12:00:00.000Z
+1       | 1952-06-14 12:00:00.000Z
+1       | 1952-07-09 12:00:00.000Z
+1       | 1952-08-07 12:00:00.000Z
+1       | 1952-11-14 12:00:00.000Z
+1       | 1952-12-25 12:00:00.000Z
+1       | 1953-01-08 12:00:00.000Z
+;
+
 dateTruncHaving
 schema::gender:s|dt:ts
 SELECT gender, max(hire_date) AS dt FROM test_emp GROUP BY gender HAVING DATE_TRUNC('year', max(hire_date)) >= '1997-01-01T00:00:00.000Z'::timestamp ORDER BY 1;
@@ -621,6 +682,16 @@ null    | 1999-04-30 00:00:00.000Z
 F       | 1997-05-19 00:00:00.000Z
 ;
 
+// Awaits fix: https://github.com/elastic/elasticsearch/issues/53565
+dateTruncHavingWithInterval-Ignore
+schema::gender:s|dt:ts
+SELECT gender, max(hire_date) AS dt FROM test_emp GROUP BY gender HAVING max(hire_date) - DATE_TRUNC('hour', INTERVAL 1 YEARS) >= '1997-01-01T00:00:00.000Z'::timestamp ORDER BY 1;
+
+ gender |         dt
+--------+-------------------------
+null    | 1999-04-30 00:00:00.000Z
+;
+
 selectDatePartWithDate
 SELECT DATE_PART('year', '2019-09-04'::date) as dp_years, DATE_PART('quarter', '2019-09-04'::date) as dp_quarter, DATE_PART('month', '2019-09-04'::date) as dp_month,
 DATE_PART('dayofyear', '2019-09-04'::date) as dp_doy, DATE_PART('day', '2019-09-04'::date) as dp_day, DATE_PART('week', '2019-09-04'::date) as dp_week,
diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
index 8f92e1ce481ac..daa8aa573436b 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
@@ -2675,6 +2675,36 @@ SELECT DATETRUNC('quarters', CAST('2019-09-04' AS DATE)) AS quarter;
 // end::truncateDateQuarter
 ;
 
+truncateIntervalCenturies
+// tag::truncateIntervalCenturies
+SELECT DATE_TRUNC('centuries', INTERVAL '199-5' YEAR TO MONTH) AS centuries;
+
+      centuries
+------------------
+ +100-0
+// end::truncateIntervalCenturies
+;
+
+truncateIntervalHour
+// tag::truncateIntervalHour
+SELECT DATE_TRUNC('hours', INTERVAL '17 22:13:12' DAY TO SECONDS) AS hour;
+
+      hour
+------------------
++17 22:00:00
+// end::truncateIntervalHour
+;
+
+truncateIntervalDay
+// tag::truncateIntervalDay
+SELECT DATE_TRUNC('days', INTERVAL '19 15:24:19' DAY TO SECONDS) AS day;
+
+      day
+------------------
++19 00:00:00
+// end::truncateIntervalDay
+;
+
 constantDayOfWeek
 // tag::dayOfWeek
 SELECT DAY_OF_WEEK(CAST('2018-02-19T10:23:27Z' AS TIMESTAMP)) AS day;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/SqlTypeResolutions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/SqlTypeResolutions.java
index 1d5af2446fb05..63dd6ce2f756e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/SqlTypeResolutions.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/SqlTypeResolutions.java
@@ -16,7 +16,7 @@
 public final class SqlTypeResolutions {
 
     private SqlTypeResolutions() {}
-    
+
     public static TypeResolution isDate(Expression e, String operationName, ParamOrdinal paramOrd) {
         return isType(e, SqlDataTypes::isDateBased, operationName, paramOrd, "date", "datetime");
     }
@@ -25,6 +25,10 @@ public static TypeResolution isDateOrTime(Expression e, String operationName, Pa
         return isType(e, SqlDataTypes::isDateOrTimeBased, operationName, paramOrd, "date", "time", "datetime");
     }
 
+    public static TypeResolution isDateOrInterval(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, SqlDataTypes::isDateOrIntervalBased, operationName, paramOrd, "date", "datetime", "an interval data type");
+    }
+
     public static TypeResolution isNumericOrDate(Expression e, String operationName, ParamOrdinal paramOrd) {
         return isType(e, dt -> dt.isNumeric() || SqlDataTypes.isDateBased(dt), operationName, paramOrd,
             "date", "datetime", "numeric");
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BinaryDateTimeFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BinaryDateTimeFunction.java
index c6855eccee8cd..8c6e4b22360db 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BinaryDateTimeFunction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BinaryDateTimeFunction.java
@@ -19,7 +19,6 @@
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString;
 import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
-import static org.elasticsearch.xpack.sql.expression.SqlTypeResolutions.isDate;
 
 public abstract class BinaryDateTimeFunction extends BinaryScalarFunction {
 
@@ -54,10 +53,7 @@ protected TypeResolution resolveType() {
                 }
             }
         }
-        resolution = isDate(right(), sourceText(), Expressions.ParamOrdinal.SECOND);
-        if (resolution.unresolved()) {
-            return resolution;
-        }
+
         return TypeResolution.TYPE_RESOLVED;
     }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DatePart.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DatePart.java
index 2349dc9dd9c9b..568adfac427de 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DatePart.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DatePart.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
@@ -23,6 +24,8 @@
 import java.util.Set;
 import java.util.function.ToIntFunction;
 
+import static org.elasticsearch.xpack.sql.expression.SqlTypeResolutions.isDate;
+
 public class DatePart extends BinaryDateTimeFunction {
 
     public enum Part implements DateTimeField {
@@ -84,6 +87,19 @@ public DataType dataType() {
         return DataTypes.INTEGER;
     }
 
+    @Override
+    protected TypeResolution resolveType() {
+        TypeResolution resolution = super.resolveType();
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        resolution = isDate(right(), sourceText(), Expressions.ParamOrdinal.SECOND);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
     @Override
     protected BinaryScalarFunction replaceChildren(Expression newDateTimePart, Expression newTimestamp) {
         return new DatePart(source(), newDateTimePart, newTimestamp, zoneId());
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTrunc.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTrunc.java
index 9b336ddaed0e7..de033bfce6dac 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTrunc.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTrunc.java
@@ -6,96 +6,158 @@
 package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
-
+import org.elasticsearch.xpack.sql.expression.literal.interval.IntervalDayTime;
+import org.elasticsearch.xpack.sql.expression.literal.interval.IntervalYearMonth;
+import java.time.Duration;
+import java.time.Period;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoField;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.function.UnaryOperator;
 
+import static org.elasticsearch.xpack.ql.util.DateUtils.SECONDS_PER_DAY;
+import static org.elasticsearch.xpack.ql.util.DateUtils.SECONDS_PER_HOUR;
+import static org.elasticsearch.xpack.ql.util.DateUtils.SECONDS_PER_MINUTE;
+import static org.elasticsearch.xpack.sql.expression.SqlTypeResolutions.isDateOrInterval;
+import static org.elasticsearch.xpack.sql.type.SqlDataTypes.isInterval;
+
 public class DateTrunc extends BinaryDateTimeFunction {
 
     public enum Part implements DateTimeField {
 
         MILLENNIUM(dt -> {
-            int year = dt.getYear();
-            int firstYearOfMillenium = year - (year % 1000);
-            return dt
-                .with(ChronoField.YEAR, firstYearOfMillenium)
-                .with(ChronoField.MONTH_OF_YEAR, 1)
-                .with(ChronoField.DAY_OF_MONTH, 1)
-                .toLocalDate().atStartOfDay(dt.getZone());
-        },"millennia"),
+                int year = dt.getYear();
+                int firstYearOfMillennium = year - (year % 1000);
+                return dt
+                    .with(ChronoField.YEAR, firstYearOfMillennium)
+                    .with(ChronoField.MONTH_OF_YEAR, 1)
+                    .with(ChronoField.DAY_OF_MONTH, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, idt.dataType()),
+            iym -> {
+                Period period = iym.interval();
+                int year = period.getYears();
+                int firstYearOfMillennium = year - (year % 1000);
+                return new IntervalYearMonth(Period.ZERO.plusYears(firstYearOfMillennium), iym.dataType());
+            }, "millennia"),
         CENTURY(dt -> {
-            int year = dt.getYear();
-            int firstYearOfCentury = year - (year % 100);
-            return dt
-                .with(ChronoField.YEAR, firstYearOfCentury)
-                .with(ChronoField.MONTH_OF_YEAR, 1)
-                .with(ChronoField.DAY_OF_MONTH, 1)
-                .toLocalDate().atStartOfDay(dt.getZone());
-        }, "centuries"),
+                int year = dt.getYear();
+                int firstYearOfCentury = year - (year % 100);
+                return dt
+                    .with(ChronoField.YEAR, firstYearOfCentury)
+                    .with(ChronoField.MONTH_OF_YEAR, 1)
+                    .with(ChronoField.DAY_OF_MONTH, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, idt.dataType()),
+            iym -> {
+                Period period = iym.interval();
+                int year = period.getYears();
+                int firstYearOfCentury = year - (year % 100);
+                return new IntervalYearMonth(Period.ZERO.plusYears(firstYearOfCentury), iym.dataType());
+            }, "centuries"),
         DECADE(dt -> {
-            int year = dt.getYear();
-            int firstYearOfDecade = year - (year % 10);
-            return dt
-                .with(ChronoField.YEAR, firstYearOfDecade)
-                .with(ChronoField.MONTH_OF_YEAR, 1)
-                .with(ChronoField.DAY_OF_MONTH, 1)
-                .toLocalDate().atStartOfDay(dt.getZone());
-        }, "decades"),
-        YEAR(dt -> dt
-            .with(ChronoField.MONTH_OF_YEAR, 1)
-            .with(ChronoField.DAY_OF_MONTH, 1)
-            .toLocalDate().atStartOfDay(dt.getZone()),
-            "years", "yy", "yyyy"),
+                int year = dt.getYear();
+                int firstYearOfDecade = year - (year % 10);
+                return dt
+                    .with(ChronoField.YEAR, firstYearOfDecade)
+                    .with(ChronoField.MONTH_OF_YEAR, 1)
+                    .with(ChronoField.DAY_OF_MONTH, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, idt.dataType()),
+            iym -> {
+                Period period = iym.interval();
+                int year = period.getYears();
+                int firstYearOfDecade = year - (year % 10);
+                return new IntervalYearMonth(Period.ZERO.plusYears(firstYearOfDecade), iym.dataType());
+            }, "decades"),
+        YEAR(dt -> {
+                return dt.with(ChronoField.MONTH_OF_YEAR, 1)
+                    .with(ChronoField.DAY_OF_MONTH, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, idt.dataType()),
+            iym -> {
+                Period period = iym.interval();
+                int year = period.getYears();
+                return new IntervalYearMonth(Period.ZERO.plusYears(year), iym.dataType());
+            }, "years", "yy", "yyyy"),
         QUARTER(dt -> {
-            int month = dt.getMonthValue();
-            int firstMonthOfQuarter = (((month - 1) / 3) * 3) + 1;
-            return dt
-                .with(ChronoField.MONTH_OF_YEAR, firstMonthOfQuarter)
-                .with(ChronoField.DAY_OF_MONTH, 1)
-                .toLocalDate().atStartOfDay(dt.getZone());
-        }, "quarters", "qq", "q"),
-        MONTH(dt -> dt
-            .with(ChronoField.DAY_OF_MONTH, 1)
-            .toLocalDate().atStartOfDay(dt.getZone()),
-            "months", "mm", "m"),
-        WEEK(dt -> dt
-            .with(ChronoField.DAY_OF_WEEK, 1)
-            .toLocalDate().atStartOfDay(dt.getZone()),
-            "weeks", "wk", "ww"),
-        DAY(dt -> dt.toLocalDate().atStartOfDay(dt.getZone()), "days", "dd", "d"),
+                int month = dt.getMonthValue();
+                int firstMonthOfQuarter = (((month - 1) / 3) * 3) + 1;
+                return dt
+                    .with(ChronoField.MONTH_OF_YEAR, firstMonthOfQuarter)
+                    .with(ChronoField.DAY_OF_MONTH, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, (idt.dataType())),
+            iym -> {
+                Period period = iym.interval();
+                int month = period.getMonths();
+                int year = period.getYears();
+                int firstMonthOfQuarter = (month / 3) * 3;
+                return new IntervalYearMonth(Period.ZERO.plusYears(year).plusMonths(firstMonthOfQuarter), iym.dataType());
+            }, "quarters", "qq", "q"),
+        MONTH(dt -> {
+                return dt.with(ChronoField.DAY_OF_MONTH, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, idt.dataType()),
+            iym -> iym, "months", "mm", "m"),
+        WEEK(dt -> {
+                return dt.with(ChronoField.DAY_OF_WEEK, 1)
+                    .toLocalDate().atStartOfDay(dt.getZone());
+            },
+            idt -> new IntervalDayTime(Duration.ZERO, idt.dataType()),
+            iym -> iym, "weeks", "wk", "ww"),
+        DAY(dt -> dt.toLocalDate().atStartOfDay(dt.getZone()),
+            idt -> truncateIntervalSmallerThanWeek(idt, ChronoUnit.DAYS),
+            iym -> iym, "days", "dd", "d"),
         HOUR(dt -> {
-            int hour = dt.getHour();
-            return dt.toLocalDate().atStartOfDay(dt.getZone())
-                .with(ChronoField.HOUR_OF_DAY, hour);
-        }, "hours", "hh"),
+                int hour = dt.getHour();
+                return dt.toLocalDate().atStartOfDay(dt.getZone())
+                    .with(ChronoField.HOUR_OF_DAY, hour);
+            },
+            idt -> truncateIntervalSmallerThanWeek(idt, ChronoUnit.HOURS),
+            iym -> iym, "hours", "hh"),
         MINUTE(dt -> {
-            int hour = dt.getHour();
-            int minute = dt.getMinute();
-            return dt.toLocalDate().atStartOfDay(dt.getZone())
-                .with(ChronoField.HOUR_OF_DAY, hour)
-                .with(ChronoField.MINUTE_OF_HOUR, minute);
-        }, "minutes", "mi", "n"),
-        SECOND(dt -> dt.with(ChronoField.NANO_OF_SECOND, 0), "seconds", "ss", "s"),
+                int hour = dt.getHour();
+                int minute = dt.getMinute();
+                return dt.toLocalDate().atStartOfDay(dt.getZone())
+                    .with(ChronoField.HOUR_OF_DAY, hour)
+                    .with(ChronoField.MINUTE_OF_HOUR, minute);
+            },
+            idt -> truncateIntervalSmallerThanWeek(idt, ChronoUnit.MINUTES),
+            iym -> iym, "minutes", "mi", "n"),
+            SECOND(dt -> dt.with(ChronoField.NANO_OF_SECOND, 0),
+            idt -> truncateIntervalSmallerThanWeek(idt, ChronoUnit.SECONDS),
+            iym -> iym, "seconds", "ss", "s"),
         MILLISECOND(dt -> {
-            int micros = dt.get(ChronoField.MICRO_OF_SECOND);
-            return dt.with(ChronoField.MILLI_OF_SECOND, (micros / 1000));
-        }, "milliseconds", "ms"),
+                int micros = dt.get(ChronoField.MICRO_OF_SECOND);
+                return dt.with(ChronoField.MILLI_OF_SECOND, (micros / 1000));
+            },
+            idt -> truncateIntervalSmallerThanWeek(idt, ChronoUnit.MILLIS),
+            iym -> iym, "milliseconds", "ms"),
         MICROSECOND(dt -> {
-            int nanos = dt.getNano();
-            return dt.with(ChronoField.MICRO_OF_SECOND, (nanos / 1000));
-        }, "microseconds", "mcs"),
-        NANOSECOND(dt -> dt, "nanoseconds", "ns");
+                int nanos = dt.getNano();
+                return dt.with(ChronoField.MICRO_OF_SECOND, (nanos / 1000));
+            },
+            idt -> idt, iym -> iym, "microseconds", "mcs"),
+        NANOSECOND(dt -> dt, idt -> idt, iym -> iym, "nanoseconds", "ns");
 
         private static final Map<String, Part> NAME_TO_PART;
         private static final List<String> VALID_VALUES;
@@ -105,11 +167,16 @@ public enum Part implements DateTimeField {
             VALID_VALUES = DateTimeField.initializeValidValues(values());
         }
 
-        private UnaryOperator<ZonedDateTime> truncateFunction;
+        private UnaryOperator<IntervalYearMonth> truncateFunctionIntervalYearMonth;
+        private UnaryOperator<ZonedDateTime> truncateFunctionZonedDateTime;
+        private UnaryOperator<IntervalDayTime> truncateFunctionIntervalDayTime;
         private Set<String> aliases;
 
-        Part(UnaryOperator<ZonedDateTime> truncateFunction, String... aliases) {
-            this.truncateFunction = truncateFunction;
+        Part(UnaryOperator<ZonedDateTime> truncateFunctionZonedDateTime, UnaryOperator<IntervalDayTime> truncateFunctionIntervalDayTime,
+             UnaryOperator<IntervalYearMonth> truncateFunctionIntervalYearMonth, String... aliases) {
+            this.truncateFunctionIntervalYearMonth = truncateFunctionIntervalYearMonth;
+            this.truncateFunctionZonedDateTime = truncateFunctionZonedDateTime;
+            this.truncateFunctionIntervalDayTime = truncateFunctionIntervalDayTime;
             this.aliases = Set.of(aliases);
         }
 
@@ -127,7 +194,50 @@ public static Part resolve(String truncateTo) {
         }
 
         public ZonedDateTime truncate(ZonedDateTime dateTime) {
-            return truncateFunction.apply(dateTime);
+            return truncateFunctionZonedDateTime.apply(dateTime);
+        }
+
+        public IntervalDayTime truncate(IntervalDayTime dateTime) {
+            return truncateFunctionIntervalDayTime.apply(dateTime);
+        }
+
+        public IntervalYearMonth truncate(IntervalYearMonth dateTime) {
+            return truncateFunctionIntervalYearMonth.apply(dateTime);
+        }
+
+        private static IntervalDayTime truncateIntervalSmallerThanWeek(IntervalDayTime r, ChronoUnit unit) {
+            Duration d = r.interval();
+            int isNegative = 1;
+            if (d.isNegative()) {
+                d = d.negated();
+                isNegative = -1;
+            }
+            long durationInSec = d.getSeconds();
+            long day = durationInSec / SECONDS_PER_DAY;
+            durationInSec = durationInSec % SECONDS_PER_DAY;
+            long hour = durationInSec / SECONDS_PER_HOUR;
+            durationInSec = durationInSec % SECONDS_PER_HOUR;
+            long min = durationInSec / SECONDS_PER_MINUTE;
+            durationInSec = durationInSec % SECONDS_PER_MINUTE;
+            long sec = durationInSec;
+            long miliseccond = TimeUnit.NANOSECONDS.toMillis(d.getNano());
+            Duration newDuration = Duration.ZERO;
+            if (unit.ordinal() <= ChronoUnit.DAYS.ordinal()) {
+                newDuration = newDuration.plusDays(day * isNegative);
+            }
+            if (unit.ordinal() <= ChronoUnit.HOURS.ordinal()) {
+                newDuration = newDuration.plusHours(hour * isNegative);
+            }
+            if (unit.ordinal() <= ChronoUnit.MINUTES.ordinal()) {
+                newDuration = newDuration.plusMinutes(min * isNegative);
+            }
+            if (unit.ordinal() <= ChronoUnit.SECONDS.ordinal()) {
+                newDuration = newDuration.plusSeconds(sec * isNegative);
+            }
+            if (unit.ordinal() <= ChronoUnit.MILLIS.ordinal()) {
+                newDuration = newDuration.plusMillis(miliseccond * isNegative);
+            }
+            return new IntervalDayTime(newDuration, r.dataType());
         }
     }
 
@@ -137,9 +247,25 @@ public DateTrunc(Source source, Expression truncateTo, Expression timestamp, Zon
 
     @Override
     public DataType dataType() {
+        if (isInterval(right().dataType())) {
+            return right().dataType();
+        }
         return DataTypes.DATETIME;
     }
 
+    @Override
+    protected TypeResolution resolveType() {
+        TypeResolution resolution = super.resolveType();
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        resolution = isDateOrInterval(right(), sourceText(), Expressions.ParamOrdinal.SECOND);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
     @Override
     protected BinaryScalarFunction replaceChildren(Expression newTruncateTo, Expression newTimestamp) {
         return new DateTrunc(source(), newTruncateTo, newTimestamp, zoneId());
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessor.java
index 23cc096878ef5..97379d3bc73d0 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessor.java
@@ -6,15 +6,18 @@
 package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
-import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTrunc.Part;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.sql.expression.literal.interval.IntervalDayTime;
+import org.elasticsearch.xpack.sql.expression.literal.interval.IntervalYearMonth;
 
 import java.io.IOException;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.List;
 
+import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTrunc.Part;
+
 public class DateTruncProcessor extends BinaryDateTimeProcessor {
 
     public static final String NAME = "dtrunc";
@@ -59,10 +62,21 @@ public static Object process(Object truncateTo, Object timestamp, ZoneId zoneId)
             }
         }
 
-        if (timestamp instanceof ZonedDateTime == false) {
-            throw new SqlIllegalArgumentException("A date/datetime is required; received [{}]", timestamp);
+        if (timestamp instanceof ZonedDateTime == false && timestamp instanceof IntervalYearMonth == false
+            && timestamp instanceof IntervalDayTime == false) {
+            throw new SqlIllegalArgumentException("A date/datetime/interval is required; received [{}]", timestamp);
+        }
+        if (truncateDateField == Part.WEEK && (timestamp instanceof IntervalDayTime || timestamp instanceof IntervalYearMonth)) {
+            throw new SqlIllegalArgumentException("Truncating intervals is not supported for {} units", truncateTo);
+        }
+
+        if (timestamp instanceof ZonedDateTime) {
+            return truncateDateField.truncate(((ZonedDateTime) timestamp).withZoneSameInstant(zoneId));
+        } else if (timestamp instanceof IntervalYearMonth) {
+            return truncateDateField.truncate((IntervalYearMonth) timestamp);
+        } else {
+            return truncateDateField.truncate((IntervalDayTime) timestamp);
         }
 
-        return truncateDateField.truncate(((ZonedDateTime) timestamp).withZoneSameInstant(zoneId));
     }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
index d303c9fa03c01..13aec510d2b6b 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
@@ -281,8 +281,11 @@ public static Integer dateDiff(String dateField, Object dateTime1, Object dateTi
         return (Integer) DateDiffProcessor.process(dateField, asDateTime(dateTime1), asDateTime(dateTime2) , ZoneId.of(tzId));
     }
 
-    public static ZonedDateTime dateTrunc(String truncateTo, Object dateTime, String tzId) {
-        return (ZonedDateTime) DateTruncProcessor.process(truncateTo, asDateTime(dateTime) , ZoneId.of(tzId));
+    public static Object dateTrunc(String truncateTo, Object dateTimeOrInterval, String tzId) {
+        if (dateTimeOrInterval instanceof IntervalDayTime || dateTimeOrInterval instanceof IntervalYearMonth) {
+           return DateTruncProcessor.process(truncateTo, dateTimeOrInterval, ZoneId.of(tzId));
+        }
+        return DateTruncProcessor.process(truncateTo, asDateTime(dateTimeOrInterval), ZoneId.of(tzId));
     }
 
     public static Integer datePart(String dateField, Object dateTime, String tzId) {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
index 616b2de40810a..d39c295b3101f 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java
@@ -255,6 +255,10 @@ public static boolean isDateOrTimeBased(DataType type) {
         return isDateBased(type) || isTimeBased(type);
     }
 
+    public static boolean isDateOrIntervalBased(DataType type) {
+        return isDateBased(type) || isInterval(type);
+    }
+
     public static boolean isGeo(DataType type) {
         return type == GEO_POINT || type == GEO_SHAPE || type == SHAPE;
     }
@@ -262,7 +266,6 @@ public static boolean isGeo(DataType type) {
     public static String format(DataType type) {
         return isDateOrTimeBased(type) ? "epoch_millis" : null;
     }
-    
 
     public static boolean isFromDocValuesOnly(DataType dataType) {
         return dataType == KEYWORD // because of ignore_above. Extracting this from _source wouldn't make sense
diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
index ea522a0e44084..6e9490dda1a94 100644
--- a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
+++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
@@ -118,7 +118,7 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
   Integer weekOfYear(Object, String)
   ZonedDateTime dateAdd(String, Integer, Object, String)
   Integer dateDiff(String, Object, Object, String)
-  ZonedDateTime dateTrunc(String, Object, String)
+  def dateTrunc(String, Object, String)
   Integer datePart(String, Object, String)
   IntervalDayTime intervalDayTime(String, String)
   IntervalYearMonth intervalYearMonth(String, String)
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
index bb54f5c7c719f..d21668aa92e47 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
@@ -215,8 +215,8 @@ public void testExtractNonDateTime() {
     public void testDateTruncInvalidArgs() {
         assertEquals("1:8: first argument of [DATE_TRUNC(int, date)] must be [string], found value [int] type [integer]",
             error("SELECT DATE_TRUNC(int, date) FROM test"));
-        assertEquals("1:8: second argument of [DATE_TRUNC(keyword, keyword)] must be [date or datetime], found value [keyword] " +
-                "type [keyword]", error("SELECT DATE_TRUNC(keyword, keyword) FROM test"));
+        assertEquals("1:8: second argument of [DATE_TRUNC(keyword, keyword)] must be [date, datetime or an interval data type]," +
+            " found value [keyword] type [keyword]", error("SELECT DATE_TRUNC(keyword, keyword) FROM test"));
         assertEquals("1:8: first argument of [DATE_TRUNC('invalid', keyword)] must be one of [MILLENNIUM, CENTURY, DECADE, " + "" +
                 "YEAR, QUARTER, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND, MILLISECOND, MICROSECOND, NANOSECOND] " +
                 "or their aliases; found value ['invalid']",
@@ -535,11 +535,11 @@ public void testUnsupportedTypeInFilter() {
         assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT * FROM test WHERE unsupported > 1"));
     }
-    
+
     public void testValidRootFieldWithUnsupportedChildren() {
         accept("SELECT x FROM test");
     }
-    
+
     public void testUnsupportedTypeInHierarchy() {
         assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])",
                 error("SELECT x.y.z.w FROM test"));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessorTests.java
index 9c7f11499a9dd..2d05ddfffb3be 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTruncProcessorTests.java
@@ -11,18 +11,29 @@
 import org.elasticsearch.xpack.ql.expression.Literal;
 import org.elasticsearch.xpack.ql.expression.gen.processor.ConstantProcessor;
 import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.sql.AbstractSqlWireSerializingTestCase;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
+import org.elasticsearch.xpack.sql.expression.literal.interval.IntervalDayTime;
+import org.elasticsearch.xpack.sql.expression.literal.interval.IntervalYearMonth;
+import org.elasticsearch.xpack.sql.proto.StringUtils;
+import org.elasticsearch.xpack.sql.type.SqlDataTypes;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
+import java.time.Duration;
+import java.time.Period;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
+import java.time.temporal.TemporalAmount;
 
 import static org.elasticsearch.xpack.ql.expression.Literal.NULL;
 import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
 import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.randomDatetimeLiteral;
+import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeTestUtils.dateTime;
 import static org.elasticsearch.xpack.sql.proto.StringUtils.ISO_DATE_WITH_NANOS;
+import static org.elasticsearch.xpack.sql.type.SqlDataTypes.INTERVAL_DAY_TO_SECOND;
+import static org.elasticsearch.xpack.sql.type.SqlDataTypes.INTERVAL_YEAR_TO_MONTH;
 
 public class DateTruncProcessorTests extends AbstractSqlWireSerializingTestCase<DateTruncProcessor> {
 
@@ -57,13 +68,18 @@ protected DateTruncProcessor mutateInstance(DateTruncProcessor instance) {
     }
 
     public void testInvalidInputs() {
+        TemporalAmount period = Period.ofYears(2018).plusMonths(11);
+        Literal yearToMonth = intervalLiteral(period, INTERVAL_YEAR_TO_MONTH);
+        TemporalAmount duration = Duration.ofDays(42).plusHours(12).plusMinutes(23).plusSeconds(12).plusNanos(143000000);
+        Literal dayToSecond = intervalLiteral(duration, INTERVAL_DAY_TO_SECOND);
+
         SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class,
             () -> new DateTrunc(Source.EMPTY, l(5), randomDatetimeLiteral(), randomZone()).makePipe().asProcessor().process(null));
         assertEquals("A string is required; received [5]", siae.getMessage());
 
         siae = expectThrows(SqlIllegalArgumentException.class,
             () -> new DateTrunc(Source.EMPTY, l("days"), l("foo"), randomZone()).makePipe().asProcessor().process(null));
-        assertEquals("A date/datetime is required; received [foo]", siae.getMessage());
+        assertEquals("A date/datetime/interval is required; received [foo]", siae.getMessage());
 
         siae = expectThrows(SqlIllegalArgumentException.class,
             () -> new DateTrunc(Source.EMPTY, l("invalid"), randomDatetimeLiteral(), randomZone()).makePipe().asProcessor().process(null));
@@ -75,6 +91,16 @@ public void testInvalidInputs() {
             () -> new DateTrunc(Source.EMPTY, l("dacede"), randomDatetimeLiteral(), randomZone()).makePipe().asProcessor().process(null));
         assertEquals("Received value [dacede] is not valid date part for truncation; did you mean [decade, decades]?",
             siae.getMessage());
+
+        siae = expectThrows(SqlIllegalArgumentException.class,
+            () -> new DateTrunc(Source.EMPTY, l("weeks"), yearToMonth, null).makePipe().asProcessor().process(null));
+        assertEquals("Truncating intervals is not supported for weeks units",
+            siae.getMessage());
+
+        siae = expectThrows(SqlIllegalArgumentException.class,
+            () -> new DateTrunc(Source.EMPTY, l("week"), dayToSecond, null).makePipe().asProcessor().process(null));
+        assertEquals("Truncating intervals is not supported for week units",
+            siae.getMessage());
     }
 
     public void testWithNulls() {
@@ -86,6 +112,10 @@ public void testWithNulls() {
     public void testTruncation() {
         ZoneId zoneId = ZoneId.of("Etc/GMT-10");
         Literal dateTime = l(dateTime(2019, 9, 3, 18, 10, 37, 123456789));
+        TemporalAmount period = Period.ofYears(2019).plusMonths(10);
+        Literal yearToMonth = intervalLiteral(period, INTERVAL_YEAR_TO_MONTH);
+        TemporalAmount duration = Duration.ofDays(105).plusHours(2).plusMinutes(45).plusSeconds(55).plusNanos(123456789);
+        Literal dayToSecond = intervalLiteral(duration, INTERVAL_DAY_TO_SECOND);
 
         assertEquals("2000-01-01T00:00:00.000+10:00",
             DateUtils.toString((ZonedDateTime) new DateTrunc(Source.EMPTY, l("millennia"), dateTime, zoneId)
@@ -129,6 +159,86 @@ public void testTruncation() {
         assertEquals("2019-09-04T04:10:37.123456789+10:00",
             toString((ZonedDateTime) new DateTrunc(Source.EMPTY, l("nanoseconds"), dateTime, zoneId)
                 .makePipe().asProcessor().process(null)));
+
+        assertEquals("+2000-0",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("millennia"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2000-0",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("CENTURY"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2010-0",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("decades"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-0",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("years"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-9",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("quarters"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("month"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("days"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("hh"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("mi"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("second"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("ms"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("mcs"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+2019-10",
+            toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("nanoseconds"), yearToMonth, null)
+                .makePipe().asProcessor().process(null)));
+
+        assertEquals("+0 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("millennia"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+0 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("CENTURY"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+0 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("decades"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+0 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("years"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+0 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("quarters"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+0 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("month"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 00:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("days"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 02:00:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("hh"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 02:45:00",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("mi"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 02:45:55",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("second"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 02:45:55.123",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("ms"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 02:45:55.123",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("microseconds"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
+        assertEquals("+105 02:45:55.123",
+            toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("nanoseconds"), dayToSecond, null)
+                .makePipe().asProcessor().process(null)));
     }
 
     public void testTruncationEdgeCases() {
@@ -152,9 +262,58 @@ public void testTruncationEdgeCases() {
         assertEquals("-1234-08-29T00:00:00.000+10:00",
             DateUtils.toString((ZonedDateTime) new DateTrunc(Source.EMPTY, l("week"), dateTime, zoneId)
                 .makePipe().asProcessor().process(null)));
+
+        Literal yearToMonth = intervalLiteral(Period.ofYears(-12523).minusMonths(10), INTERVAL_YEAR_TO_MONTH);
+        assertEquals("-12000-0", toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("millennia"), yearToMonth, null)
+            .makePipe().asProcessor().process(null)));
+
+        yearToMonth = intervalLiteral(Period.ofYears(-32543).minusMonths(10), INTERVAL_YEAR_TO_MONTH);
+        assertEquals("-32500-0", toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("centuries"), yearToMonth, null)
+            .makePipe().asProcessor().process(null)));
+
+        yearToMonth = intervalLiteral(Period.ofYears(-24321).minusMonths(10), INTERVAL_YEAR_TO_MONTH);
+        assertEquals("-24320-0", toString((IntervalYearMonth) new DateTrunc(Source.EMPTY, l("decades"), yearToMonth, null)
+            .makePipe().asProcessor().process(null)));
+
+        Literal dayToSecond = intervalLiteral(Duration.ofDays(-435).minusHours(23).minusMinutes(45).minusSeconds(55).minusNanos(123000000),
+            INTERVAL_DAY_TO_SECOND);
+        assertEquals("-435 00:00:00", toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("days"), dayToSecond, null)
+            .makePipe().asProcessor().process(null)));
+
+        dayToSecond = intervalLiteral(Duration.ofDays(-4231).minusHours(23).minusMinutes(45).minusSeconds(55).minusNanos(234000000),
+            INTERVAL_DAY_TO_SECOND);
+        assertEquals("-4231 23:00:00", toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("hh"), dayToSecond, null)
+            .makePipe().asProcessor().process(null)));
+
+        dayToSecond = intervalLiteral(Duration.ofDays(-124).minusHours(0).minusMinutes(59).minusSeconds(11).minusNanos(564000000),
+            INTERVAL_DAY_TO_SECOND);
+        assertEquals("-124 00:59:00", toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("mi"), dayToSecond, null)
+            .makePipe().asProcessor().process(null)));
+
+        dayToSecond = intervalLiteral(Duration.ofDays(-534).minusHours(23).minusMinutes(59).minusSeconds(59).minusNanos(245000000),
+            INTERVAL_DAY_TO_SECOND);
+        assertEquals("-534 23:59:59", toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("seconds"), dayToSecond, null)
+            .makePipe().asProcessor().process(null)));
+
+        dayToSecond = intervalLiteral(Duration.ofDays(-127).minusHours(17).minusMinutes(59).minusSeconds(59).minusNanos(987654321),
+            INTERVAL_DAY_TO_SECOND);
+        assertEquals("-127 17:59:59.987", toString((IntervalDayTime) new DateTrunc(Source.EMPTY, l("ms"), dayToSecond, null)
+            .makePipe().asProcessor().process(null)));
+    }
+    private String toString(IntervalYearMonth intervalYearMonth) {
+        return StringUtils.toString(intervalYearMonth);
     }
 
+    private String toString(IntervalDayTime intervalDayTime) {
+        return StringUtils.toString(intervalDayTime);
+    }
     private String toString(ZonedDateTime dateTime) {
         return ISO_DATE_WITH_NANOS.format(dateTime);
     }
+
+    private static Literal intervalLiteral(TemporalAmount value, DataType intervalType) {
+        Object interval = value instanceof Period ? new IntervalYearMonth((Period) value, intervalType)
+            : new IntervalDayTime((Duration) value, intervalType);
+        return new Literal(EMPTY, interval, SqlDataTypes.fromJava(interval));
+    }
 }

From 1af04175a19df8d099295b7eca768289d3992f32 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Mon, 23 Mar 2020 13:42:10 +0100
Subject: [PATCH 201/285] Async search: remove version from response (#53960)

The goal of the version field was to quickly show when you can expect to find something new in the search response, compared to when nothing has changed. This can also be done by looking at the `_shards` section and `num_reduce_phases` returned with the search response. In fact when there has been one or more additional reduction of the results, you can expect new results in the search response. Otherwise, the `_shards` section could notify of additional failures of shards that have completed the query, but that is not a guarantee that their results will be exposed (only when the following partial reduction is performed their results will be available).

That said this commit clarifies this in the docs and removes the version field from the async search response
---
 .../asyncsearch/AsyncSearchResponse.java      | 40 ++++++-------------
 .../client/asyncsearch/AsyncSearchIT.java     |  1 -
 .../asyncsearch/AsyncSearchResponseTests.java |  4 +-
 docs/reference/search/async-search.asciidoc   | 33 ++++++++-------
 .../xpack/search/MutableSearchResponse.java   |  9 +----
 .../search/AsyncSearchResponseTests.java      |  7 ++--
 .../search/action/AsyncSearchResponse.java    | 19 +--------
 .../test/async_search/10_basic.yml            |  5 ---
 8 files changed, 37 insertions(+), 81 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java
index 47dd444ea5463..07d3ce81fea8c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponse.java
@@ -41,7 +41,6 @@
 public class AsyncSearchResponse implements ToXContentObject  {
     @Nullable
     private final String id;
-    private final int version;
     @Nullable
     private final SearchResponse searchResponse;
     @Nullable
@@ -55,15 +54,13 @@ public class AsyncSearchResponse implements ToXContentObject  {
     /**
      * Creates an {@link AsyncSearchResponse} with the arguments that are always present in the server response
      */
-    AsyncSearchResponse(int version,
-                               boolean isPartial,
-                               boolean isRunning,
-                               long startTimeMillis,
-                               long expirationTimeMillis,
-                               @Nullable String id,
-                               @Nullable SearchResponse searchResponse,
-                               @Nullable ElasticsearchException error) {
-        this.version = version;
+    AsyncSearchResponse(boolean isPartial,
+                        boolean isRunning,
+                        long startTimeMillis,
+                        long expirationTimeMillis,
+                        @Nullable String id,
+                        @Nullable SearchResponse searchResponse,
+                        @Nullable ElasticsearchException error) {
         this.isPartial = isPartial;
         this.isRunning = isRunning;
         this.startTimeMillis = startTimeMillis;
@@ -81,13 +78,6 @@ public String getId() {
         return id;
     }
 
-    /**
-     * Returns the version of this response.
-     */
-    public int getVersion() {
-        return version;
-    }
-
     /**
      * Returns the current {@link SearchResponse} or <code>null</code> if not available.
      *
@@ -145,7 +135,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (id != null) {
             builder.field("id", id);
         }
-        builder.field("version", version);
         builder.field("is_partial", isPartial);
         builder.field("is_running", isRunning);
         builder.field("start_time_in_millis", startTimeMillis);
@@ -165,7 +154,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     public static final ParseField ID_FIELD = new ParseField("id");
-    public static final ParseField VERSION_FIELD = new ParseField("version");
     public static final ParseField IS_PARTIAL_FIELD = new ParseField("is_partial");
     public static final ParseField IS_RUNNING_FIELD = new ParseField("is_running");
     public static final ParseField START_TIME_FIELD = new ParseField("start_time_in_millis");
@@ -176,16 +164,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     public static final ConstructingObjectParser<AsyncSearchResponse, Void> PARSER = new ConstructingObjectParser<>(
             "submit_async_search_response", true,
             args -> new AsyncSearchResponse(
-                    (int) args[0],
+                    (boolean) args[0],
                     (boolean) args[1],
-                    (boolean) args[2],
+                    (long) args[2],
                     (long) args[3],
-                    (long) args[4],
-                    (String) args[5],
-                    (SearchResponse) args[6],
-                    (ElasticsearchException) args[7]));
+                    (String) args[4],
+                    (SearchResponse) args[5],
+                    (ElasticsearchException) args[6]));
     static {
-        PARSER.declareInt(constructorArg(), VERSION_FIELD);
         PARSER.declareBoolean(constructorArg(), IS_PARTIAL_FIELD);
         PARSER.declareBoolean(constructorArg(), IS_RUNNING_FIELD);
         PARSER.declareLong(constructorArg(), START_TIME_FIELD);
@@ -203,7 +189,7 @@ private static SearchResponse parseSearchResponse(XContentParser p) throws IOExc
         return SearchResponse.innerFromXContent(p);
     }
 
-    public static AsyncSearchResponse fromXContent(XContentParser parser) throws IOException {
+    public static AsyncSearchResponse fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
index a1c608b3c6ef4..d0f6ddc93080f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
@@ -40,7 +40,6 @@ public void testSubmitAsyncSearchRequest() throws IOException {
         // 15 sec should be enough to make sure we always complete right away
         request.setWaitForCompletion(new TimeValue(15, TimeUnit.SECONDS));
         AsyncSearchResponse response = highLevelClient().asyncSearch().submitAsyncSearch(request, RequestOptions.DEFAULT);
-        assertTrue(response.getVersion() >= 0);
         assertFalse(response.isPartial());
         assertTrue(response.getStartTime() > 0);
         assertTrue(response.getExpirationTime() > 0);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
index 6caf69db9ac6c..08c0da25e5bb6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
@@ -36,7 +36,6 @@ public class AsyncSearchResponseTests
 
     @Override
     protected org.elasticsearch.xpack.core.search.action.AsyncSearchResponse createServerTestInstance(XContentType xContentType) {
-        int version = randomIntBetween(0, Integer.MAX_VALUE);
         boolean isPartial = randomBoolean();
         boolean isRunning = randomBoolean();
         long startTimeMillis = randomLongBetween(0, Long.MAX_VALUE);
@@ -48,7 +47,7 @@ protected org.elasticsearch.xpack.core.search.action.AsyncSearchResponse createS
                 : new SearchResponse(InternalSearchResponse.empty(), randomAlphaOfLength(10), 1, 1, 0, randomIntBetween(0, 10000),
                         ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY);
         org.elasticsearch.xpack.core.search.action.AsyncSearchResponse testResponse =
-                new org.elasticsearch.xpack.core.search.action.AsyncSearchResponse(id, version, searchResponse, error, isPartial, isRunning,
+                new org.elasticsearch.xpack.core.search.action.AsyncSearchResponse(id, searchResponse, error, isPartial, isRunning,
                         startTimeMillis, expirationTimeMillis);
         return testResponse;
     }
@@ -62,7 +61,6 @@ protected AsyncSearchResponse doParseToClientInstance(XContentParser parser) thr
     protected void assertInstances(org.elasticsearch.xpack.core.search.action.AsyncSearchResponse expected, AsyncSearchResponse parsed) {
         assertNotSame(parsed, expected);
         assertEquals(expected.getId(), parsed.getId());
-        assertEquals(expected.getVersion(), parsed.getVersion());
         assertEquals(expected.isRunning(), parsed.isRunning());
         assertEquals(expected.isPartial(), parsed.isPartial());
         assertEquals(expected.getStartTime(), parsed.getStartTime());
diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
index 1b94e4000f38f..32b89f02d0523 100644
--- a/docs/reference/search/async-search.asciidoc
+++ b/docs/reference/search/async-search.asciidoc
@@ -43,7 +43,6 @@ results are returned as part of the <<search-api-response-body,`response`>> obje
 --------------------------------------------------
 {
   "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=", <1>
-  "version" : 0,
   "is_partial" : true, <2>
   "is_running" : true, <3>
   "start_time_in_millis" : 1583945890986,
@@ -135,17 +134,16 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
 --------------------------------------------------
 {
   "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=",
-  "version" : 2, <1>
-  "is_partial" : true, <2>
-  "is_running" : true, <3>
+  "is_partial" : true, <1>
+  "is_running" : true, <2>
   "start_time_in_millis" : 1583945890986,
-  "expiration_time_in_millis" : 1584377890986, <4>
+  "expiration_time_in_millis" : 1584377890986, <3>
   "response" : {
     "took" : 12144,
     "timed_out" : false,
-    "num_reduce_phases" : 38,
+    "num_reduce_phases" : 38, <4>
     "_shards" : {
-      "total" : 562,
+      "total" : 562, <5>
       "successful" : 188,
       "skipped" : 0,
       "failed" : 0
@@ -158,7 +156,7 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
       "max_score" : null,
       "hits" : [ ]
     },
-    "aggregations" : { <5>
+    "aggregations" : { <6>
       "sale_date" :  {
         "buckets" : []
       }
@@ -178,15 +176,16 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
 // TESTRESPONSE[s/"buckets" : \[\]/"buckets": $body.response.aggregations.sale_date.buckets/]
 // TESTRESPONSE[s/"num_reduce_phases" : 38,//]
 
-<1> The returned `version` is useful to identify whether the response contains
-additional results compared to previously obtained responses. If the version
-stays the same, no new results have become available, otherwise a higher version
-number indicates that more shards have completed their execution of the query
-and their partial results are also included in the response.
-<2> Whether the returned search results are partial or final
-<3> Whether the search is still being executed or it has completed
-<4> When the async search will expire
-<5> Partial aggregations results, coming from the shards that have already
+<1> Whether the returned search results are partial or final
+<2> Whether the search is still being executed or it has completed
+<3> When the async search will expire
+<4> Indicates how many reduction of the results have been performed. If this
+number increases compared to the last retrieved results, you can expect
+additional results included in the search response
+<5> Indicates how many shards have executed the query. Note that in order for
+shard results to be included in the search response, they need to be reduced
+first.
+<6> Partial aggregations results, coming from the shards that have already
 completed the execution of the query.
 
 The `wait_for_completion` parameter, which defaults to `1`, can also be provided
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
index 82e311fc8bb8a..cce587abd15d3 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
@@ -40,7 +40,6 @@ class MutableSearchResponse {
     private final AtomicArray<ShardSearchFailure> shardFailures;
     private final Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier;
 
-    private int version;
     private boolean isPartial;
     private boolean isFinalReduce;
     private int successfulShards;
@@ -63,7 +62,6 @@ class MutableSearchResponse {
         this.skippedShards = skippedShards;
         this.clusters = clusters;
         this.aggReduceContextSupplier = aggReduceContextSupplier;
-        this.version = 0;
         this.shardFailures = totalShards == -1 ? null : new AtomicArray<>(totalShards-skippedShards);
         this.isPartial = true;
         this.sections = totalShards == -1 ? null : new InternalSearchResponse(
@@ -83,7 +81,6 @@ synchronized void updatePartialResponse(int successfulShards, SearchResponseSect
             throw new IllegalStateException("received partial response out of order: "
                 + newSections.getNumReducePhases() + " < " + sections.getNumReducePhases());
         }
-        ++ version;
         this.successfulShards = successfulShards;
         this.sections = newSections;
         this.isPartial = true;
@@ -96,7 +93,6 @@ synchronized void updatePartialResponse(int successfulShards, SearchResponseSect
      */
     synchronized void updateFinalResponse(int successfulShards, SearchResponseSections newSections) {
         failIfFrozen();
-        ++ version;
         this.successfulShards = successfulShards;
         this.sections = newSections;
         this.isPartial = false;
@@ -110,7 +106,6 @@ synchronized void updateFinalResponse(int successfulShards, SearchResponseSectio
      */
     synchronized void updateWithFailure(Exception exc) {
         failIfFrozen();
-        ++ version;
         this.isPartial = true;
         this.failure = ElasticsearchException.guessRootCauses(exc)[0];
         this.frozen = true;
@@ -147,7 +142,7 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, lon
         } else {
             resp = null;
         }
-        return new AsyncSearchResponse(task.getSearchId().getEncoded(), version, resp, failure, isPartial,
+        return new AsyncSearchResponse(task.getSearchId().getEncoded(), resp, failure, isPartial,
             frozen == false, task.getStartTime(), expirationTime);
     }
 
@@ -159,7 +154,7 @@ private void failIfFrozen() {
 
     private ShardSearchFailure[] buildShardFailures() {
         if (shardFailures == null) {
-            return new ShardSearchFailure[0];
+            return ShardSearchFailure.EMPTY_ARRAY;
         }
         List<ShardSearchFailure> failures = new ArrayList<>();
         for (int i = 0; i < shardFailures.length(); i++) {
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
index 925ce9d7897eb..9b68453e5306b 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
@@ -91,15 +91,15 @@ static AsyncSearchResponse randomAsyncSearchResponse(String searchId, SearchResp
         int rand = randomIntBetween(0, 2);
         switch (rand) {
             case 0:
-                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(),
+                return new AsyncSearchResponse(searchId, randomBoolean(),
                     randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong());
 
             case 1:
-                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, null,
+                return new AsyncSearchResponse(searchId, searchResponse, null,
                     randomBoolean(), randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong());
 
             case 2:
-                return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse,
+                return new AsyncSearchResponse(searchId, searchResponse,
                     new ElasticsearchException(new IOException("boum")), randomBoolean(), randomBoolean(),
                     randomNonNegativeLong(), randomNonNegativeLong());
 
@@ -120,7 +120,6 @@ static SearchResponse randomSearchResponse() {
 
     static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) {
         assertEquals(expected.getId(), actual.getId());
-        assertEquals(expected.getVersion(), actual.getVersion());
         assertEquals(expected.status(), actual.status());
         assertEquals(expected.getFailure() == null, actual.getFailure() == null);
         assertEquals(expected.isRunning(), actual.isRunning());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
index 80d1439f80abc..ad46baf891864 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
@@ -25,7 +25,6 @@
 public class AsyncSearchResponse extends ActionResponse implements StatusToXContentObject {
     @Nullable
     private final String id;
-    private final int version;
     @Nullable
     private final SearchResponse searchResponse;
     @Nullable
@@ -40,19 +39,17 @@ public class AsyncSearchResponse extends ActionResponse implements StatusToXCont
      * Creates an {@link AsyncSearchResponse} with meta-information only (not-modified).
      */
     public AsyncSearchResponse(String id,
-                               int version,
                                boolean isPartial,
                                boolean isRunning,
                                long startTimeMillis,
                                long expirationTimeMillis) {
-        this(id, version, null, null, isPartial, isRunning, startTimeMillis, expirationTimeMillis);
+        this(id, null, null, isPartial, isRunning, startTimeMillis, expirationTimeMillis);
     }
 
     /**
      * Creates a new {@link AsyncSearchResponse}
      *
      * @param id The id of the search for further retrieval, <code>null</code> if not stored.
-     * @param version The version number of this response.
      * @param searchResponse The actual search response.
      * @param error The error if the search failed, <code>null</code> if the search is running
      *                or has completed without failure.
@@ -61,7 +58,6 @@ public AsyncSearchResponse(String id,
      * @param startTimeMillis The start date of the search in milliseconds since epoch.
      */
     public AsyncSearchResponse(String id,
-                               int version,
                                SearchResponse searchResponse,
                                ElasticsearchException error,
                                boolean isPartial,
@@ -69,7 +65,6 @@ public AsyncSearchResponse(String id,
                                long startTimeMillis,
                                long expirationTimeMillis) {
         this.id = id;
-        this.version = version;
         this.error = error;
         this.searchResponse = searchResponse;
         this.isPartial = isPartial;
@@ -80,7 +75,6 @@ public AsyncSearchResponse(String id,
 
     public AsyncSearchResponse(StreamInput in) throws IOException {
         this.id = in.readOptionalString();
-        this.version = in.readVInt();
         this.error = in.readOptionalWriteable(ElasticsearchException::new);
         this.searchResponse = in.readOptionalWriteable(SearchResponse::new);
         this.isPartial = in.readBoolean();
@@ -92,7 +86,6 @@ public AsyncSearchResponse(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(id);
-        out.writeVInt(version);
         out.writeOptionalWriteable(error);
         out.writeOptionalWriteable(searchResponse);
         out.writeBoolean(isPartial);
@@ -102,7 +95,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     public AsyncSearchResponse clone(String id) {
-        return new AsyncSearchResponse(id, version, searchResponse, error, isPartial, false, startTimeMillis, expirationTimeMillis);
+        return new AsyncSearchResponse(id, searchResponse, error, isPartial, false, startTimeMillis, expirationTimeMillis);
     }
 
     /**
@@ -113,13 +106,6 @@ public String getId() {
         return id;
     }
 
-    /**
-     * Returns the version of this response.
-     */
-    public int getVersion() {
-        return version;
-    }
-
     /**
      * Returns the current {@link SearchResponse} or <code>null</code> if not available.
      *
@@ -189,7 +175,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (id != null) {
             builder.field("id", id);
         }
-        builder.field("version", version);
         builder.field("is_partial", isPartial);
         builder.field("is_running", isRunning);
         builder.field("start_time_in_millis", startTimeMillis);
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
index dd78157e79bef..c373217b06c99 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
@@ -54,7 +54,6 @@
           sort: max
 
   - is_false: id
-  - match:  { version:                          6 }
   - match:  { is_partial:                   false }
   - length: { response.hits.hits:               3 }
   - match:  { response.hits.hits.0._source.max: 1 }
@@ -73,7 +72,6 @@
                 field: max
           sort: max
 
-  - match:  { version:                          6 }
   - match:  { is_partial:                   false }
   - length: { response.hits.hits:               3 }
   - match:  { response.hits.hits.0._source.max: 1 }
@@ -95,7 +93,6 @@
           sort: max
 
   - set:    { id:                                  id }
-  - match:  { version:                              6 }
   - match:  { is_partial:                       false }
   - is_false: response._clusters
   - length: { response.hits.hits:                   3 }
@@ -106,7 +103,6 @@
       async_search.get:
         id: "$id"
 
-  - match:  { version:                            6 }
   - match:  { is_partial:                     false }
   - is_false: response._clusters
   - length: { response.hits.hits:                 3 }
@@ -119,7 +115,6 @@
         id: "$id"
         typed_keys: true
 
-  - match:  { version:                              6 }
   - match:  { is_partial:                       false }
   - is_false: response._clusters
   - length: { response.hits.hits:                   3 }

From b1f4f3213792e4da3834ec49f168d9da40163764 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Mon, 23 Mar 2020 13:42:35 +0100
Subject: [PATCH 202/285] Async Search: replicas to auto expand from 0 to 1
 (#53964)

This way single node clusters that are green don't go yellow once async search is used, while
all the others still have one replica.
---
 .../xpack/search/AsyncSearchIndexService.java    |  6 +++---
 .../search/AsyncSearchIndexServiceTests.java     | 16 ++++++++++++++++
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
index d1ec9fac7c06f..94d5e7797cad1 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
@@ -66,14 +66,14 @@ class AsyncSearchIndexService {
     public static final String EXPIRATION_TIME_FIELD = "expiration_time";
     public static final String RESULT_FIELD = "result";
 
-    public static Settings settings() {
+    private static Settings settings() {
         return Settings.builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
             .build();
     }
 
-    public static XContentBuilder mappings() throws IOException {
+    private static XContentBuilder mappings() throws IOException {
         XContentBuilder builder = jsonBuilder()
             .startObject()
                 .startObject(SINGLE_MAPPING_NAME)
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
index afc7627b2e4df..c66c61f93439b 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java
@@ -5,6 +5,10 @@
  */
 package org.elasticsearch.xpack.search;
 
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -17,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses;
 import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse;
@@ -100,4 +105,15 @@ public void testEnsuredAuthenticatedUserIsSame() throws IOException {
         assertFalse(indexService.ensureAuthenticatedUserIsSame(original, runAsDiffType));
         assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType));
     }
+
+    public void testSettings() throws ExecutionException, InterruptedException {
+        PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        indexService.createIndexIfNecessary(future);
+        future.get();
+        GetIndexResponse getIndexResponse = client().admin().indices().getIndex(
+            new GetIndexRequest().indices(AsyncSearchIndexService.INDEX)).actionGet();
+        Settings settings = getIndexResponse.getSettings().get(AsyncSearchIndexService.INDEX);
+        assertEquals("1", settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS));
+        assertEquals("0-1", settings.get(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS));
+    }
 }

From d56d2cf08c97944c163dc36be3608ff800353246 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 23 Mar 2020 13:50:02 +0100
Subject: [PATCH 203/285] Upgrade to AWS SDK 1.11.749 (#53962)

Upgrading AWS SDK to v1.11.749.
Required building clients inside privileged contexts because some class loading that requires privileges now happens there and working around a new SDK bug in the S3 client builder.

Closes #53191
---
 plugins/discovery-ec2/build.gradle            |  2 +-
 .../aws-java-sdk-core-1.11.636.jar.sha1       |  1 -
 .../aws-java-sdk-core-1.11.749.jar.sha1       |  1 +
 .../aws-java-sdk-ec2-1.11.636.jar.sha1        |  1 -
 .../aws-java-sdk-ec2-1.11.749.jar.sha1        |  1 +
 .../discovery/ec2/AmazonEC2Fixture.java       |  8 ++++++++
 .../discovery/ec2/AwsEc2ServiceImpl.java      | 20 ++++++++++---------
 .../ec2/Ec2DiscoveryPluginTests.java          | 14 +++++--------
 plugins/repository-s3/build.gradle            |  2 +-
 .../aws-java-sdk-core-1.11.636.jar.sha1       |  1 -
 .../aws-java-sdk-core-1.11.749.jar.sha1       |  1 +
 .../aws-java-sdk-s3-1.11.636.jar.sha1         |  1 -
 .../aws-java-sdk-s3-1.11.749.jar.sha1         |  1 +
 .../licenses/jmespath-java-1.11.636.jar.sha1  |  1 -
 .../licenses/jmespath-java-1.11.749.jar.sha1  |  1 +
 .../repositories/s3/S3Service.java            |  9 +++++++--
 16 files changed, 38 insertions(+), 27 deletions(-)
 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1

diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index 72d3c5b294bd3..c45d0f9f457c0 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -25,7 +25,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.636'
+  'aws': '1.11.749'
 ]
 
 dependencies {
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1
deleted file mode 100644
index b9ee9c102dbcb..0000000000000
--- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-84c9f180f8f60f6f1433c9c5253fcb704593b121
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1
new file mode 100644
index 0000000000000..7bc18d6d4f681
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1
@@ -0,0 +1 @@
+1da5c1549295cfeebc67fc1c7539785a9441755b
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1
deleted file mode 100644
index ed737c808c1de..0000000000000
--- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d32fc4ae314dbee9717302a3119cba0f735c04b1
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1
new file mode 100644
index 0000000000000..c7c7220005fc3
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1
@@ -0,0 +1 @@
+0865e0937c6500acf62ce9c8964eac76a8718f5f
\ No newline at end of file
diff --git a/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java b/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java
index 32abcdc43e645..ecaaceea5cc26 100644
--- a/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java
+++ b/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java
@@ -22,6 +22,7 @@
 import org.apache.http.NameValuePair;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.SuppressForbidden;
@@ -106,6 +107,13 @@ protected Response handle(final Request request) throws IOException {
             return new Response(RestStatus.OK.getStatus(), headers, "my_iam_profile".getBytes(UTF_8));
         }
 
+        if (instanceProfile && "/latest/api/token".equals(request.getPath())
+            && HttpPut.METHOD_NAME.equals(request.getMethod())) {
+            // TODO: Implement IMDSv2 behavior here. For now this just returns a 403 which makes the SDK fall back to IMDSv1
+            //       which is implemented in this fixture
+            return new Response(RestStatus.FORBIDDEN.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
+        }
+
         if ((containerCredentials &&
             "/ecs_credentials_endpoint".equals(request.getPath()) &&
             HttpGet.METHOD_NAME.equals(request.getMethod())) ||
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
index ac18775fd1592..1d6f69927ab6d 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
@@ -24,9 +24,10 @@
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.services.ec2.AmazonEC2;
-import com.amazonaws.services.ec2.AmazonEC2Client;
+import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -45,17 +46,18 @@ class AwsEc2ServiceImpl implements AwsEc2Service {
     private AmazonEC2 buildClient(Ec2ClientSettings clientSettings) {
         final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
         final ClientConfiguration configuration = buildConfiguration(clientSettings);
-        final AmazonEC2 client = buildClient(credentials, configuration);
-        if (Strings.hasText(clientSettings.endpoint)) {
-            logger.debug("using explicit ec2 endpoint [{}]", clientSettings.endpoint);
-            client.setEndpoint(clientSettings.endpoint);
-        }
-        return client;
+        return buildClient(credentials, configuration, clientSettings.endpoint);
     }
 
     // proxy for testing
-    AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
-        return new AmazonEC2Client(credentials, configuration);
+    AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) {
+        final AmazonEC2ClientBuilder builder = AmazonEC2ClientBuilder.standard().withCredentials(credentials)
+            .withClientConfiguration(configuration);
+        if (Strings.hasText(endpoint)) {
+            logger.debug("using explicit ec2 endpoint [{}]", endpoint);
+            builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
+        }
+        return SocketAccess.doPrivileged(builder::build);
     }
 
     // pkg private for tests
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
index 661b5815b4029..2e53a5f614101 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
@@ -39,7 +39,6 @@
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class Ec2DiscoveryPluginTests extends ESTestCase {
 
@@ -96,7 +95,7 @@ public void testNodeAttributesErrorLenient() throws Exception {
     public void testDefaultEndpoint() throws IOException {
         try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(Settings.EMPTY)) {
             final String endpoint = ((AmazonEC2Mock) plugin.ec2Service.client().client()).endpoint;
-            assertThat(endpoint, nullValue());
+            assertThat(endpoint, is(""));
         }
     }
 
@@ -199,8 +198,9 @@ private static class Ec2DiscoveryPluginMock extends Ec2DiscoveryPlugin {
         Ec2DiscoveryPluginMock(Settings settings) {
             super(settings, new AwsEc2ServiceImpl() {
                 @Override
-                AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
-                    return new AmazonEC2Mock(credentials, configuration);
+                AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration,
+                                      String endpoint) {
+                    return new AmazonEC2Mock(credentials, configuration, endpoint);
                 }
             });
         }
@@ -212,13 +212,9 @@ private static class AmazonEC2Mock extends AbstractAmazonEC2 {
         final AWSCredentialsProvider credentials;
         final ClientConfiguration configuration;
 
-        AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
+        AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) {
             this.credentials = credentials;
             this.configuration = configuration;
-        }
-
-        @Override
-        public void setEndpoint(String endpoint) throws IllegalArgumentException {
             this.endpoint = endpoint;
         }
 
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 04624f8512976..a7c8d49a50728 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -30,7 +30,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.636'
+  'aws': '1.11.749'
 ]
 
 dependencies {
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1
deleted file mode 100644
index b9ee9c102dbcb..0000000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-84c9f180f8f60f6f1433c9c5253fcb704593b121
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1
new file mode 100644
index 0000000000000..7bc18d6d4f681
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1
@@ -0,0 +1 @@
+1da5c1549295cfeebc67fc1c7539785a9441755b
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1
deleted file mode 100644
index 1e05e98d240d2..0000000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f86fc1993ac8122f6f02a8eb9b467b5f945cd76b
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1
new file mode 100644
index 0000000000000..af794dc59dd7f
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1
@@ -0,0 +1 @@
+7d069f82723907ccdbd0c91ef0ac76046f5c9652
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1
deleted file mode 100644
index 70c0d3633af07..0000000000000
--- a/plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e468c349ce410171a1d5df7fa0fa377d52c5d651
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1
new file mode 100644
index 0000000000000..3467802d074c7
--- /dev/null
+++ b/plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1
@@ -0,0 +1 @@
+778866bc557dba508ee0eab2a0c5bfde468e49e6
\ No newline at end of file
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
index 77117550dc0d9..73149bf92073a 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
@@ -141,7 +141,12 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) {
         builder.withCredentials(buildCredentials(logger, clientSettings));
         builder.withClientConfiguration(buildConfiguration(clientSettings));
 
-        final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
+        String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
+        if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) {
+            // Manually add the schema to the endpoint to work around https://github.com/aws/aws-sdk-java/issues/2274
+            // TODO: Remove this once fixed in the AWS SDK
+            endpoint = clientSettings.protocol.toString() + "://" + endpoint;
+        }
         final String region = Strings.hasLength(clientSettings.region) ? clientSettings.region : null;
         logger.debug("using endpoint [{}] and region [{}]", endpoint, region);
 
@@ -160,7 +165,7 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) {
         if (clientSettings.disableChunkedEncoding) {
             builder.disableChunkedEncoding();
         }
-        return builder.build();
+        return SocketAccess.doPrivileged(builder::build);
     }
 
     // pkg private for tests

From abcee01a96f72c7c5466c31a7f7ad68c0d512a6c Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 23 Mar 2020 14:04:17 +0100
Subject: [PATCH 204/285] adjusted skip version (#53969)

---
 .../rest-api-spec/test/indices.data_stream/10_basic.yml       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
index 49754005b6db5..035c20c1600bf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -1,8 +1,8 @@
 ---
 "Test stubs":
   - skip:
-      version: " - 7.99.99"
-      reason: not backported yet
+      version: " - 7.6.99"
+      reason: only available in 7.7+
 
   - do:
       indices.create_data_stream:

From aa56f91fba5ac90a135b8fdb982947afa8492d23 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Mon, 23 Mar 2020 14:17:53 +0100
Subject: [PATCH 205/285] [DOCS] address timing issue in async search docs
 tests (#53910)

The docs snippets for submit async search have proven difficult to test as it is not possible to guarantee that you get a response that is not final, even when providing `wait_for_completion=0`. In the docs we want to show though a proper long-running query, and its first response should be partial rather than final.

With this commit we adapt the docs snippets to show a partial response, and replace under the hood all that's needed to make the snippets tests succeed when we get a final response. Also, increased the timeout so we always get a final response.

Closes #53887
Closes #53891
---
 docs/reference/search/async-search.asciidoc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
index 32b89f02d0523..7f5957a94e41c 100644
--- a/docs/reference/search/async-search.asciidoc
+++ b/docs/reference/search/async-search.asciidoc
@@ -30,9 +30,8 @@ POST /sales*/_async_search?size=0
     }
 }
 --------------------------------------------------
-// TEST[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/53887"]
 // TEST[setup:sales]
-// TEST[s/size=0/size=0&wait_for_completion=0/]
+// TEST[s/size=0/size=0&wait_for_completion=10s&clean_on_completion=false/]
 
 The response contains an identifier of the search being executed.
 You can use this ID to later retrieve the search's final results.
@@ -69,12 +68,17 @@ results are returned as part of the <<search-api-response-body,`response`>> obje
 }
 --------------------------------------------------
 // TESTRESPONSE[s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/$body.id/]
+// TESTRESPONSE[s/"is_partial" : true/"is_partial": $body.is_partial/]
+// TESTRESPONSE[s/"is_running" : true/"is_running": $body.is_running/]
 // TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/]
 // TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/]
 // TESTRESPONSE[s/"took" : 1122/"took": $body.response.took/]
+// TESTRESPONSE[s/"num_reduce_phases" : 0,//]
 // TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/]
 // TESTRESPONSE[s/"successful" : 3/"successful": $body.response._shards.successful/]
 // TESTRESPONSE[s/"value" : 157483/"value": $body.response.hits.total.value/]
+// TESTRESPONSE[s/"relation" : "gte"/"relation": $body.response.hits.total.relation/]
+// TESTRESPONSE[s/"hits" : \[ \]\n\s\s\s\s\}/"hits" : \[\]},"aggregations":  $body.response.aggregations/]
 
 <1> Identifier of the async search that can be used to monitor its progress, retrieve its results, and/or delete it.
 <2> Whether the returned search results are partial or final
@@ -141,7 +145,7 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
   "response" : {
     "took" : 12144,
     "timed_out" : false,
-    "num_reduce_phases" : 38, <4>
+    "num_reduce_phases" : 46, <4>
     "_shards" : {
       "total" : 562, <5>
       "successful" : 188,
@@ -174,7 +178,7 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
 // TESTRESPONSE[s/"successful" : 188/"successful": $body.response._shards.successful/]
 // TESTRESPONSE[s/"value" : 456433/"value": $body.response.hits.total.value/]
 // TESTRESPONSE[s/"buckets" : \[\]/"buckets": $body.response.aggregations.sale_date.buckets/]
-// TESTRESPONSE[s/"num_reduce_phases" : 38,//]
+// TESTRESPONSE[s/"num_reduce_phases" : 46,//]
 
 <1> Whether the returned search results are partial or final
 <2> Whether the search is still being executed or it has completed

From 87c910b36f886d3e3b83c65270cef7d7aa3aaa5a Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 23 Mar 2020 14:24:58 +0100
Subject: [PATCH 206/285] Better Incrementality for Snapshots of Unchanged
 Shards (#52182)

Use sequence numbers and force merge UUID to determine whether a shard has changed or not instead before falling back to comparing files to get incremental snapshots on primary fail-over.
---
 .../BlobStoreIndexShardSnapshots.java         |  18 +-
 .../snapshots/blobstore/SnapshotFiles.java    |  21 +-
 .../repositories/FilterRepository.java        |   8 +-
 .../repositories/Repository.java              |   8 +-
 .../blobstore/BlobStoreRepository.java        | 137 ++++++-----
 .../snapshots/SnapshotShardsService.java      |  30 ++-
 .../RepositoriesServiceTests.java             |   4 +-
 .../repositories/fs/FsRepositoryTests.java    |   8 +-
 .../snapshots/BlobStoreIncrementalityIT.java  | 219 ++++++++++++++++++
 .../RepositoryFilterUserMetadataIT.java       |  10 +-
 .../index/shard/IndexShardTestCase.java       |   3 +-
 .../index/shard/RestoreOnlyRepository.java    |   4 +-
 .../xpack/ccr/repository/CcrRepository.java   |   6 +-
 .../SourceOnlySnapshotRepository.java         |   8 +-
 .../SourceOnlySnapshotShardTests.java         |  14 +-
 15 files changed, 397 insertions(+), 101 deletions(-)
 create mode 100644 server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java

diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
index 309a3c3a38694..c34a449ad7f79 100644
--- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
+++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
@@ -135,6 +135,7 @@ static final class Fields {
 
     static final class ParseFields {
         static final ParseField FILES = new ParseField("files");
+        static final ParseField SHARD_STATE_ID = new ParseField("shard_state_id");
         static final ParseField SNAPSHOTS = new ParseField("snapshots");
     }
 
@@ -207,6 +208,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.value(fileInfo.name());
             }
             builder.endArray();
+            if (snapshot.shardStateIdentifier() != null) {
+                builder.field(ParseFields.SHARD_STATE_ID.getPreferredName(), snapshot.shardStateIdentifier());
+            }
             builder.endObject();
         }
         builder.endObject();
@@ -219,6 +223,8 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
             token = parser.nextToken();
         }
         Map<String, List<String>> snapshotsMap = new HashMap<>();
+        Map<String, String> historyUUIDs = new HashMap<>();
+        Map<String, Long> globalCheckpoints = new HashMap<>();
         Map<String, FileInfo> files = new HashMap<>();
         if (token == XContentParser.Token.START_OBJECT) {
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -250,15 +256,16 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                             if (token == XContentParser.Token.FIELD_NAME) {
                                 currentFieldName = parser.currentName();
-                                if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
-                                    if (ParseFields.FILES.match(currentFieldName, parser.getDeprecationHandler()) == false) {
-                                        throw new ElasticsearchParseException("unknown array [{}]", currentFieldName);
-                                    }
+                                if (ParseFields.FILES.match(currentFieldName, parser.getDeprecationHandler()) &&
+                                    parser.nextToken() == XContentParser.Token.START_ARRAY) {
                                     List<String> fileNames = new ArrayList<>();
                                     while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
                                         fileNames.add(parser.text());
                                     }
                                     snapshotsMap.put(snapshot, fileNames);
+                                } else if (ParseFields.SHARD_STATE_ID.match(currentFieldName, parser.getDeprecationHandler())) {
+                                    parser.nextToken();
+                                    historyUUIDs.put(snapshot, parser.text());
                                 }
                             }
                         }
@@ -277,7 +284,8 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
                 assert fileInfo != null;
                 fileInfosBuilder.add(fileInfo);
             }
-            snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder)));
+            snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder),
+                historyUUIDs.get(entry.getKey())));
         }
         return new BlobStoreIndexShardSnapshots(files, Collections.unmodifiableList(snapshots));
     }
diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java
index 5da7b2b1f8c6b..039ea9405a158 100644
--- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java
+++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.snapshots.blobstore;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
 
 import java.util.HashMap;
@@ -33,6 +34,9 @@ public class SnapshotFiles {
 
     private final List<FileInfo> indexFiles;
 
+    @Nullable
+    private final String shardStateIdentifier;
+
     private Map<String, FileInfo> physicalFiles = null;
 
     /**
@@ -45,12 +49,23 @@ public String snapshot() {
     }
 
     /**
-     * @param snapshot   snapshot name
-     * @param indexFiles index files
+     * @param snapshot             snapshot name
+     * @param indexFiles           index files
+     * @param shardStateIdentifier unique identifier for the state of the shard that this snapshot was taken from
      */
-    public SnapshotFiles(String snapshot, List<FileInfo> indexFiles ) {
+    public SnapshotFiles(String snapshot, List<FileInfo> indexFiles, @Nullable String shardStateIdentifier) {
         this.snapshot = snapshot;
         this.indexFiles = indexFiles;
+        this.shardStateIdentifier = shardStateIdentifier;
+    }
+
+    /**
+     * Returns an identifier for the shard state that can be used to check whether a shard has changed between
+     * snapshots or not.
+     */
+    @Nullable
+    public String shardStateIdentifier() {
+        return shardStateIdentifier;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
index f41137c763289..ec708976f4e75 100644
--- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
@@ -121,10 +121,10 @@ public boolean isReadOnly() {
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
-        in.snapshotShard(
-            store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, repositoryMetaVersion, userMetadata, listener);
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
+        in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, shardStateIdentifier, snapshotStatus,
+            repositoryMetaVersion, userMetadata, listener);
     }
     @Override
     public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState,
diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java
index bca034564bdd9..c2e93c172eb94 100644
--- a/server/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
@@ -199,14 +200,17 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations,
      * @param snapshotId            snapshot id
      * @param indexId               id for the index being snapshotted
      * @param snapshotIndexCommit   commit point
+     * @param shardStateIdentifier  a unique identifier of the state of the shard that is stored with the shard's snapshot and used
+     *                              to detect if the shard has changed between snapshots. If {@code null} is passed as the identifier
+     *                              snapshotting will be done by inspecting the physical files referenced by {@code snapshotIndexCommit}
      * @param snapshotStatus        snapshot status
      * @param repositoryMetaVersion version of the updated repository metadata to write
      * @param userMetadata          user metadata of the snapshot found in {@link SnapshotsInProgress.Entry#userMetadata()}
      * @param listener              listener invoked on completion
      */
     void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit,
-                       IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map<String, Object> userMetadata,
-                       ActionListener<String> listener);
+                       @Nullable String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
+                       Map<String, Object> userMetadata, ActionListener<String> listener);
 
     /**
      * Restores snapshot of the shard.
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index fdb70cffb133f..7102bbaa943f9 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -125,6 +125,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
@@ -1534,8 +1535,8 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
         final ShardId shardId = store.shardId();
         final long startTime = threadPool.absoluteTimeInMillis();
         try {
@@ -1561,76 +1562,92 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s
                 throw new IndexShardSnapshotFailedException(shardId,
                     "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting");
             }
-
-            final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
-            final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
-            store.incRef();
-            final Collection<String> fileNames;
-            final Store.MetadataSnapshot metadataFromStore;
-            try {
-                // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
-                try {
-                    logger.trace(
-                        "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
-                    metadataFromStore = store.getMetadata(snapshotIndexCommit);
-                    fileNames = snapshotIndexCommit.getFileNames();
-                } catch (IOException e) {
-                    throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
+            // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository
+            final List<BlobStoreIndexShardSnapshot.FileInfo> filesFromSegmentInfos = Optional.ofNullable(shardStateIdentifier).map(id -> {
+                for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) {
+                    if (id.equals(snapshotFileSet.shardStateIdentifier())) {
+                        return snapshotFileSet.indexFiles();
+                    }
                 }
-            } finally {
-                store.decRef();
-            }
+                return null;
+            }).orElse(null);
+
+            final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles;
             int indexIncrementalFileCount = 0;
             int indexTotalNumberOfFiles = 0;
             long indexIncrementalSize = 0;
-            long indexTotalFileCount = 0;
-            for (String fileName : fileNames) {
-                if (snapshotStatus.isAborted()) {
-                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
-                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+            long indexTotalFileSize = 0;
+            final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
+            // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files
+            // in the commit with files already in the repository
+            if (filesFromSegmentInfos == null) {
+                indexCommitPointFiles = new ArrayList<>();
+                store.incRef();
+                final Collection<String> fileNames;
+                final Store.MetadataSnapshot metadataFromStore;
+                try {
+                    // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
+                    try {
+                        logger.trace(
+                            "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
+                        metadataFromStore = store.getMetadata(snapshotIndexCommit);
+                        fileNames = snapshotIndexCommit.getFileNames();
+                    } catch (IOException e) {
+                        throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
+                    }
+                } finally {
+                    store.decRef();
                 }
+                for (String fileName : fileNames) {
+                    if (snapshotStatus.isAborted()) {
+                        logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
+                        throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+                    }
 
-                logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
-                final StoreFileMetaData md = metadataFromStore.get(fileName);
-                BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
-                List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
-                if (filesInfo != null) {
-                    for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
-                        if (fileInfo.isSame(md)) {
-                            // a commit point file with the same name, size and checksum was already copied to repository
-                            // we will reuse it for this snapshot
-                            existingFileInfo = fileInfo;
-                            break;
+                    logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
+                    final StoreFileMetaData md = metadataFromStore.get(fileName);
+                    BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
+                    List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
+                    if (filesInfo != null) {
+                        for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
+                            if (fileInfo.isSame(md)) {
+                                // a commit point file with the same name, size and checksum was already copied to repository
+                                // we will reuse it for this snapshot
+                                existingFileInfo = fileInfo;
+                                break;
+                            }
                         }
                     }
-                }
 
-                // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
-                // directly in the shard level metadata in this case
-                final boolean needsWrite = md.hashEqualsContents() == false;
-                indexTotalFileCount += md.length();
-                indexTotalNumberOfFiles++;
-
-                if (existingFileInfo == null) {
-                    indexIncrementalFileCount++;
-                    indexIncrementalSize += md.length();
-                    // create a new FileInfo
-                    BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo =
-                        new BlobStoreIndexShardSnapshot.FileInfo(
-                            (needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(),
-                            md, chunkSize());
-                    indexCommitPointFiles.add(snapshotFileInfo);
-                    if (needsWrite) {
-                        filesToSnapshot.add(snapshotFileInfo);
+                    // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
+                    // directly in the shard level metadata in this case
+                    final boolean needsWrite = md.hashEqualsContents() == false;
+                    indexTotalFileSize += md.length();
+                    indexTotalNumberOfFiles++;
+
+                    if (existingFileInfo == null) {
+                        indexIncrementalFileCount++;
+                        indexIncrementalSize += md.length();
+                        // create a new FileInfo
+                        BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo =
+                            new BlobStoreIndexShardSnapshot.FileInfo(
+                                (needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(),
+                                md, chunkSize());
+                        indexCommitPointFiles.add(snapshotFileInfo);
+                        if (needsWrite) {
+                            filesToSnapshot.add(snapshotFileInfo);
+                        }
+                        assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store);
+                    } else {
+                        indexCommitPointFiles.add(existingFileInfo);
                     }
-                    assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store);
-                } else {
-                    indexCommitPointFiles.add(existingFileInfo);
                 }
+            } else {
+                indexCommitPointFiles = filesFromSegmentInfos;
             }
 
             snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount,
-                indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileCount);
+                indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileSize);
 
             final StepListener<Collection<Void>> allFilesUploadedListener = new StepListener<>();
             allFilesUploadedListener.whenComplete(v -> {
@@ -1655,7 +1672,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s
                 }
                 // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
                 List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
-                newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
+                newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), shardStateIdentifier));
                 for (SnapshotFiles point : snapshots) {
                     newSnapshotsList.add(point);
                 }
@@ -1742,7 +1759,7 @@ public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, Sh
         final BlobContainer container = shardContainer(indexId, snapshotShardId);
         executor.execute(ActionRunnable.wrap(restoreListener, l -> {
             final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId);
-            final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
+            final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), null);
             new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) {
                 @Override
                 protected void restoreFiles(List<BlobStoreIndexShardSnapshot.FileInfo> filesToRecover, Store store,
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 33ec18da99660..a39c2691a9db4 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.index.IndexCommit;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -55,6 +56,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
@@ -340,8 +342,9 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde
             try {
                 // we flush first to make sure we get the latest writes snapshotted
                 snapshotRef = indexShard.acquireLastIndexCommit(true);
+                final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit();
                 repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId,
-                    snapshotRef.getIndexCommit(), snapshotStatus, version, userMetadata,
+                    snapshotRef.getIndexCommit(), getShardStateId(indexShard, snapshotIndexCommit), snapshotStatus, version, userMetadata,
                     ActionListener.runBefore(listener, snapshotRef::close));
             } catch (Exception e) {
                 IOUtils.close(snapshotRef);
@@ -352,6 +355,31 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde
         }
     }
 
+    /**
+     * Generates an identifier from the current state of a shard that can be used to detect whether a shard's contents
+     * have changed between two snapshots.
+     * A shard is assumed to have unchanged contents if its global- and local checkpoint are equal, its maximum
+     * sequence number has not changed and its history- and force-merge-uuid have not changed.
+     * The method returns {@code null} if global and local checkpoint are different for a shard since no safe unique
+     * shard state id can be used in this case because of the possibility of a primary failover leading to different
+     * shard content for the same sequence number on a subsequent snapshot.
+     *
+     * @param indexShard          Shard
+     * @param snapshotIndexCommit IndexCommit for shard
+     * @return shard state id or {@code null} if none can be used
+     */
+    @Nullable
+    private static String getShardStateId(IndexShard indexShard, IndexCommit snapshotIndexCommit) throws IOException {
+        final Map<String, String> userCommitData = snapshotIndexCommit.getUserData();
+        final SequenceNumbers.CommitInfo seqNumInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userCommitData.entrySet());
+        final long maxSeqNo = seqNumInfo.maxSeqNo;
+        if (maxSeqNo != seqNumInfo.localCheckpoint || maxSeqNo != indexShard.getLastSyncedGlobalCheckpoint()) {
+            return null;
+        }
+        return userCommitData.get(Engine.HISTORY_UUID_KEY) + "-" +
+            userCommitData.getOrDefault(Engine.FORCE_MERGE_UUID_KEY, "na") + "-" + maxSeqNo;
+    }
+
     /**
      * Checks if any shards were processed that the new master doesn't know about
      */
diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
index 8778cbdc8689f..083872ce83768 100644
--- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
@@ -201,8 +201,8 @@ public boolean isReadOnly() {
 
         @Override
         public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                                  IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                                  Map<String, Object> userMetadata, ActionListener<String> listener) {
+                                  IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                                  Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
 
         }
 
diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
index 49ca709118074..22b704389aa26 100644
--- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
@@ -105,8 +105,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException {
             final PlainActionFuture<String> future1 = PlainActionFuture.newFuture();
             runGeneric(threadPool, () -> {
                 IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null);
-                repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, snapshotStatus, Version.CURRENT,
-                    Collections.emptyMap(), future1);
+                repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, null,
+                    snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1);
                 future1.actionGet();
                 IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
                 assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
@@ -134,8 +134,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException {
             final PlainActionFuture<String> future2 = PlainActionFuture.newFuture();
             runGeneric(threadPool, () -> {
                 IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
-                repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, snapshotStatus, Version.CURRENT,
-                    Collections.emptyMap(), future2);
+                repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit,
+                    null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2);
                 future2.actionGet();
                 IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
                 assertEquals(2, copy.getIncrementalFileCount());
diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
new file mode 100644
index 0000000000000..897e75a2e0d9f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase {
+
+    public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedException, ExecutionException, IOException {
+        internalCluster().startMasterOnlyNode();
+        final String primaryNode = internalCluster().startDataOnlyNode();
+        final String indexName = "test-index";
+        createIndex(indexName, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0).build());
+        ensureYellow(indexName);
+        final String newPrimary = internalCluster().startDataOnlyNode();
+        final Collection<String> toDelete = new ArrayList<>();
+        ensureGreen(indexName);
+
+        logger.info("--> adding some documents to test index");
+        for (int j = 0; j < randomIntBetween(1, 10); ++j) {
+            final BulkRequest bulkRequest = new BulkRequest();
+            for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) {
+                bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i));
+            }
+            final BulkResponse bulkResponse = client().bulk(bulkRequest).get();
+            for (BulkItemResponse item : bulkResponse.getItems()) {
+                if (randomBoolean()) {
+                    toDelete.add(item.getId());
+                }
+            }
+        }
+        refresh(indexName);
+
+        final long documentCountOriginal = getCountForIndex(indexName);
+
+        final String snapshot1 = "snap-1";
+        final String repo = "test-repo";
+        logger.info("--> creating repository");
+        assertThat(client().admin().cluster().preparePutRepository(repo)
+                .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(),
+            is(true));
+
+        logger.info("--> creating snapshot 1");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> Shutting down initial primary node [{}]", primaryNode);
+        stopNode(primaryNode);
+
+        ensureYellow(indexName);
+        final String snapshot2 = "snap-2";
+        logger.info("--> creating snapshot 2");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get();
+
+        assertTwoIdenticalShardSnapshots(repo, indexName, snapshot1, snapshot2);
+
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot1, "-copy-1");
+        assertCountInIndexThenDelete(indexName + "-copy-1", documentCountOriginal);
+
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot2, "-copy-2");
+        assertCountInIndexThenDelete(indexName + "-copy-2", documentCountOriginal);
+
+        internalCluster().startDataOnlyNode();
+        ensureGreen(indexName);
+
+        logger.info("--> delete some documents from test index");
+        for (final String id : toDelete) {
+            assertThat(client().prepareDelete(indexName, id).get().getResult(), is(DocWriteResponse.Result.DELETED));
+        }
+
+        refresh(indexName);
+
+        final String snapshot3 = "snap-3";
+        logger.info("--> creating snapshot 3");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot3).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> Shutting down new primary node [{}]", newPrimary);
+        stopNode(newPrimary);
+        ensureYellow(indexName);
+
+        final String snapshot4 = "snap-4";
+        logger.info("--> creating snapshot 4");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot4).setIndices(indexName).setWaitForCompletion(true).get();
+
+        assertTwoIdenticalShardSnapshots(repo, indexName, snapshot3, snapshot4);
+
+        final long countAfterDelete = documentCountOriginal - toDelete.size();
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot3, "-copy-3");
+        assertCountInIndexThenDelete(indexName + "-copy-3", countAfterDelete);
+
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot4, "-copy-4");
+        assertCountInIndexThenDelete(indexName + "-copy-4", countAfterDelete);
+    }
+
+    public void testForceMergeCausesFullSnapshot() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        final String indexName = "test-index";
+        createIndex(indexName, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build());
+        ensureGreen(indexName);
+
+        logger.info("--> adding some documents to test index and flush in between to get at least two segments");
+        for (int j = 0; j < 2; j++) {
+            final BulkRequest bulkRequest = new BulkRequest();
+            for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) {
+                bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i));
+            }
+            client().bulk(bulkRequest).get();
+            flushAndRefresh(indexName);
+        }
+        final IndexStats indexStats = client().admin().indices().prepareStats(indexName).get().getIndex(indexName);
+        assertThat(indexStats.getIndexShards().get(0).getPrimary().getSegments().getCount(), greaterThan(1L));
+
+        final String snapshot1 = "snap-1";
+        final String repo = "test-repo";
+        logger.info("--> creating repository");
+        assertThat(client().admin().cluster().preparePutRepository(repo)
+                .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(),
+            is(true));
+
+        logger.info("--> creating snapshot 1");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> force merging down to a single segment");
+        final ForceMergeResponse forceMergeResponse =
+            client().admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).setFlush(true).get();
+        assertThat(forceMergeResponse.getFailedShards(), is(0));
+
+        final String snapshot2 = "snap-2";
+        logger.info("--> creating snapshot 2");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> asserting that the two snapshots refer to different files in the repository");
+        final SnapshotsStatusResponse response =
+            client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot2).get();
+        final SnapshotStats secondSnapshotShardStatus =
+            response.getSnapshots().get(0).getIndices().get(indexName).getShards().get(0).getStats();
+        assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), greaterThan(0));
+    }
+
+    private void assertCountInIndexThenDelete(String index, long expectedCount) throws ExecutionException, InterruptedException {
+        logger.info("--> asserting that index [{}] contains [{}] documents", index, expectedCount);
+        assertThat(getCountForIndex(index), is(expectedCount));
+        logger.info("--> deleting index [{}]", index);
+        assertThat(client().admin().indices().prepareDelete(index).get().isAcknowledged(), is(true));
+    }
+
+    private void assertTwoIdenticalShardSnapshots(String repo, String indexName, String snapshot1, String snapshot2) {
+        logger.info(
+            "--> asserting that snapshots [{}] and [{}] are referring to the same files in the repository", snapshot1, snapshot2);
+        final SnapshotsStatusResponse response =
+            client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot1, snapshot2).get();
+        final SnapshotStats firstSnapshotShardStatus =
+            response.getSnapshots().get(0).getIndices().get(indexName).getShards().get(0).getStats();
+        final int totalFilesInShard = firstSnapshotShardStatus.getTotalFileCount();
+        assertThat(totalFilesInShard, greaterThan(0));
+        assertThat(firstSnapshotShardStatus.getIncrementalFileCount(), is(totalFilesInShard));
+        final SnapshotStats secondSnapshotShardStatus =
+            response.getSnapshots().get(1).getIndices().get(indexName).getShards().get(0).getStats();
+        assertThat(secondSnapshotShardStatus.getTotalFileCount(), is(totalFilesInShard));
+        assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), is(0));
+    }
+
+    private void ensureRestoreSingleShardSuccessfully(String repo, String indexName, String snapshot, String indexSuffix) {
+        logger.info("--> restoring [{}]", snapshot);
+        final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot)
+            .setIndices(indexName).setRenamePattern("(.+)").setRenameReplacement("$1" + indexSuffix).setWaitForCompletion(true).get();
+        final RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo();
+        assertThat(restoreInfo.totalShards(), is(1));
+        assertThat(restoreInfo.failedShards(), is(0));
+    }
+
+    private long getCountForIndex(String indexName) throws ExecutionException, InterruptedException {
+        return client().search(new SearchRequest(new SearchRequest(indexName).source(
+            new SearchSourceBuilder().size(0).trackTotalHits(true)))).get().getHits().getTotalHits().value;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
index 1f09afdc76908..62eeea8b24f34 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
@@ -95,12 +95,12 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera
 
                     @Override
                     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus,
-                                              Version repositoryMetaVersion, Map<String, Object> userMetadata,
-                                              ActionListener<String> listener) {
+                                              IndexCommit snapshotIndexCommit, String shardStateIdentifier,
+                                              IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
+                                              Map<String, Object> userMetadata, ActionListener<String> listener) {
                         assertThat(userMetadata, is(Collections.singletonMap(MOCK_FILTERED_META, initialMetaValue)));
-                        super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus,
-                            repositoryMetaVersion, userMetadata, listener);
+                        super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, shardStateIdentifier,
+                            snapshotStatus, repositoryMetaVersion, userMetadata, listener);
                     }
 
                     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index e73c145c3b5b8..b95281530d844 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -838,7 +838,8 @@ protected String snapshotShard(final IndexShard shard,
         final String shardGen;
         try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(true)) {
             repository.snapshotShard(shard.store(), shard.mapperService(), snapshot.getSnapshotId(), indexId,
-                indexCommitRef.getIndexCommit(), snapshotStatus, Version.CURRENT, Collections.emptyMap(), future);
+                indexCommitRef.getIndexCommit(), null, snapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future);
             shardGen = future.actionGet();
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
index 4fae0fc14952a..9d697d04dc4ed 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
@@ -133,8 +133,8 @@ public boolean isReadOnly() {
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
index 7f4c045f22643..d4203e335994c 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
@@ -294,8 +294,8 @@ public boolean isReadOnly() {
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
         throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE);
     }
 
@@ -487,7 +487,7 @@ void restoreFiles(Store store, ActionListener<Void> listener) {
                 ByteSizeValue fileSize = new ByteSizeValue(fileMetaData.length());
                 fileInfos.add(new FileInfo(fileMetaData.name(), fileMetaData, fileSize));
             }
-            SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos);
+            SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos, null);
             restore(snapshotFiles, store, listener);
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
index e65fbdace62d6..7fa8bd27b4cfc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
@@ -120,8 +120,8 @@ private static MetaData metadataToSnapshot(Collection<IndexId> indices, MetaData
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
         if (mapperService.documentMapper() != null // if there is no mapping this is null
             && mapperService.documentMapper().sourceMapper().isComplete() == false) {
             listener.onFailure(
@@ -161,8 +161,8 @@ protected void closeInternal() {
                 Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name()));
             toClose.add(reader);
             IndexCommit indexCommit = reader.getIndexCommit();
-            super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, repositoryMetaVersion,
-                userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose)));
+            super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, shardStateIdentifier, snapshotStatus,
+                repositoryMetaVersion, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose)));
         } catch (IOException e) {
             try {
                 IOUtils.close(toClose);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
index 62b1a43e4ef95..443b4630740a4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
@@ -102,7 +102,8 @@ public void testSourceIncomplete() throws IOException {
             IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing("-1");
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, future::actionGet);
             assertEquals(
                 "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source",
@@ -128,7 +129,8 @@ public void testIncrementalSnapshot() throws IOException {
             SnapshotId snapshotId = new SnapshotId("test", "test");
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             shardGeneration = future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
@@ -144,7 +146,8 @@ public void testIncrementalSnapshot() throws IOException {
             IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             shardGeneration = future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             // we processed the segments_N file plus _1.si, _1.fnm, _1.fdx, _1.fdt, _1.fdm
@@ -160,7 +163,8 @@ public void testIncrementalSnapshot() throws IOException {
             IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             // we processed the segments_N file plus _1_1.liv
@@ -208,7 +212,7 @@ public void testRestoreMinmal() throws IOException {
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> {
                 repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(),
-                    indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future);
+                    null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future);
                 future.actionGet();
                 final PlainActionFuture<SnapshotInfo> finFuture = PlainActionFuture.newFuture();
                 repository.finalizeSnapshot(snapshotId,

From face37514bfcde744b31644a350c7a532f3ed5b4 Mon Sep 17 00:00:00 2001
From: weizijun <weizijun1989@gmail.com>
Date: Mon, 23 Mar 2020 21:27:13 +0800
Subject: [PATCH 207/285] /_cat/shards support path stats (#53461)

* _cat/shards support path stats

* fix some style case

* fix some style case

* fix rest-api-spec cat.shards error

* fix rest-api-spec cat.shards bwc error

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../test/cat.shards/10_basic.yml              |   6 +-
 .../rest/action/cat/RestShardsAction.java     |   9 +-
 .../action/cat/RestShardsActionTests.java     | 121 ++++++++++++++++++
 3 files changed, 133 insertions(+), 3 deletions(-)
 create mode 100644 server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
index de5e632975752..aa4abc7a11eae 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
@@ -1,8 +1,8 @@
 ---
 "Help":
   - skip:
-      version: " - 7.1.99"
-      reason:  external refresh stats were added in 7.2.0
+      version: " - 7.99.99"
+      reason:  shard path stats were added in 8.0.0
   - do:
       cat.shards:
         help: true
@@ -78,6 +78,8 @@
                     warmer.current                   .+   \n
                     warmer.total                     .+   \n
                     warmer.total_time                .+   \n
+                    path.data                        .+   \n
+                    path.state                       .+   \n
                $/
 ---
 "Test cat shards output":
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 40971a425e07e..fd83b2d34bea9 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -200,6 +200,9 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("warmer.total", "alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops");
         table.addCell("warmer.total_time", "alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers");
 
+        table.addCell("path.data", "alias:pd,dataPath;default:false;text-align:right;desc:shard data path");
+        table.addCell("path.state", "alias:ps,statsPath;default:false;text-align:right;desc:shard state path");
+
         table.endHeaders();
         return table;
     }
@@ -214,7 +217,8 @@ private static <S, T> Object getOrNull(S stats, Function<S, T> accessor, Functio
         return null;
     }
 
-    private Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
+    // package private for testing
+    Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
         Table table = getTableWithHeader(request);
 
         for (ShardRouting shard : state.getState().routingTable().allShards()) {
@@ -351,6 +355,9 @@ private Table buildTable(RestRequest request, ClusterStateResponse state, Indice
             table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
             table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
 
+            table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
+            table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
+
             table.endRow();
         }
 
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
new file mode 100644
index 0000000000000..f661521376cf5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.cat;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RestShardsActionTests extends ESTestCase {
+
+    public void testBuildTable() {
+        final int numShards = randomIntBetween(1, 5);
+        DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT);
+
+        List<ShardRouting> shardRoutings = new ArrayList<>(numShards);
+        Map<ShardRouting, ShardStats> shardStatsMap = new HashMap<>();
+        String index = "index";
+        for (int i = 0; i < numShards; i++) {
+            ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3));
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(index, i, localNode.getId(), randomBoolean(), shardRoutingState);
+            Path path = createTempDir().resolve("indices").resolve(shardRouting.shardId().getIndex().getUUID())
+                .resolve(String.valueOf(shardRouting.shardId().id()));
+            ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardRouting.shardId()),
+                null, null, null, null);
+            shardStatsMap.put(shardRouting, shardStats);
+            shardRoutings.add(shardRouting);
+        }
+
+        IndexStats indexStats = mock(IndexStats.class);
+        when(indexStats.getPrimaries()).thenReturn(new CommonStats());
+        when(indexStats.getTotal()).thenReturn(new CommonStats());
+
+        IndicesStatsResponse stats = mock(IndicesStatsResponse.class);
+        when(stats.asMap()).thenReturn(shardStatsMap);
+
+        DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
+        when(discoveryNodes.get(localNode.getId())).thenReturn(localNode);
+
+        ClusterStateResponse state = mock(ClusterStateResponse.class);
+        RoutingTable routingTable = mock(RoutingTable.class);
+        when(routingTable.allShards()).thenReturn(shardRoutings);
+        ClusterState clusterState = mock(ClusterState.class);
+        when(clusterState.routingTable()).thenReturn(routingTable);
+        when(clusterState.nodes()).thenReturn(discoveryNodes);
+        when(state.getState()).thenReturn(clusterState);
+
+        final RestShardsAction action = new RestShardsAction();
+        final Table table = action.buildTable(new FakeRestRequest(), state, stats);
+
+        // now, verify the table is correct
+        List<Table.Cell> headers = table.getHeaders();
+        assertThat(headers.get(0).value, equalTo("index"));
+        assertThat(headers.get(1).value, equalTo("shard"));
+        assertThat(headers.get(2).value, equalTo("prirep"));
+        assertThat(headers.get(3).value, equalTo("state"));
+        assertThat(headers.get(4).value, equalTo("docs"));
+        assertThat(headers.get(5).value, equalTo("store"));
+        assertThat(headers.get(6).value, equalTo("ip"));
+        assertThat(headers.get(7).value, equalTo("id"));
+        assertThat(headers.get(8).value, equalTo("node"));
+
+        final List<List<Table.Cell>> rows = table.getRows();
+        assertThat(rows.size(), equalTo(numShards));
+
+        Iterator<ShardRouting> shardRoutingsIt = shardRoutings.iterator();
+        for (final List<Table.Cell> row : rows) {
+            ShardRouting shardRouting = shardRoutingsIt.next();
+            ShardStats shardStats = shardStatsMap.get(shardRouting);
+            assertThat(row.get(0).value, equalTo(shardRouting.getIndexName()));
+            assertThat(row.get(1).value, equalTo(shardRouting.getId()));
+            assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r"));
+            assertThat(row.get(3).value, equalTo(shardRouting.state()));
+            assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
+            assertThat(row.get(7).value, equalTo(localNode.getId()));
+            assertThat(row.get(69).value, equalTo(shardStats.getDataPath()));
+            assertThat(row.get(70).value, equalTo(shardStats.getStatePath()));
+        }
+    }
+}

From 3ceb60becfce3425e114537d886a327f16800890 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Mon, 23 Mar 2020 15:01:35 +0100
Subject: [PATCH 208/285] Add async_search get and delete APIs to HLRC (#53828)

This commit adds the "_async_searhc" get and delete APIs to the
AsyncSearchClient in the High Level Rest Client.

Relates to #49091
---
 .../client/AsyncSearchClient.java             | 58 +++++++++++-
 .../client/AsyncSearchRequestConverters.java  | 29 ++++++
 .../asyncsearch/DeleteAsyncSearchRequest.java | 55 +++++++++++
 .../asyncsearch/GetAsyncSearchRequest.java    | 93 +++++++++++++++++++
 .../AsyncSearchRequestConvertersTests.java    | 35 +++++++
 .../client/asyncsearch/AsyncSearchIT.java     | 48 ++++++----
 .../GetAsyncSearchRequestTests.java           | 41 ++++++++
 .../xpack/search/AsyncSearchIndexService.java |  2 +-
 8 files changed, 341 insertions(+), 20 deletions(-)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
index ab47b1c5e792a..bbf9c15e20cdf 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
@@ -21,7 +21,10 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.asyncsearch.AsyncSearchResponse;
+import org.elasticsearch.client.asyncsearch.DeleteAsyncSearchRequest;
+import org.elasticsearch.client.asyncsearch.GetAsyncSearchRequest;
 import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
+import org.elasticsearch.client.core.AcknowledgedResponse;
 
 import java.io.IOException;
 
@@ -42,7 +45,7 @@ public class AsyncSearchClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request, RequestOptions options) throws IOException {
+    public AsyncSearchResponse submit(SubmitAsyncSearchRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::submitAsyncSearch, options,
                 AsyncSearchResponse::fromXContent, emptySet());
     }
@@ -57,10 +60,61 @@ public AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request, R
      * @param listener the listener to be notified upon request completion
      * @return cancellable that may be used to cancel the request
      */
-    public Cancellable submitAsyncSearchAsync(SubmitAsyncSearchRequest request, RequestOptions options,
+    public Cancellable submitAsync(SubmitAsyncSearchRequest request, RequestOptions options,
                                                ActionListener<AsyncSearchResponse> listener) {
         return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::submitAsyncSearch, options,
                 AsyncSearchResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Get an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     *
+     */
+    public AsyncSearchResponse get(GetAsyncSearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::getAsyncSearch, options,
+                AsyncSearchResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable getAsync(GetAsyncSearchRequest request, RequestOptions options,
+            ActionListener<AsyncSearchResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::getAsyncSearch, options,
+                AsyncSearchResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Delete an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse delete(DeleteAsyncSearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::deleteAsyncSearch, options,
+                AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously delete an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable deleteAsync(DeleteAsyncSearchRequest request, RequestOptions options,
+                                               ActionListener<AcknowledgedResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::deleteAsyncSearch, options,
+                AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
index 2d91bd926ef4d..8a63589a55c51 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.client.RequestConverters.Params;
+import org.elasticsearch.client.asyncsearch.DeleteAsyncSearchRequest;
+import org.elasticsearch.client.asyncsearch.GetAsyncSearchRequest;
 import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 
@@ -71,4 +75,29 @@ static void addSearchRequestParams(Params params, SubmitAsyncSearchRequest reque
         }
         params.withBatchedReduceSize(request.getBatchedReduceSize());
     }
+
+    static Request getAsyncSearch(GetAsyncSearchRequest asyncSearchRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_async_search")
+                .addPathPart(asyncSearchRequest.getId())
+                .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params params = new RequestConverters.Params();
+        if (asyncSearchRequest.getKeepAlive() != null) {
+            params.putParam("keep_alive", asyncSearchRequest.getKeepAlive().getStringRep());
+        }
+        if (asyncSearchRequest.getWaitForCompletion() != null) {
+            params.putParam("wait_for_completion", asyncSearchRequest.getWaitForCompletion().getStringRep());
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request deleteAsyncSearch(DeleteAsyncSearchRequest deleteAsyncSearchRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_async_search")
+                .addPathPart(deleteAsyncSearchRequest.getId())
+                .build();
+        return new Request(HttpDelete.METHOD_NAME, endpoint);
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
new file mode 100644
index 0000000000000..3b37293212da0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+public class DeleteAsyncSearchRequest implements Validatable {
+
+    private final String id;
+
+    public DeleteAsyncSearchRequest(String id) {
+        this.id = id;
+}
+
+    public String getId() {
+        return this.id;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        DeleteAsyncSearchRequest request = (DeleteAsyncSearchRequest) o;
+        return Objects.equals(getId(), request.getId());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
new file mode 100644
index 0000000000000..11ad059349481
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.Objects;
+import java.util.Optional;
+
+public class GetAsyncSearchRequest implements Validatable {
+
+    private TimeValue waitForCompletion;
+    private TimeValue keepAlive;
+
+    public static final long MIN_KEEPALIVE = TimeValue.timeValueMinutes(1).millis();
+
+    private final String id;
+
+    public GetAsyncSearchRequest(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return this.id;
+    }
+
+    public TimeValue getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    public void setWaitForCompletion(TimeValue waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+    }
+
+    public TimeValue getKeepAlive() {
+        return keepAlive;
+    }
+
+    public void setKeepAlive(TimeValue keepAlive) {
+        this.keepAlive = keepAlive;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        final ValidationException validationException = new ValidationException();
+        if (keepAlive != null && keepAlive.getMillis() < MIN_KEEPALIVE) {
+            validationException.addValidationError("keep_alive must be greater than 1 minute, got: " + keepAlive.toString());
+        }
+        if (validationException.validationErrors().isEmpty()) {
+            return Optional.empty();
+        }
+        return Optional.of(validationException);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetAsyncSearchRequest request = (GetAsyncSearchRequest) o;
+        return Objects.equals(getId(), request.getId())
+                && Objects.equals(getKeepAlive(), request.getKeepAlive())
+                && Objects.equals(getWaitForCompletion(), request.getWaitForCompletion());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId(), getKeepAlive(), getWaitForCompletion());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
index df8fc65c751e6..a3e2c0cea7d9c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.client.asyncsearch.DeleteAsyncSearchRequest;
+import org.elasticsearch.client.asyncsearch.GetAsyncSearchRequest;
 import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -112,4 +116,35 @@ private static void setRandomSearchParams(SubmitAsyncSearchRequest request, Map<
         expectedParams.put("max_concurrent_shard_requests", Integer.toString(request.getMaxConcurrentShardRequests()));
     }
 
+    public void testGetAsyncSearch() throws Exception {
+        String id = randomAlphaOfLengthBetween(5, 10);
+        Map<String, String> expectedParams = new HashMap<>();
+        GetAsyncSearchRequest submitRequest = new GetAsyncSearchRequest(id);
+        if (randomBoolean()) {
+            TimeValue keepAlive = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setKeepAlive(keepAlive);
+            expectedParams.put("keep_alive", keepAlive.getStringRep());
+        }
+        if (randomBoolean()) {
+            TimeValue waitForCompletion = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setWaitForCompletion(waitForCompletion);
+            expectedParams.put("wait_for_completion", waitForCompletion.getStringRep());
+        }
+
+        Request request = AsyncSearchRequestConverters.getAsyncSearch(submitRequest);
+        String endpoint = "/_async_search/" + id;
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+    }
+
+    public void testDeleteAsyncSearch() throws Exception {
+        String id = randomAlphaOfLengthBetween(5, 10);
+        DeleteAsyncSearchRequest deleteRequest = new DeleteAsyncSearchRequest(id);
+
+        Request request = AsyncSearchRequestConverters.deleteAsyncSearch(deleteRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_async_search/" + id, request.getEndpoint());
+        assertTrue(request.getParameters().isEmpty());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
index d0f6ddc93080f..38e7351e58836 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
@@ -21,36 +21,50 @@
 
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
-import java.util.concurrent.TimeUnit;
 
 public class AsyncSearchIT extends ESRestHighLevelClientTestCase {
 
-    public void testSubmitAsyncSearchRequest() throws IOException {
+    public void testAsyncSearch() throws IOException {
         String index = "test-index";
         createIndex(index, Settings.EMPTY);
 
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
-        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(sourceBuilder, index);
-        // 15 sec should be enough to make sure we always complete right away
-        request.setWaitForCompletion(new TimeValue(15, TimeUnit.SECONDS));
-        AsyncSearchResponse response = highLevelClient().asyncSearch().submitAsyncSearch(request, RequestOptions.DEFAULT);
-        assertFalse(response.isPartial());
-        assertTrue(response.getStartTime() > 0);
-        assertTrue(response.getExpirationTime() > 0);
-        assertNotNull(response.getSearchResponse());
-        if (response.isRunning() == false) {
-            assertNull(response.getId());
-            assertFalse(response.isPartial());
+        SubmitAsyncSearchRequest submitRequest = new SubmitAsyncSearchRequest(sourceBuilder, index);
+        submitRequest.setCleanOnCompletion(false);
+        AsyncSearchResponse submitResponse = highLevelClient().asyncSearch().submit(submitRequest, RequestOptions.DEFAULT);
+        assertNotNull(submitResponse.getId());
+        assertFalse(submitResponse.isPartial());
+        assertTrue(submitResponse.getStartTime() > 0);
+        assertTrue(submitResponse.getExpirationTime() > 0);
+        assertNotNull(submitResponse.getSearchResponse());
+        if (submitResponse.isRunning() == false) {
+            assertFalse(submitResponse.isPartial());
         } else {
-            assertTrue(response.isPartial());
-            assertNotNull(response.getId());
+            assertTrue(submitResponse.isPartial());
+        }
+
+        GetAsyncSearchRequest getRequest = new GetAsyncSearchRequest(submitResponse.getId());
+        AsyncSearchResponse getResponse = highLevelClient().asyncSearch().get(getRequest, RequestOptions.DEFAULT);
+        while (getResponse.isRunning()) {
+            getResponse = highLevelClient().asyncSearch().get(getRequest, RequestOptions.DEFAULT);
         }
-    }
 
+        assertFalse(getResponse.isRunning());
+        assertFalse(getResponse.isPartial());
+        assertTrue(getResponse.getStartTime() > 0);
+        assertTrue(getResponse.getExpirationTime() > 0);
+        assertNotNull(getResponse.getSearchResponse());
+
+        DeleteAsyncSearchRequest deleteRequest = new DeleteAsyncSearchRequest(submitResponse.getId());
+        AcknowledgedResponse deleteAsyncSearchResponse = highLevelClient().asyncSearch().delete(deleteRequest,
+                RequestOptions.DEFAULT);
+        assertNotNull(deleteAsyncSearchResponse);
+        assertNotNull(deleteAsyncSearchResponse.isAcknowledged());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java
new file mode 100644
index 0000000000000..b6861b218cd28
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.TimeUnit;
+
+public class GetAsyncSearchRequestTests extends ESTestCase {
+
+    public void testValidation() {
+        GetAsyncSearchRequest getAsyncSearchRequest = new GetAsyncSearchRequest(randomAlphaOfLength(10));
+        getAsyncSearchRequest.setKeepAlive(new TimeValue(0));
+        assertTrue(getAsyncSearchRequest.validate().isPresent());
+        ValidationException validationException = getAsyncSearchRequest.validate().get();
+        assertEquals(1, validationException.validationErrors().size());
+        assertEquals("Validation Failed: 1: keep_alive must be greater than 1 minute, got: 0s;", validationException.getMessage());
+
+        getAsyncSearchRequest.setKeepAlive(new TimeValue(1, TimeUnit.MINUTES));
+        assertFalse(getAsyncSearchRequest.validate().isPresent());
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
index 94d5e7797cad1..289e9087c8678 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
@@ -38,9 +38,9 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
-import org.elasticsearch.xpack.core.security.SecurityContext;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;

From b2a6bd80bcb5533fbf6a605e7294f36b8aca7518 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Mon, 23 Mar 2020 14:47:42 +0000
Subject: [PATCH 209/285] lang-expressions should use DoubleValuesSource, not
 ValueSource (#53661)

DoubleValuesSource is the type-safe replacement for ValueSource in the lucene
core. Most of elasticsearch has moved to use these, but lang-expressions is still
using the old version. This commit migrates lang-expressions as well.
---
 .../expression/CountMethodValueSource.java    | 49 +++++++----------
 .../script/expression/DateField.java          | 10 ++--
 .../expression/DateMethodValueSource.java     | 39 +++++++-------
 .../script/expression/DateObject.java         | 18 +++----
 .../expression/DateObjectValueSource.java     | 37 ++++++-------
 .../expression/EmptyMemberValueSource.java    | 37 ++++++-------
 .../expression/ExpressionScriptEngine.java    | 30 +++++------
 .../FieldDataBasedDoubleValuesSource.java     | 53 +++++++++++++++++++
 .../expression/FieldDataValueSource.java      | 36 ++++++-------
 .../expression/GeoEmptyValueSource.java       | 43 +++++++--------
 .../script/expression/GeoField.java           | 14 ++---
 .../expression/GeoLatitudeValueSource.java    | 40 ++++++--------
 .../expression/GeoLongitudeValueSource.java   | 37 ++++++-------
 .../script/expression/NumericField.java       |  6 +--
 .../ReplaceableConstDoubleValueSource.java    |  2 +-
 .../ReplaceableConstDoubleValues.java         |  6 +--
 16 files changed, 234 insertions(+), 223 deletions(-)
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java

diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
index d70c6790317f9..7d15d9a9ba83f 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
@@ -19,44 +19,36 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
+import java.io.IOException;
+
 /**
  * A ValueSource to create FunctionValues to get the count of the number of values in a field for a document.
  */
-final class CountMethodValueSource extends ValueSource {
-    IndexFieldData<?> fieldData;
+final class CountMethodValueSource extends FieldDataBasedDoubleValuesSource {
 
     CountMethodValueSource(IndexFieldData<?> fieldData) {
-        Objects.requireNonNull(fieldData);
-
-        this.fieldData = fieldData;
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
+    public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) {
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(ctx);
         final SortedNumericDoubleValues values = leafData.getDoubleValues();
+        return new DoubleValues() {
+            @Override
+            public double doubleValue() {
+                return values.docValueCount();
+            }
 
-        return new DoubleDocValues(this) {
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return values.docValueCount();
-                } else {
-                    return 0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
             }
         };
     }
@@ -65,19 +57,18 @@ public double doubleVal(int doc) throws IOException {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
-        FieldDataValueSource that = (FieldDataValueSource) o;
-
+        CountMethodValueSource that = (CountMethodValueSource) o;
         return fieldData.equals(that.fieldData);
     }
 
     @Override
-    public int hashCode() {
-        return 31 * getClass().hashCode() + fieldData.hashCode();
+    public String toString() {
+        return "count: field(" + fieldData.getFieldName() + ")";
     }
 
     @Override
-    public String description() {
-        return "count: field(" + fieldData.getFieldName() + ")";
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
index 2d50f160812dc..3ce572cc43710 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
@@ -19,12 +19,12 @@
  * under the License.
  */
 
-import java.util.Calendar;
-
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.util.Calendar;
+
 /**
  * Expressions API for date fields.
  */
@@ -56,7 +56,7 @@ private DateField() {}
     static final String GET_MINUTES_METHOD      = "getMinutes";
     static final String GET_SECONDS_METHOD      = "getSeconds";
 
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case VALUE_VARIABLE:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
@@ -69,7 +69,7 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
         }
     }
 
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case GETVALUE_METHOD:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
index ee8c604a39ba2..f1c918e2d2447 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
@@ -19,21 +19,19 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TimeZone;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.TimeZone;
+
 /** Extracts a portion of a date field with {@code Calendar.get()} */
 class DateMethodValueSource extends FieldDataValueSource {
 
@@ -50,27 +48,26 @@ class DateMethodValueSource extends FieldDataValueSource {
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
-        return new DoubleDocValues(this) {
+        return new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                calendar.setTimeInMillis((long)docValues.doubleValue());
+                return calendar.get(calendarType);
+            }
+
             @Override
-            public double doubleVal(int docId) throws IOException {
-                if (docValues.advanceExact(docId)) {
-                    long millis = (long)docValues.doubleValue();
-                    calendar.setTimeInMillis(millis);
-                    return calendar.get(calendarType);
-                } else {
-                    return 0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return docValues.advanceExact(doc);
             }
         };
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return methodName + ": field(" + fieldData.getFieldName() + ")";
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
index 66cc56370a32d..e5d12bcff1152 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
@@ -19,7 +19,7 @@
  * under the License.
  */
 
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
 import org.joda.time.ReadableDateTime;
@@ -30,7 +30,7 @@
 final class DateObject {
     // no instance
     private DateObject() {}
-    
+
     // supported variables
     static final String CENTURY_OF_ERA_VARIABLE       = "centuryOfEra";
     static final String DAY_OF_MONTH_VARIABLE         = "dayOfMonth";
@@ -50,7 +50,7 @@ private DateObject() {}
     static final String YEAR_VARIABLE                 = "year";
     static final String YEAR_OF_CENTURY_VARIABLE      = "yearOfCentury";
     static final String YEAR_OF_ERA_VARIABLE          = "yearOfEra";
-    
+
     // supported methods
     static final String GETCENTURY_OF_ERA_METHOD      = "getCenturyOfEra";
     static final String GETDAY_OF_MONTH_METHOD        = "getDayOfMonth";
@@ -70,8 +70,8 @@ private DateObject() {}
     static final String GETYEAR_METHOD                = "getYear";
     static final String GETYEAR_OF_CENTURY_METHOD     = "getYearOfCentury";
     static final String GETYEAR_OF_ERA_METHOD         = "getYearOfEra";
-    
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case CENTURY_OF_ERA_VARIABLE:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getCenturyOfEra);
@@ -110,12 +110,12 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
             case YEAR_OF_ERA_VARIABLE:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getYearOfEra);
             default:
-                throw new IllegalArgumentException("Member variable [" + variable + 
+                throw new IllegalArgumentException("Member variable [" + variable +
                                                    "] does not exist for date object on field [" + fieldName + "].");
         }
     }
-    
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case GETCENTURY_OF_ERA_METHOD:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getCenturyOfEra);
@@ -154,7 +154,7 @@ static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, Stri
             case GETYEAR_OF_ERA_METHOD:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getYearOfEra);
             default:
-                throw new IllegalArgumentException("Member method [" + method + 
+                throw new IllegalArgumentException("Member method [" + method +
                                                    "] does not exist for date object on field [" + fieldName + "].");
         }
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
index 89f991c174d30..d49dd6c0a1943 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
@@ -19,14 +19,8 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.ToIntFunction;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
@@ -35,6 +29,10 @@
 import org.joda.time.MutableDateTime;
 import org.joda.time.ReadableDateTime;
 
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.ToIntFunction;
+
 /** Extracts a portion of a date field with joda time */
 class DateObjectValueSource extends FieldDataValueSource {
 
@@ -52,27 +50,26 @@ class DateObjectValueSource extends FieldDataValueSource {
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         MutableDateTime joda = new MutableDateTime(0, DateTimeZone.UTC);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                joda.setMillis((long)docValues.doubleValue());
+                return function.applyAsInt(joda);
+            }
+
             @Override
-            public double doubleVal(int docId) throws IOException {
-                if (docValues.advanceExact(docId)) {
-                    long millis = (long)docValues.doubleValue();
-                    joda.setMillis(millis);
-                    return function.applyAsInt(joda);
-                } else {
-                    return 0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return docValues.advanceExact(doc);
             }
-        };
+        }, 0);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return methodName + ": field(" + fieldData.getFieldName() + ")";
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
index 750b03762a7f9..26d300547fbac 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
@@ -20,44 +20,39 @@
 package org.elasticsearch.script.expression;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
 import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
 
 /**
  * ValueSource to return non-zero if a field is missing.
  * <p>
  * This is essentially sugar over !count()
  */
-final class EmptyMemberValueSource extends ValueSource {
-    final IndexFieldData<?> fieldData;
+final class EmptyMemberValueSource extends FieldDataBasedDoubleValuesSource {
 
     EmptyMemberValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final SortedNumericDoubleValues values = leafData.getDoubleValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return 0;
-                } else {
-                    return 1;
-                }
+            public double doubleValue() {
+                return 0;
             }
-        };
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
+            }
+        }, 1);
     }
 
     @Override
@@ -71,12 +66,12 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         EmptyMemberValueSource other = (EmptyMemberValueSource) obj;
-        if (!fieldData.equals(other.fieldData)) return false;
-        return true;
+        return fieldData.equals(other.fieldData);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "empty: field(" + fieldData.getFieldName() + ")";
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
index b89a6231a67da..19865bd361e6c 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
@@ -23,8 +23,8 @@
 import org.apache.lucene.expressions.SimpleBindings;
 import org.apache.lucene.expressions.js.JavascriptCompiler;
 import org.apache.lucene.expressions.js.VariableContext;
-import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
@@ -251,9 +251,9 @@ private static NumberSortScript.LeafFactory newSortScript(Expression expr, Searc
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    needsScores |= valueSource.getSortField(false).needsScores();
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    final DoubleValuesSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.needsScores();
+                    bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -275,8 +275,7 @@ private static TermsSetQueryScript.LeafFactory newTermsSetQueryScript(Expression
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    bindings.add(variable, getDocValueSource(variable, lookup));
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -310,9 +309,9 @@ private static AggregationScript.LeafFactory newAggregationScript(Expression exp
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    needsScores |= valueSource.getSortField(false).needsScores();
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    final DoubleValuesSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.needsScores();
+                    bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -329,8 +328,7 @@ private static FieldScript.LeafFactory newFieldScript(Expression expr, SearchLoo
                 if (vars != null && vars.containsKey(variable)) {
                     bindFromParams(vars, bindings, variable);
                 } else {
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    bindings.add(variable, getDocValueSource(variable, lookup));
                 }
             } catch (Exception e) {
                 throw convertToScriptException("link error", expr.sourceText, variable, e);
@@ -382,9 +380,9 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    needsScores |= valueSource.getSortField(false).needsScores();
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    final DoubleValuesSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.needsScores();
+                    bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -412,7 +410,7 @@ private static ScriptException convertToScriptException(String message, String s
         throw new ScriptException(message, cause, stack, source, NAME);
     }
 
-    private static ValueSource getDocValueSource(String variable, SearchLookup lookup) throws ParseException {
+    private static DoubleValuesSource getDocValueSource(String variable, SearchLookup lookup) throws ParseException {
         VariableContext[] parts = VariableContext.parse(variable);
         if (parts[0].text.equals("doc") == false) {
             throw new ParseException("Unknown variable [" + parts[0].text + "]", 0);
@@ -463,7 +461,7 @@ private static ValueSource getDocValueSource(String variable, SearchLookup looku
         }
 
         IndexFieldData<?> fieldData = lookup.doc().getForField(fieldType);
-        final ValueSource valueSource;
+        final DoubleValuesSource valueSource;
         if (fieldType instanceof GeoPointFieldType) {
             // geo
             if (methodname == null) {
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java
new file mode 100644
index 0000000000000..ca36ba26bd74e
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+
+import java.util.Objects;
+
+abstract class FieldDataBasedDoubleValuesSource extends DoubleValuesSource {
+
+    FieldDataBasedDoubleValuesSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    protected final IndexFieldData<?> fieldData;
+
+    @Override
+    public boolean needsScores() {
+        return false;
+    }
+
+    @Override
+    public DoubleValuesSource rewrite(IndexSearcher reader) {
+        return this;
+    }
+
+    @Override
+    public boolean isCacheable(LeafReaderContext ctx) {
+        return DocValues.isCacheable(ctx, fieldData.getFieldName());
+    }
+
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
index 63050a07da8bf..7ab8c3dd8f8bd 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
@@ -19,29 +19,26 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.io.IOException;
+import java.util.Objects;
+
 /**
  * A {@link ValueSource} wrapper for field data.
  */
-class FieldDataValueSource extends ValueSource {
+class FieldDataValueSource extends FieldDataBasedDoubleValuesSource {
 
-    final IndexFieldData<?> fieldData;
     final MultiValueMode multiValueMode;
 
     protected FieldDataValueSource(IndexFieldData<?> fieldData, MultiValueMode multiValueMode) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
         this.multiValueMode = Objects.requireNonNull(multiValueMode);
     }
 
@@ -65,24 +62,25 @@ public int hashCode() {
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
-        return new DoubleDocValues(this) {
-          @Override
-          public double doubleVal(int doc) throws IOException {
-            if (docValues.advanceExact(doc)) {
+        return new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
                 return docValues.doubleValue();
-            } else {
-                return 0;
             }
-          }
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                return docValues.advanceExact(doc);
+            }
         };
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "field(" + fieldData.getFieldName() + ")";
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
index 2949e1dd9287a..5e3a20f532e7f 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
@@ -19,43 +19,38 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.io.IOException;
+
 /**
  * ValueSource to return non-zero if a field is missing.
  */
-final class GeoEmptyValueSource extends ValueSource {
-    IndexFieldData<?> fieldData;
+final class GeoEmptyValueSource extends FieldDataBasedDoubleValuesSource {
 
     GeoEmptyValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return 1;
-                } else {
-                    return 0;
-                }
+            public double doubleValue() {
+                return 1;
             }
-        };
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
+            }
+        }, 0);
     }
 
     @Override
@@ -69,12 +64,12 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         GeoEmptyValueSource other = (GeoEmptyValueSource) obj;
-        if (!fieldData.equals(other.fieldData)) return false;
-        return true;
+        return fieldData.equals(other.fieldData);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "empty: field(" + fieldData.getFieldName() + ")";
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
index 0ce74a187df77..4b15e30aba414 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
@@ -19,7 +19,7 @@
  * under the License.
  */
 
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 
 /**
@@ -28,18 +28,18 @@
 final class GeoField {
     // no instance
     private GeoField() {}
-    
+
     // supported variables
     static final String EMPTY_VARIABLE        = "empty";
     static final String LAT_VARIABLE          = "lat";
     static final String LON_VARIABLE          = "lon";
-    
+
     // supported methods
     static final String ISEMPTY_METHOD        = "isEmpty";
     static final String GETLAT_METHOD         = "getLat";
     static final String GETLON_METHOD         = "getLon";
-    
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case EMPTY_VARIABLE:
                 return new GeoEmptyValueSource(fieldData);
@@ -51,8 +51,8 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
                 throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for geo field [" + fieldName + "].");
         }
     }
-    
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case ISEMPTY_METHOD:
                 return new GeoEmptyValueSource(fieldData);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
index 3c04de1658669..8b9a3282ec78b 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
@@ -19,43 +19,38 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.io.IOException;
+
 /**
  * ValueSource to return latitudes as a double "stream" for geopoint fields
  */
-final class GeoLatitudeValueSource extends ValueSource {
-    final IndexFieldData<?> fieldData;
+final class GeoLatitudeValueSource extends FieldDataBasedDoubleValuesSource {
 
     GeoLatitudeValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                return values.nextValue().getLat();
+            }
+
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return values.nextValue().getLat();
-                } else {
-                    return 0.0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
             }
-        };
+        }, 0);
     }
 
     @Override
@@ -69,12 +64,11 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         GeoLatitudeValueSource other = (GeoLatitudeValueSource) obj;
-        if (!fieldData.equals(other.fieldData)) return false;
-        return true;
+        return fieldData.equals(other.fieldData);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "lat: field(" + fieldData.getFieldName() + ")";
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
index 8b2c5d83c0257..9bd285f8da871 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
@@ -19,43 +19,38 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.io.IOException;
+
 /**
  * ValueSource to return longitudes as a double "stream" for geopoint fields
  */
-final class GeoLongitudeValueSource extends ValueSource {
-    final IndexFieldData<?> fieldData;
+final class GeoLongitudeValueSource extends FieldDataBasedDoubleValuesSource {
 
     GeoLongitudeValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                return values.nextValue().getLon();
+            }
+
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return values.nextValue().getLon();
-                } else {
-                    return 0.0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
             }
-        };
+        }, 0.0);
     }
 
     @Override
@@ -74,7 +69,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "lon: field(" + fieldData.getFieldName() + ")";
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
index 06875632134d7..0eeff96ba08e4 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
@@ -19,7 +19,7 @@
  * under the License.
  */
 
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -46,7 +46,7 @@ private NumericField() {}
     static final String SUM_METHOD              = "sum";
     static final String COUNT_METHOD            = "count";
 
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case VALUE_VARIABLE:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
@@ -60,7 +60,7 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
         }
     }
 
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case GETVALUE_METHOD:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
index a0cfd721adb5a..c237ca3101673 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
@@ -80,7 +80,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
     }
 
     @Override
-    public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
+    public DoubleValuesSource rewrite(IndexSearcher reader) {
         return this;
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
index 557f9ad812e0c..4efc8f9791cd9 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
@@ -21,8 +21,6 @@
 
 import org.apache.lucene.search.DoubleValues;
 
-import java.io.IOException;
-
 /**
  * A support class for an executable expression script that allows the double returned
  * by a {@link DoubleValues} to be modified.
@@ -35,12 +33,12 @@ void setValue(double value) {
     }
 
     @Override
-    public double doubleValue() throws IOException {
+    public double doubleValue() {
         return value;
     }
 
     @Override
-    public boolean advanceExact(int doc) throws IOException {
+    public boolean advanceExact(int doc) {
         return true;
     }
 }

From 0ad4f52d84ddf706a3549c30f7a0e2ac429927e3 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 23 Mar 2020 16:23:08 +0100
Subject: [PATCH 210/285] Increase ensureGreen() timeout in
 CloseWhileRelocatingShardsIT (#53981)

The test in CloseWhileRelocatingShardsIT failed recently
multiple times (3) when waiting for initial indices to be
become green. Looking at the execution logs from  #53544
 it appears at the very beginning of the test and when
the WindowsFS file system is picked up (which is known
to slow down tests).

This commit simply increases the timeout for the first
ensureGreen() to 60 seconds. If the test continues to fail,
we might want to test a larger timeout or disable
WindowsFS for this test.

Closes #53544
---
 .../indices/state/CloseWhileRelocatingShardsIT.java            | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
index ce424e61787ee..e83d9b14a3845 100644
--- a/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
@@ -117,7 +118,7 @@ public void testCloseWhileRelocatingShards() throws Exception {
             indices[i] = indexName;
         }
 
-        ensureGreen(indices);
+        ensureGreen(TimeValue.timeValueSeconds(60L),indices);
         assertAcked(client().admin().cluster().prepareUpdateSettings()
             .setTransientSettings(Settings.builder()
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())));

From 975d4f5c864a9c3da87aa2ec0fe41517a41d9291 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 23 Mar 2020 08:47:35 -0700
Subject: [PATCH 211/285] Don't include HLRC on downstream classpath twice
 (#53983)

---
 .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index be8d69da60c4b..922afdbd770a0 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -453,7 +453,9 @@ class BuildPlugin implements Plugin<Project> {
 
         project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
             // Ensure that when we are compiling against the "original" JAR that we also include any "shadow" dependencies on the compile classpath
-            project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME))
+            project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME).dependencies.all { Dependency dependency ->
+                project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).dependencies.add(dependency)
+            }
         }
     }
 

From 1ca52fc2925defab1f0ce40489976e4e1812e75a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Mar 2020 11:50:29 -0400
Subject: [PATCH 212/285] Try to save memory on aggregations (#53793)

This delays deserializing the aggregation response try until *right*
before we merge the objects.
---
 .../action/search/SearchPhaseController.java  |  76 +++++---
 .../common/io/stream/DelayableWriteable.java  | 131 ++++++++++++++
 .../common/io/stream/FilterStreamInput.java   |   5 +
 .../NamedWriteableAwareStreamInput.java       |   5 +
 .../common/io/stream/StreamInput.java         |   8 +
 .../search/query/QuerySearchResult.java       | 111 ++++++++----
 .../io/stream/DelayableWriteableTests.java    | 163 ++++++++++++++++++
 .../search/query/QuerySearchResultTests.java  |   4 +-
 8 files changed, 439 insertions(+), 64 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
 create mode 100644 server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index f2cf6b199dcbe..9648f74e96b27 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -19,8 +19,17 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
-import com.carrotsearch.hppc.ObjectObjectHashMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.IntFunction;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
@@ -58,16 +67,8 @@
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.function.IntFunction;
-import java.util.stream.Collectors;
+import com.carrotsearch.hppc.IntArrayList;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
 
 public final class SearchPhaseController {
     private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
@@ -429,7 +430,7 @@ public ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResul
      * @see QuerySearchResult#consumeProfileResult()
      */
     private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResult> queryResults,
-                                                List<InternalAggregations> bufferedAggs, List<TopDocs> bufferedTopDocs,
+                                                List<Supplier<InternalAggregations>> bufferedAggs, List<TopDocs> bufferedTopDocs,
                                                 TopDocsStats topDocsStats, int numReducePhases, boolean isScrollRequest,
                                                 InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
                                                 boolean performFinalReduce) {
@@ -453,7 +454,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
         final boolean hasSuggest = firstResult.suggest() != null;
         final boolean hasProfileResults = firstResult.hasProfileResults();
         final boolean consumeAggs;
-        final List<InternalAggregations> aggregationsList;
+        final List<Supplier<InternalAggregations>> aggregationsList;
         if (bufferedAggs != null) {
             consumeAggs = false;
             // we already have results from intermediate reduces and just need to perform the final reduce
@@ -492,7 +493,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
                 }
             }
             if (consumeAggs) {
-                aggregationsList.add((InternalAggregations) result.consumeAggs());
+                aggregationsList.add(result.consumeAggs());
             }
             if (hasProfileResults) {
                 String key = result.getSearchShardTarget().toString();
@@ -508,8 +509,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
             reducedSuggest = new Suggest(Suggest.reduce(groupedSuggestions));
             reducedCompletionSuggestions = reducedSuggest.filter(CompletionSuggestion.class);
         }
-        final InternalAggregations aggregations = aggregationsList.isEmpty() ? null : InternalAggregations.topLevelReduce(aggregationsList,
-                    performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction());
+        final InternalAggregations aggregations = reduceAggs(aggReduceContextBuilder, performFinalReduce, aggregationsList);
         final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
         final SortedTopDocs sortedTopDocs = sortDocs(isScrollRequest, queryResults, bufferedTopDocs, topDocsStats, from, size,
             reducedCompletionSuggestions);
@@ -519,6 +519,24 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
             firstResult.sortValueFormats(), numReducePhases, size, from, false);
     }
 
+    private InternalAggregations reduceAggs(
+        InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
+        boolean performFinalReduce,
+        List<Supplier<InternalAggregations>> aggregationsList
+    ) {
+        /*
+         * Parse the aggregations, clearing the list as we go so bits backing
+         * the DelayedWriteable can be collected immediately.
+         */
+        List<InternalAggregations> toReduce = new ArrayList<>(aggregationsList.size());
+        for (int i = 0; i < aggregationsList.size(); i++) {
+            toReduce.add(aggregationsList.get(i).get());
+            aggregationsList.set(i, null);
+        }
+        return aggregationsList.isEmpty() ? null : InternalAggregations.topLevelReduce(toReduce,
+            performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction());
+    }
+
     /*
      * Returns the size of the requested top documents (from + size)
      */
@@ -600,7 +618,7 @@ public InternalSearchResponse buildResponse(SearchHits hits) {
      */
     static final class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhaseResult> {
         private final SearchShardTarget[] processedShards;
-        private final InternalAggregations[] aggsBuffer;
+        private final Supplier<InternalAggregations>[] aggsBuffer;
         private final TopDocs[] topDocsBuffer;
         private final boolean hasAggs;
         private final boolean hasTopDocs;
@@ -642,7 +660,9 @@ private QueryPhaseResultConsumer(SearchProgressListener progressListener, Search
             this.progressListener = progressListener;
             this.processedShards = new SearchShardTarget[expectedResultSize];
             // no need to buffer anything if we have less expected results. in this case we don't consume any results ahead of time.
-            this.aggsBuffer = new InternalAggregations[hasAggs ? bufferSize : 0];
+            @SuppressWarnings("unchecked")
+            Supplier<InternalAggregations>[] aggsBuffer = new Supplier[hasAggs ? bufferSize : 0];
+            this.aggsBuffer = aggsBuffer;
             this.topDocsBuffer = new TopDocs[hasTopDocs ? bufferSize : 0];
             this.hasTopDocs = hasTopDocs;
             this.hasAggs = hasAggs;
@@ -665,10 +685,14 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
             if (querySearchResult.isNull() == false) {
                 if (index == bufferSize) {
                     if (hasAggs) {
-                        ReduceContext reduceContext = aggReduceContextBuilder.forPartialReduction();
-                        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(Arrays.asList(aggsBuffer), reduceContext);
-                        Arrays.fill(aggsBuffer, null);
-                        aggsBuffer[0] = reducedAggs;
+                        List<InternalAggregations> aggs = new ArrayList<>(aggsBuffer.length);
+                        for (int i = 0; i < aggsBuffer.length; i++) {
+                            aggs.add(aggsBuffer[i].get());
+                            aggsBuffer[i] = null; // null the buffer so it can be GCed now.
+                        }
+                        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(
+                                aggs, aggReduceContextBuilder.forPartialReduction());
+                        aggsBuffer[0] = () -> reducedAggs;
                     }
                     if (hasTopDocs) {
                         TopDocs reducedTopDocs = mergeTopDocs(Arrays.asList(topDocsBuffer),
@@ -681,12 +705,12 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
                     index = 1;
                     if (hasAggs || hasTopDocs) {
                         progressListener.notifyPartialReduce(SearchProgressListener.buildSearchShards(processedShards),
-                            topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0] : null, numReducePhases);
+                            topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0].get() : null, numReducePhases);
                     }
                 }
                 final int i = index++;
                 if (hasAggs) {
-                    aggsBuffer[i] = (InternalAggregations) querySearchResult.consumeAggs();
+                    aggsBuffer[i] = querySearchResult.consumeAggs();
                 }
                 if (hasTopDocs) {
                     final TopDocsAndMaxScore topDocs = querySearchResult.consumeTopDocs(); // can't be null
@@ -698,7 +722,7 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
             processedShards[querySearchResult.getShardIndex()] = querySearchResult.getSearchShardTarget();
         }
 
-        private synchronized List<InternalAggregations> getRemainingAggs() {
+        private synchronized List<Supplier<InternalAggregations>> getRemainingAggs() {
             return hasAggs ? Arrays.asList(aggsBuffer).subList(0, index) : null;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
new file mode 100644
index 0000000000000..e75b81fff4ef4
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io.stream;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+/**
+ * A holder for {@link Writeable}s that can delays reading the underlying
+ * {@linkplain Writeable} when it is read from a remote node.
+ */
+public abstract class DelayableWriteable<T extends Writeable> implements Supplier<T>, Writeable {
+    /**
+     * Build a {@linkplain DelayableWriteable} that wraps an existing object
+     * but is serialized so that deserializing it can be delayed.
+     */
+    public static <T extends Writeable> DelayableWriteable<T> referencing(T reference) {
+        return new Referencing<>(reference);
+    }
+    /**
+     * Build a {@linkplain DelayableWriteable} that copies a buffer from
+     * the provided {@linkplain StreamInput} and deserializes the buffer
+     * when {@link Supplier#get()} is called.
+     */
+    public static <T extends Writeable> DelayableWriteable<T> delayed(Writeable.Reader<T> reader, StreamInput in) throws IOException {
+        return new Delayed<>(reader, in);
+    }
+
+    private DelayableWriteable() {}
+
+    public abstract boolean isDelayed();
+
+    private static class Referencing<T extends Writeable> extends DelayableWriteable<T> {
+        private T reference;
+
+        Referencing(T reference) {
+            this.reference = reference;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            try (BytesStreamOutput buffer = new BytesStreamOutput()) {
+                reference.writeTo(buffer);
+                out.writeBytesReference(buffer.bytes());
+            }
+        }
+
+        @Override
+        public T get() {
+            return reference;
+        }
+
+        @Override
+        public boolean isDelayed() {
+            return false;
+        }
+    }
+
+    private static class Delayed<T extends Writeable> extends DelayableWriteable<T> {
+        private final Writeable.Reader<T> reader;
+        private final Version remoteVersion;
+        private final BytesReference serialized;
+        private final NamedWriteableRegistry registry;
+
+        Delayed(Writeable.Reader<T> reader, StreamInput in) throws IOException {
+            this.reader = reader;
+            remoteVersion = in.getVersion();
+            serialized = in.readBytesReference();
+            registry = in.namedWriteableRegistry();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (out.getVersion() == remoteVersion) {
+                /*
+                 * If the version *does* line up we can just copy the bytes
+                 * which is good because this is how shard request caching
+                 * works.
+                 */
+                out.writeBytesReference(serialized);
+            } else {
+                /*
+                 * If the version doesn't line up then we have to deserialize
+                 * into the Writeable and re-serialize it against the new
+                 * output stream so it can apply any backwards compatibility
+                 * differences in the wire protocol. This ain't efficient but
+                 * it should be quite rare.
+                 */
+                referencing(get()).writeTo(out);
+            }
+        }
+
+        @Override
+        public T get() {
+            try {
+                try (StreamInput in = registry == null ?
+                        serialized.streamInput() : new NamedWriteableAwareStreamInput(serialized.streamInput(), registry)) {
+                    in.setVersion(remoteVersion);
+                    return reader.read(in);
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("unexpected error expanding aggregations", e);
+            }
+        }
+
+        @Override
+        public boolean isDelayed() {
+            return true;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
index 2fa700634b851..6d7d9fd6952fd 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
@@ -94,4 +94,9 @@ public void setVersion(Version version) {
     protected void ensureCanReadBytes(int length) throws EOFException {
         delegate.ensureCanReadBytes(length);
     }
+
+    @Override
+    public NamedWriteableRegistry namedWriteableRegistry() {
+        return delegate.namedWriteableRegistry();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
index 5db80a711eeb9..149231b1fe433 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
@@ -52,4 +52,9 @@ public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused
             + "] than it was read from [" + name + "].";
         return c;
     }
+
+    @Override
+    public NamedWriteableRegistry namedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index d30b226dfb2dc..7de344d95dc46 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -1097,6 +1097,14 @@ public <T extends Exception> T readException() throws IOException {
         return null;
     }
 
+    /**
+     * Get the registry of named writeables is his stream has one,
+     * {@code null} otherwise.
+     */
+    public NamedWriteableRegistry namedWriteableRegistry() {
+        return null;
+    }
+
     /**
      * Reads a {@link NamedWriteable} from the current stream, by first reading its name and then looking for
      * the corresponding entry in the registry by name, so that the proper object can be read and returned.
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 18af04add2082..2863f45de85f2 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -19,16 +19,24 @@
 
 package org.elasticsearch.search.query;
 
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.common.lucene.Lucene.readTopDocs;
+import static org.elasticsearch.common.lucene.Lucene.writeTopDocs;
+
+import java.io.IOException;
+import java.util.List;
+
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.DelayableWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -37,14 +45,6 @@
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.common.lucene.Lucene.readTopDocs;
-import static org.elasticsearch.common.lucene.Lucene.writeTopDocs;
-
 public final class QuerySearchResult extends SearchPhaseResult {
 
     private int from;
@@ -54,7 +54,14 @@ public final class QuerySearchResult extends SearchPhaseResult {
     private TotalHits totalHits;
     private float maxScore = Float.NaN;
     private DocValueFormat[] sortValueFormats;
-    private InternalAggregations aggregations;
+    /**
+     * Aggregation results. We wrap them in
+     * {@linkplain DelayableWriteable} because
+     * {@link InternalAggregation} is usually made up of many small objects
+     * which have a fairly high overhead in the JVM. So we delay deserializing
+     * them until just before we need them.
+     */
+    private DelayableWriteable<InternalAggregations> aggregations;
     private boolean hasAggs;
     private Suggest suggest;
     private boolean searchTimedOut;
@@ -196,21 +203,21 @@ public boolean hasAggs() {
      * Returns and nulls out the aggregation for this search results. This allows to free up memory once the aggregation is consumed.
      * @throws IllegalStateException if the aggregations have already been consumed.
      */
-    public Aggregations consumeAggs() {
+    public DelayableWriteable<InternalAggregations> consumeAggs() {
         if (aggregations == null) {
             throw new IllegalStateException("aggs already consumed");
         }
-        Aggregations aggs = aggregations;
+        DelayableWriteable<InternalAggregations> aggs = aggregations;
         aggregations = null;
         return aggs;
     }
 
     public void aggregations(InternalAggregations aggregations) {
-        this.aggregations = aggregations;
+        this.aggregations = aggregations == null ? null : DelayableWriteable.referencing(aggregations);
         hasAggs = aggregations != null;
     }
 
-    public InternalAggregations aggregations() {
+    public DelayableWriteable<InternalAggregations> aggregations() {
         return aggregations;
     }
 
@@ -313,19 +320,32 @@ public void readFromWithId(SearchContextId id, StreamInput in) throws IOExceptio
             }
         }
         setTopDocs(readTopDocs(in));
-        if (hasAggs = in.readBoolean()) {
-            aggregations = new InternalAggregations(in);
-        }
-        if (in.getVersion().before(Version.V_7_2_0)) {
-            List<SiblingPipelineAggregator> pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream()
-                .map(a -> (SiblingPipelineAggregator) a).collect(Collectors.toList());
-            if (hasAggs && pipelineAggregators.isEmpty() == false) {
-                List<InternalAggregation> internalAggs = aggregations.asList().stream()
-                    .map(agg -> (InternalAggregation) agg).collect(Collectors.toList());
-                //Earlier versions serialize sibling pipeline aggs separately as they used to be set to QuerySearchResult directly, while
-                //later versions include them in InternalAggregations. Note that despite serializing sibling pipeline aggs as part of
-                //InternalAggregations is supported since 6.7.0, the shards set sibling pipeline aggs to InternalAggregations only from 7.1.
-                this.aggregations = new InternalAggregations(internalAggs, pipelineAggregators);
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            InternalAggregations readAggs = null;
+            if (hasAggs = in.readBoolean()) {
+                readAggs = new InternalAggregations(in);
+            }
+            if (in.getVersion().before(Version.V_7_2_0)) {
+                List<SiblingPipelineAggregator> pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream()
+                    .map(a -> (SiblingPipelineAggregator) a).collect(toList());
+                if (hasAggs && pipelineAggregators.isEmpty() == false) {
+                    List<InternalAggregation> internalAggs = readAggs.copyResults();
+                    /*
+                     * Earlier versions serialize sibling pipeline aggs
+                     * separately as they used to be set to QuerySearchResult
+                     * directly, while later versions include them in
+                     * InternalAggregations. Note that despite serializing
+                     * sibling pipeline aggs as part of nternalAggregations is
+                     * supported since 6.7.0, the shards set sibling pipeline
+                     * aggs to InternalAggregations only from 7.1.
+                     */
+                    readAggs = new InternalAggregations(internalAggs, pipelineAggregators);
+                }
+            }
+            aggregations = DelayableWriteable.referencing(readAggs);
+        } else {
+            if (hasAggs = in.readBoolean()) {
+                aggregations = DelayableWriteable.delayed(InternalAggregations::new, in);
             }
         }
         if (in.readBoolean()) {
@@ -364,18 +384,37 @@ public void writeToNoId(StreamOutput out) throws IOException {
         writeTopDocs(out, topDocsAndMaxScore);
         if (aggregations == null) {
             out.writeBoolean(false);
+            if (out.getVersion().before(Version.V_7_2_0)) {
+                /*
+                 * Earlier versions expect sibling pipeline aggs separately
+                 * as they used to be set to QuerySearchResult directly, while
+                 * later versions expect them in InternalAggregations. Note
+                 * that despite serializing sibling pipeline aggs as part of
+                 * InternalAggregations is supported since 6.7.0, the shards
+                 * set sibling pipeline aggs to InternalAggregations only from
+                 * 7.1 on.
+                 */
+                out.writeNamedWriteableList(emptyList());
+            }
         } else {
             out.writeBoolean(true);
-            aggregations.writeTo(out);
-        }
-        if (out.getVersion().before(Version.V_7_2_0)) {
-            //Earlier versions expect sibling pipeline aggs separately as they used to be set to QuerySearchResult directly,
-            //while later versions expect them in InternalAggregations. Note that despite serializing sibling pipeline aggs as part of
-            //InternalAggregations is supported since 6.7.0, the shards set sibling pipeline aggs to InternalAggregations only from 7.1 on.
-            if (aggregations == null) {
-                out.writeNamedWriteableList(Collections.emptyList());
+            if (out.getVersion().before(Version.V_8_0_0)) {
+                InternalAggregations aggs = aggregations.get();
+                aggs.writeTo(out);
+                if (out.getVersion().before(Version.V_7_2_0)) {
+                    /*
+                     * Earlier versions expect sibling pipeline aggs separately
+                     * as they used to be set to QuerySearchResult directly, while
+                     * later versions expect them in InternalAggregations. Note
+                     * that despite serializing sibling pipeline aggs as part of
+                     * InternalAggregations is supported since 6.7.0, the shards
+                     * set sibling pipeline aggs to InternalAggregations only from
+                     * 7.1 on.
+                     */
+                    out.writeNamedWriteableList(aggs.getTopLevelPipelineAggregators());
+                }
             } else {
-                out.writeNamedWriteableList(aggregations.getTopLevelPipelineAggregators());
+                aggregations.writeTo(out);
             }
         }
         if (suggest == null) {
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
new file mode 100644
index 0000000000000..fbd8702608ead
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io.stream;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DelayableWriteableTests extends ESTestCase {
+    // NOTE: we don't use AbstractWireSerializingTestCase because we don't implement equals and hashCode.
+    public static class Example implements NamedWriteable {
+        private final String s;
+
+        public Example(String s) {
+            this.s = s;
+        }
+
+        public Example(StreamInput in) throws IOException {
+            s = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(s);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "example";
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            Example other = (Example) obj;
+            return s.equals(other.s);
+        }
+
+        @Override
+        public int hashCode() {
+            return s.hashCode();
+        }
+    }
+
+    public static class NamedHolder implements Writeable {
+        private final Example e;
+
+        public NamedHolder(Example e) {
+            this.e = e;
+        }
+
+        public NamedHolder(StreamInput in) throws IOException {
+            e = in.readNamedWriteable(Example.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeNamedWriteable(e);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            NamedHolder other = (NamedHolder) obj;
+            return e.equals(other.e);
+        }
+
+        @Override
+        public int hashCode() {
+            return e.hashCode();
+        }
+    }
+
+    public void testRoundTripFromReferencing() throws IOException {
+        Example e = new Example(randomAlphaOfLength(5));
+        DelayableWriteable<Example> original = DelayableWriteable.referencing(e);
+        assertFalse(original.isDelayed());
+        roundTripTestCase(original, Example::new);
+    }
+
+    public void testRoundTripFromReferencingWithNamedWriteable() throws IOException {
+        NamedHolder n = new NamedHolder(new Example(randomAlphaOfLength(5)));
+        DelayableWriteable<NamedHolder> original = DelayableWriteable.referencing(n);
+        assertFalse(original.isDelayed());
+        roundTripTestCase(original, NamedHolder::new);
+    }
+
+    public void testRoundTripFromDelayed() throws IOException {
+        Example e = new Example(randomAlphaOfLength(5));
+        DelayableWriteable<Example> original = roundTrip(DelayableWriteable.referencing(e), Example::new, Version.CURRENT);
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, Example::new);
+    }
+
+    public void testRoundTripFromDelayedWithNamedWriteable() throws IOException {
+        NamedHolder n = new NamedHolder(new Example(randomAlphaOfLength(5)));
+        DelayableWriteable<NamedHolder> original = roundTrip(DelayableWriteable.referencing(n), NamedHolder::new, Version.CURRENT);
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, NamedHolder::new);
+    }
+
+    public void testRoundTripFromDelayedFromOldVersion() throws IOException {
+        Example e = new Example(randomAlphaOfLength(5));
+        DelayableWriteable<Example> original = roundTrip(DelayableWriteable.referencing(e), Example::new, randomOldVersion());
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, Example::new);
+    }
+
+    public void testRoundTripFromDelayedFromOldVersionWithNamedWriteable() throws IOException {
+        NamedHolder n = new NamedHolder(new Example(randomAlphaOfLength(5)));
+        DelayableWriteable<NamedHolder> original = roundTrip(DelayableWriteable.referencing(n), NamedHolder::new, randomOldVersion());
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, NamedHolder::new);
+    }
+
+    private <T extends Writeable> void roundTripTestCase(DelayableWriteable<T> original, Writeable.Reader<T> reader) throws IOException {
+        DelayableWriteable<T> roundTripped = roundTrip(original, reader, Version.CURRENT);
+        assertTrue(roundTripped.isDelayed());
+        assertThat(roundTripped.get(), equalTo(original.get()));
+    }
+
+    private <T extends Writeable> DelayableWriteable<T> roundTrip(DelayableWriteable<T> original,
+            Writeable.Reader<T> reader, Version version) throws IOException {
+        return copyInstance(original, writableRegistry(), (out, d) -> d.writeTo(out),
+            in -> DelayableWriteable.delayed(reader, in), version);
+    }
+
+    @Override
+    protected NamedWriteableRegistry writableRegistry() {
+        return new NamedWriteableRegistry(singletonList(
+                new NamedWriteableRegistry.Entry(Example.class, "example", Example::new)));
+    }
+
+    private static Version randomOldVersion() {
+        return randomValueOtherThanMany(Version.CURRENT::before, () -> VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index 04a0d71b650a9..3174c2c4fe8f0 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -85,8 +85,8 @@ public void testSerialization() throws Exception {
         assertEquals(querySearchResult.size(), deserialized.size());
         assertEquals(querySearchResult.hasAggs(), deserialized.hasAggs());
         if (deserialized.hasAggs()) {
-            Aggregations aggs = querySearchResult.consumeAggs();
-            Aggregations deserializedAggs = deserialized.consumeAggs();
+            Aggregations aggs = querySearchResult.consumeAggs().get();
+            Aggregations deserializedAggs = deserialized.consumeAggs().get();
             assertEquals(aggs.asList(), deserializedAggs.asList());
             List<SiblingPipelineAggregator> pipelineAggs = ((InternalAggregations) aggs).getTopLevelPipelineAggregators();
             List<SiblingPipelineAggregator> deserializedPipelineAggs =

From 8264bdd36a2db3c204387c4f8c8a846001bee899 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 23 Mar 2020 09:11:16 -0700
Subject: [PATCH 213/285] Revert "Introduce system index APIs for Kibana
 (#52385)" (#53912)

This reverts commit 4c0e8f12cbc078c1b85a67af76ed8a4be11a6f01.

It should be re-added once #53909 is addressed.
---
 build.gradle                                  |   4 +-
 modules/kibana/build.gradle                   |  31 ---
 .../elasticsearch/kibana/KibanaPlugin.java    | 145 ----------
 .../kibana/KibanaPluginTests.java             |  46 ----
 .../kibana/KibanaSystemIndexIT.java           | 249 ------------------
 .../tasksplugin/TasksPlugin.java              |   3 +-
 .../tasksplugin/TasksPluginTests.java         |   3 +-
 .../elasticsearch/action/ActionModule.java    |  11 +-
 .../PublicationTransportHandler.java          |   6 +-
 .../common/compress/DeflateCompressor.java    |   9 +-
 .../org/elasticsearch/common/io/Streams.java  |  11 -
 .../common/settings/ClusterSettings.java      |  18 +-
 .../common/util/concurrent/ThreadContext.java |  87 +-----
 .../java/org/elasticsearch/node/Node.java     |   4 +-
 .../plugins/SystemIndexPlugin.java            |   4 +-
 .../elasticsearch/rest/BaseRestHandler.java   |  53 ----
 .../elasticsearch/rest/RestController.java    |  17 +-
 .../action/RestCancellableNodeClient.java     |  52 ++--
 .../rest/action/document/RestIndexAction.java |  11 +-
 .../CompressibleBytesOutputStream.java        |  11 -
 .../transport/InboundMessage.java             |  35 ++-
 .../transport/OutboundMessage.java            |   3 +-
 .../transport/TransportLogger.java            |  16 +-
 .../action/ActionModuleTests.java             |   6 +-
 .../common/compress/DeflateCompressTests.java |   7 -
 .../util/concurrent/ThreadContextTests.java   | 133 +---------
 .../rest/RestControllerTests.java             |  33 +--
 .../rest/RestHttpResponseHeadersTests.java    |   2 +-
 .../indices/RestValidateQueryActionTests.java |   2 +-
 .../cat/RestCatRecoveryActionTests.java       |   2 +-
 .../action/cat/RestIndicesActionTests.java    |   3 +-
 .../action/document/RestIndexActionTests.java |   7 +-
 .../CompressibleBytesOutputStreamTests.java   |  13 -
 .../test/rest/RestActionTestCase.java         |   2 +-
 .../AbstractSimpleTransportTestCase.java      |  12 +-
 .../xpack/enrich/EnrichPlugin.java            |   2 +-
 .../xpack/logstash/Logstash.java              |   3 +-
 .../xpack/ml/MachineLearning.java             |   2 +-
 .../xpack/security/Security.java              |   2 +-
 .../authc/AuthenticationServiceTests.java     |   2 +-
 .../xpack/transform/Transform.java            |   2 +-
 .../elasticsearch/xpack/watcher/Watcher.java  |   2 +-
 42 files changed, 116 insertions(+), 950 deletions(-)
 delete mode 100644 modules/kibana/build.gradle
 delete mode 100644 modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
 delete mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java
 delete mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java

diff --git a/build.gradle b/build.gradle
index 17cd5f6b93cd0..da21449b30325 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53912" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle
deleted file mode 100644
index f9d11e5a6c58b..0000000000000
--- a/modules/kibana/build.gradle
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-esplugin {
-  description 'Plugin exposing APIs for Kibana system indices'
-  classname 'org.elasticsearch.kibana.KibanaPlugin'
-}
-
-dependencies {
-  compile project(path: ':modules:reindex', configuration: 'runtime')
-}
-
-testClusters.integTest {
-  module file(project(':modules:reindex').tasks.bundlePlugin.archiveFile)
-}
diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
deleted file mode 100644
index f917c477493cc..0000000000000
--- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.kibana;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.index.reindex.RestDeleteByQueryAction;
-import org.elasticsearch.indices.SystemIndexDescriptor;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SystemIndexPlugin;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction;
-import org.elasticsearch.rest.action.admin.indices.RestIndexPutAliasAction;
-import org.elasticsearch.rest.action.admin.indices.RestRefreshAction;
-import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction;
-import org.elasticsearch.rest.action.document.RestBulkAction;
-import org.elasticsearch.rest.action.document.RestDeleteAction;
-import org.elasticsearch.rest.action.document.RestGetAction;
-import org.elasticsearch.rest.action.document.RestIndexAction;
-import org.elasticsearch.rest.action.document.RestIndexAction.AutoIdHandler;
-import org.elasticsearch.rest.action.document.RestIndexAction.CreateHandler;
-import org.elasticsearch.rest.action.document.RestMultiGetAction;
-import org.elasticsearch.rest.action.document.RestUpdateAction;
-import org.elasticsearch.rest.action.search.RestClearScrollAction;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.rest.action.search.RestSearchScrollAction;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
-
-    public static final Setting<List<String>> KIBANA_INDEX_NAMES_SETTING = Setting.listSetting("kibana.system_indices",
-        List.of(".kibana*", ".reporting"), Function.identity(), Property.NodeScope);
-
-    @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return KIBANA_INDEX_NAMES_SETTING.get(settings).stream()
-            .map(pattern -> new SystemIndexDescriptor(pattern, "System index used by kibana"))
-            .collect(Collectors.toUnmodifiableList());
-    }
-
-    @Override
-    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
-                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
-                                             IndexNameExpressionResolver indexNameExpressionResolver,
-                                             Supplier<DiscoveryNodes> nodesInCluster) {
-        // TODO need to figure out what subset of system indices Kibana should have access to via these APIs
-        final List<String> allowedIndexPatterns = List.of();
-        return List.of(
-            // Based on https://github.com/elastic/kibana/issues/49764
-            // apis needed to perform migrations... ideally these will go away
-            new KibanaWrappedRestHandler(new RestCreateIndexAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestGetAliasesAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestIndexPutAliasAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestRefreshAction(), allowedIndexPatterns),
-
-            // apis needed to access saved objects
-            new KibanaWrappedRestHandler(new RestGetAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestMultiGetAction(settings), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestSearchAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestBulkAction(settings), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestDeleteAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestDeleteByQueryAction(), allowedIndexPatterns),
-
-            // api used for testing
-            new KibanaWrappedRestHandler(new RestUpdateSettingsAction(), allowedIndexPatterns),
-
-            // apis used specifically by reporting
-            new KibanaWrappedRestHandler(new RestGetIndicesAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestIndexAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new CreateHandler(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new AutoIdHandler(nodesInCluster), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestUpdateAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestSearchScrollAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestClearScrollAction(), allowedIndexPatterns)
-        );
-
-    }
-
-    @Override
-    public List<Setting<?>> getSettings() {
-        return List.of(KIBANA_INDEX_NAMES_SETTING);
-    }
-
-    static class KibanaWrappedRestHandler extends BaseRestHandler.Wrapper {
-
-        private final List<String> allowedIndexPatterns;
-
-        KibanaWrappedRestHandler(BaseRestHandler delegate, List<String> allowedIndexPatterns) {
-            super(delegate);
-            this.allowedIndexPatterns = allowedIndexPatterns;
-        }
-
-        @Override
-        public String getName() {
-            return "kibana_" + super.getName();
-        }
-
-        @Override
-        public List<Route> routes() {
-            return super.routes().stream().map(route -> new Route(route.getMethod(), "/_kibana" + route.getPath()))
-                .collect(Collectors.toUnmodifiableList());
-        }
-
-        @Override
-        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-            client.threadPool().getThreadContext().allowSystemIndexAccess(allowedIndexPatterns);
-            return super.prepareRequest(request, client);
-        }
-    }
-}
diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java
deleted file mode 100644
index 5094dd7178bcb..0000000000000
--- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java
+++ /dev/null
@@ -1,46 +0,0 @@
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.kibana;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.SystemIndexDescriptor;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.is;
-
-public class KibanaPluginTests extends ESTestCase {
-
-    public void testKibanaIndexNames() {
-        assertThat(new KibanaPlugin().getSettings(), contains(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING));
-        assertThat(new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY).stream()
-            .map(SystemIndexDescriptor::getIndexPattern).collect(Collectors.toUnmodifiableList()),
-            contains(".kibana*", ".reporting"));
-        final List<String> names = List.of("." + randomAlphaOfLength(4), "." + randomAlphaOfLength(6));
-        final List<String> namesFromDescriptors = new KibanaPlugin().getSystemIndexDescriptors(
-                Settings.builder().putList(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING.getKey(), names).build()
-            ).stream().map(SystemIndexDescriptor::getIndexPattern).collect(Collectors.toUnmodifiableList());
-        assertThat(namesFromDescriptors, is(names));
-    }
-}
diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
deleted file mode 100644
index f3901112e839f..0000000000000
--- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.kibana;
-
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.ESRestTestCase;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.is;
-
-public class KibanaSystemIndexIT extends ESRestTestCase {
-
-    public void testCreateIndex() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testAliases() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("PUT", "/_kibana/.kibana-1/_alias/.kibana");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("GET", "/_kibana/_aliases");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(EntityUtils.toString(response.getEntity()), containsString(".kibana"));
-    }
-
-    public void testBulkToKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testRefresh() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("GET", "/_kibana/.kibana/_refresh");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request getRequest = new Request("GET", "/_kibana/.kibana/_doc/1");
-        Response getResponse = client().performRequest(getRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-    }
-
-    public void testGetFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request getRequest = new Request("GET", "/_kibana/.kibana/_doc/1");
-        Response getResponse = client().performRequest(getRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-    }
-
-    public void testMultiGetFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request getRequest = new Request("GET", "/_kibana/_mget");
-        getRequest.setJsonEntity("{ \"docs\" : [ { \"_index\" : \".kibana\", \"_id\" : \"1\" }, " +
-            "{ \"_index\" : \".kibana\", \"_id\" : \"2\" } ] }\n");
-        Response getResponse = client().performRequest(getRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-        assertThat(responseBody, containsString("baz"));
-        assertThat(responseBody, containsString("tag"));
-    }
-
-    public void testSearchFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request searchRequest = new Request("GET", "/_kibana/.kibana/_search");
-        searchRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n");
-        Response getResponse = client().performRequest(searchRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-        assertThat(responseBody, containsString("baz"));
-        assertThat(responseBody, containsString("tag"));
-    }
-
-    public void testDeleteFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request deleteRequest = new Request("DELETE", "/_kibana/.kibana/_doc/1");
-        Response deleteResponse = client().performRequest(deleteRequest);
-        assertThat(deleteResponse.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testDeleteByQueryFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request dbqRequest = new Request("POST", "/_kibana/.kibana/_delete_by_query");
-        dbqRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n");
-        Response dbqResponse = client().performRequest(dbqRequest);
-        assertThat(dbqResponse.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testUpdateIndexSettings() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("PUT", "/_kibana/.kibana-1/_settings");
-        request.setJsonEntity("{ \"index.blocks.read_only\" : false }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testGetIndex() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("GET", "/_kibana/.kibana-1");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(EntityUtils.toString(response.getEntity()), containsString(".kibana-1"));
-    }
-
-    public void testIndexingAndUpdatingDocs() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1/_doc/1");
-        request.setJsonEntity("{ \"foo\" : \"bar\" }");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(201));
-
-        request = new Request("PUT", "/_kibana/.kibana-1/_create/2");
-        request.setJsonEntity("{ \"foo\" : \"bar\" }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(201));
-
-        request = new Request("POST", "/_kibana/.kibana-1/_doc");
-        request.setJsonEntity("{ \"foo\" : \"bar\" }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(201));
-
-        request = new Request("GET", "/_kibana/.kibana-1/_refresh");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("POST", "/_kibana/.kibana-1/_update/1");
-        request.setJsonEntity("{ \"doc\" : { \"foo\" : \"baz\" } }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testScrollingDocs() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"3\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request searchRequest = new Request("GET", "/_kibana/.kibana/_search");
-        searchRequest.setJsonEntity("{ \"size\" : 1,\n\"query\" : { \"match_all\" : {} } }\n");
-        searchRequest.addParameter("scroll", "1m");
-        response = client().performRequest(searchRequest);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
-        assertNotNull(map.get("_scroll_id"));
-        String scrollId = (String) map.get("_scroll_id");
-
-        Request scrollRequest = new Request("POST", "/_kibana/_search/scroll");
-        scrollRequest.addParameter("scroll_id", scrollId);
-        scrollRequest.addParameter("scroll", "1m");
-        response = client().performRequest(scrollRequest);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
-        assertNotNull(map.get("_scroll_id"));
-        scrollId = (String) map.get("_scroll_id");
-
-        Request clearScrollRequest = new Request("DELETE", "/_kibana/_search/scroll");
-        clearScrollRequest.addParameter("scroll_id", scrollId);
-        response = client().performRequest(clearScrollRequest);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-}
diff --git a/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java b/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java
index 0467b9419c778..b7d63991877db 100644
--- a/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java
+++ b/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.tasksplugin;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
@@ -35,7 +34,7 @@
 public class TasksPlugin extends Plugin implements SystemIndexPlugin {
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(new SystemIndexDescriptor(TASK_INDEX, this.getClass().getSimpleName()));
     }
 }
diff --git a/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java b/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java
index 23b873e377eb3..48ec1e06098f3 100644
--- a/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java
+++ b/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.tasksplugin;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
@@ -28,6 +27,6 @@ public class TasksPluginTests extends ESTestCase {
     public void testDummy() {
         // This is a dummy test case to satisfy the conventions
         TasksPlugin plugin = new TasksPlugin();
-        assertThat(plugin.getSystemIndexDescriptors(Settings.EMPTY), Matchers.hasSize(1));
+        assertThat(plugin.getSystemIndexDescriptors(), Matchers.hasSize(1));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 09814a4059b71..c64ba9771ff39 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -215,6 +215,7 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.TypeLiteral;
@@ -419,17 +420,19 @@ public class ActionModule extends AbstractModule {
     private final RestController restController;
     private final RequestValidators<PutMappingRequest> mappingRequestValidators;
     private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
+    private final ClusterService clusterService;
 
     public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
                         IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
                         ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
-                        CircuitBreakerService circuitBreakerService, UsageService usageService) {
+                        CircuitBreakerService circuitBreakerService, UsageService usageService, ClusterService clusterService) {
         this.settings = settings;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.indexScopedSettings = indexScopedSettings;
         this.clusterSettings = clusterSettings;
         this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
+        this.clusterService = clusterService;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
         autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
@@ -454,10 +457,10 @@ public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpr
         indicesAliasesRequestRequestValidators = new RequestValidators<>(
                 actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList()));
 
-        final boolean restrictSystemIndices = RestController.RESTRICT_SYSTEM_INDICES.get(settings);
-        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService, restrictSystemIndices);
+        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService);
     }
 
+
     public Map<String, ActionHandler<?, ?>> getActions() {
         return actions;
     }
@@ -698,7 +701,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
 
         registerHandler.accept(new RestIndexAction());
         registerHandler.accept(new CreateHandler());
-        registerHandler.accept(new AutoIdHandler(nodesInCluster));
+        registerHandler.accept(new AutoIdHandler(clusterService));
         registerHandler.accept(new RestGetAction());
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index 1a6db35a8091e..f5c325d3915d4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -321,8 +321,8 @@ private void sendClusterStateDiff(ClusterState clusterState,
 
     public static BytesReference serializeFullClusterState(ClusterState clusterState, Version nodeVersion) throws IOException {
         final BytesStreamOutput bStream = new BytesStreamOutput();
-        bStream.setVersion(nodeVersion);
         try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) {
+            stream.setVersion(nodeVersion);
             stream.writeBoolean(true);
             clusterState.writeTo(stream);
         }
@@ -331,8 +331,8 @@ public static BytesReference serializeFullClusterState(ClusterState clusterState
 
     public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVersion) throws IOException {
         final BytesStreamOutput bStream = new BytesStreamOutput();
-        bStream.setVersion(nodeVersion);
         try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) {
+            stream.setVersion(nodeVersion);
             stream.writeBoolean(false);
             diff.writeTo(stream);
         }
@@ -342,12 +342,12 @@ public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVe
     private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException {
         final Compressor compressor = CompressorFactory.compressor(request.bytes());
         StreamInput in = request.bytes().streamInput();
-        in.setVersion(request.version());
         try {
             if (compressor != null) {
                 in = compressor.streamInput(in);
             }
             in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
+            in.setVersion(request.version());
             // If true we received full cluster state - otherwise diffs
             if (in.readBoolean()) {
                 final ClusterState incomingState;
diff --git a/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java b/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
index 646e6c6138230..794a8db4960c6 100644
--- a/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
+++ b/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
@@ -85,7 +85,7 @@ public StreamInput streamInput(StreamInput in) throws IOException {
         final Inflater inflater = new Inflater(nowrap);
         InputStream decompressedIn = new InflaterInputStream(in, inflater, BUFFER_SIZE);
         decompressedIn = new BufferedInputStream(decompressedIn, BUFFER_SIZE);
-        final InputStreamStreamInput inputStreamStreamInput = new InputStreamStreamInput(decompressedIn) {
+        return new InputStreamStreamInput(decompressedIn) {
             final AtomicBoolean closed = new AtomicBoolean(false);
 
             public void close() throws IOException {
@@ -99,9 +99,6 @@ public void close() throws IOException {
                 }
             }
         };
-
-        inputStreamStreamInput.setVersion(in.getVersion());
-        return inputStreamStreamInput;
     }
 
     @Override
@@ -112,7 +109,7 @@ public StreamOutput streamOutput(StreamOutput out) throws IOException {
         final boolean syncFlush = true;
         DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
         OutputStream compressedOut = new BufferedOutputStream(deflaterOutputStream, BUFFER_SIZE);
-        final OutputStreamStreamOutput outputStreamStreamOutput = new OutputStreamStreamOutput(compressedOut) {
+        return new OutputStreamStreamOutput(compressedOut) {
             final AtomicBoolean closed = new AtomicBoolean(false);
 
             public void close() throws IOException {
@@ -126,7 +123,5 @@ public void close() throws IOException {
                 }
             }
         };
-        outputStreamStreamOutput.setVersion(out.getVersion());
-        return outputStreamStreamOutput;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/Streams.java b/server/src/main/java/org/elasticsearch/common/io/Streams.java
index 3747c4d895a20..222f94e65ef6a 100644
--- a/server/src/main/java/org/elasticsearch/common/io/Streams.java
+++ b/server/src/main/java/org/elasticsearch/common/io/Streams.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.common.io;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStream;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -297,15 +296,5 @@ public void reset() throws IOException {
         public BytesReference bytes() {
             return delegate.bytes();
         }
-
-        @Override
-        public Version getVersion() {
-            return delegate.getVersion();
-        }
-
-        @Override
-        public void setVersion(Version version) {
-            delegate.setVersion(version);
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index d121752017c58..88b2e83c4212e 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.common.settings;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.Build;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
 import org.elasticsearch.action.search.TransportSearchAction;
@@ -98,7 +97,6 @@
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
@@ -113,7 +111,6 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -183,9 +180,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         }
     }
 
-    public static final Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS;
-    static {
-        final Set<Setting<?>> alwaysEnabled = Set.of(
+    public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Set.of(
             AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
             AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
             BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
@@ -482,15 +477,6 @@ public void apply(Settings value, Settings current, Settings previous) {
             HandshakingTransportAddressConnector.PROBE_CONNECT_TIMEOUT_SETTING,
             HandshakingTransportAddressConnector.PROBE_HANDSHAKE_TIMEOUT_SETTING);
 
-        if (Build.CURRENT.isSnapshot()) {
-            Set<Setting<?>> modifiable = new HashSet<>(alwaysEnabled);
-            modifiable.add(RestController.RESTRICT_SYSTEM_INDICES);
-            BUILT_IN_CLUSTER_SETTINGS = Set.copyOf(modifiable);
-        } else {
-            BUILT_IN_CLUSTER_SETTINGS = alwaysEnabled;
-        }
-    }
-
-    static final List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
+    static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
 
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index edc2c563ae2a3..e4cb14857932f 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -20,10 +20,8 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.client.OriginSettingClient;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -50,7 +48,6 @@
 import java.util.stream.Collector;
 import java.util.stream.Stream;
 
-import static java.util.Collections.emptyList;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE;
 
@@ -65,7 +62,7 @@
  * Consumers of ThreadContext usually don't need to interact with adding or stashing contexts. Every elasticsearch thread is managed by
  * a thread pool or executor being responsible for stashing and restoring the threads context. For instance if a network request is
  * received, all headers are deserialized from the network and directly added as the headers of the threads {@link ThreadContext}
- * (see {@link #readFrom(StreamInput)}. In order to not modify the context that is currently active on this thread the network code
+ * (see {@link #readHeaders(StreamInput)}. In order to not modify the context that is currently active on this thread the network code
  * uses a try/with pattern to stash it's current context, read headers into a fresh one and once the request is handled or a handler thread
  * is forked (which in turn inherits the context) it restores the previous context. For instance:
  * </p>
@@ -233,18 +230,17 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     /**
-     * Reads the values from the stream into the current context
+     * Reads the headers from the stream into the current context
      */
-    public void readFrom(StreamInput in) throws IOException {
+    public void readHeaders(StreamInput in) throws IOException {
         final Tuple<Map<String, String>, Map<String, Set<String>>> streamTuple = readHeadersFromStream(in);
         final Map<String, String>  requestHeaders = streamTuple.v1();
         final Map<String, Set<String>> responseHeaders = streamTuple.v2();
-        final List<String> allowedSystemIndices = readAllowedSystemIndices(in);
         final ThreadContextStruct struct;
-        if (requestHeaders.isEmpty() && responseHeaders.isEmpty() && allowedSystemIndices.isEmpty()) {
+        if (requestHeaders.isEmpty() && responseHeaders.isEmpty()) {
             struct = ThreadContextStruct.EMPTY;
         } else {
-            struct = new ThreadContextStruct(requestHeaders, responseHeaders, Collections.emptyMap(), allowedSystemIndices, false, 0L);
+            struct = new ThreadContextStruct(requestHeaders, responseHeaders, Collections.emptyMap(), false);
         }
         threadLocal.set(struct);
     }
@@ -271,14 +267,6 @@ public static Tuple<Map<String, String>, Map<String, Set<String>>> readHeadersFr
         return new Tuple<>(requestHeaders, responseHeaders);
     }
 
-    public static List<String> readAllowedSystemIndices(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            return in.readOptionalStringList();
-        } else {
-            return emptyList();
-        }
-    }
-
     /**
      * Returns the header for the given key or <code>null</code> if not present
      */
@@ -422,36 +410,6 @@ public boolean isSystemContext() {
         return threadLocal.get().isSystemContext;
     }
 
-    /**
-     * Returns <code>true</code> if a request made within this context can access system indices
-     */
-    public boolean isSystemIndexAccessAllowed() {
-        return threadLocal.get().allowedSystemIndexPatterns != null;
-    }
-
-    /**
-     * Sets the context to disallow access to system indices
-     */
-    public void disallowSystemIndexAccess() {
-        threadLocal.set(threadLocal.get().setAllowSystemIndices(null));
-    }
-
-    /**
-     * Sets the context to allow access to system indices
-     */
-    public void allowSystemIndexAccess(List<String> patterns) {
-        threadLocal.set(threadLocal.get().setAllowSystemIndices(patterns));
-    }
-
-    /**
-     * Returns the list of allowed system index patterns or {@code null} if none are allowed. An
-     * empty list indicates that all system indices are allowed to be accessed.
-     */
-    @Nullable
-    public List<String> allowedSystemIndexPatterns() {
-        return threadLocal.get().allowedSystemIndexPatterns;
-    }
-
     @FunctionalInterface
     public interface StoredContext extends AutoCloseable {
         @Override
@@ -483,7 +441,6 @@ private static final class ThreadContextStruct {
         private final Map<String, String> requestHeaders;
         private final Map<String, Object> transientHeaders;
         private final Map<String, Set<String>> responseHeaders;
-        private final List<String> allowedSystemIndexPatterns;
         private final boolean isSystemContext;
         //saving current warning headers' size not to recalculate the size with every new warning header
         private final long warningHeadersSize;
@@ -498,40 +455,29 @@ private ThreadContextStruct setSystemContext() {
         private ThreadContextStruct(Map<String, String> requestHeaders,
                                     Map<String, Set<String>> responseHeaders,
                                     Map<String, Object> transientHeaders, boolean isSystemContext) {
-            this(requestHeaders, responseHeaders, transientHeaders, emptyList(), isSystemContext, 0L);
+            this.requestHeaders = requestHeaders;
+            this.responseHeaders = responseHeaders;
+            this.transientHeaders = transientHeaders;
+            this.isSystemContext = isSystemContext;
+            this.warningHeadersSize = 0L;
         }
 
         private ThreadContextStruct(Map<String, String> requestHeaders,
                                     Map<String, Set<String>> responseHeaders,
                                     Map<String, Object> transientHeaders, boolean isSystemContext,
                                     long warningHeadersSize) {
-            this(requestHeaders, responseHeaders, transientHeaders, emptyList(), isSystemContext, warningHeadersSize);
-        }
-
-        private ThreadContextStruct(Map<String, String> requestHeaders,
-                                    Map<String, Set<String>> responseHeaders,
-                                    Map<String, Object> transientHeaders,
-                                    List<String> allowedSystemIndexPatterns,
-                                    boolean isSystemContext,
-                                    long warningHeadersSize) {
             this.requestHeaders = requestHeaders;
             this.responseHeaders = responseHeaders;
             this.transientHeaders = transientHeaders;
             this.isSystemContext = isSystemContext;
             this.warningHeadersSize = warningHeadersSize;
-            this.allowedSystemIndexPatterns = allowedSystemIndexPatterns;
         }
 
         /**
          * This represents the default context and it should only ever be called by {@link #DEFAULT_CONTEXT}.
          */
         private ThreadContextStruct() {
-            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), emptyList(), false, 0L);
-        }
-
-        private ThreadContextStruct setAllowSystemIndices(List<String> allowedSystemIndexPatterns) {
-            final List<String> copy = allowedSystemIndexPatterns == null ? null : List.copyOf(allowedSystemIndexPatterns);
-            return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, copy, isSystemContext, warningHeadersSize);
+            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), false);
         }
 
         private ThreadContextStruct putRequest(String key, String value) {
@@ -575,8 +521,7 @@ private ThreadContextStruct putResponseHeaders(Map<String, Set<String>> headers)
                     newResponseHeaders.put(key, entry.getValue());
                 }
             }
-            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, allowedSystemIndexPatterns,
-                isSystemContext, 0L);
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext);
         }
 
         private ThreadContextStruct putResponse(final String key, final String value, final Function<String, String> uniqueValue,
@@ -626,8 +571,7 @@ private ThreadContextStruct putResponse(final String key, final String value, fi
                     return this;
                 }
             }
-            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, allowedSystemIndexPatterns,
-                isSystemContext, newWarningHeaderSize);
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext, newWarningHeaderSize);
         }
 
 
@@ -663,9 +607,6 @@ private void writeTo(StreamOutput out, Map<String, String> defaultHeaders) throw
             }
 
             out.writeMap(responseHeaders, StreamOutput::writeString, StreamOutput::writeStringCollection);
-            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-                out.writeOptionalStringCollection(allowedSystemIndexPatterns);
-            }
         }
     }
 
@@ -683,7 +624,7 @@ private ContextPreservingRunnable(Runnable in) {
 
         @Override
         public void run() {
-            try (ThreadContext.StoredContext ignore = stashContext()) {
+            try (ThreadContext.StoredContext ignore = stashContext()){
                 ctx.restore();
                 in.run();
             }
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 224c640c941ff..fb60be6f68548 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -440,7 +440,7 @@ protected Node(final Environment initialEnvironment,
                 .stream()
                 .collect(Collectors.toUnmodifiableMap(
                     plugin -> plugin.getClass().getSimpleName(),
-                    plugin -> plugin.getSystemIndexDescriptors(settings)));
+                    plugin -> plugin.getSystemIndexDescriptors()));
             SystemIndexDescriptor.checkForOverlappingPatterns(systemIndexDescriptorMap);
 
             final List<SystemIndexDescriptor> systemIndexDescriptors = systemIndexDescriptorMap.values().stream()
@@ -476,7 +476,7 @@ protected Node(final Environment initialEnvironment,
 
             ActionModule actionModule = new ActionModule(settings, clusterModule.getIndexNameExpressionResolver(),
                 settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
-                threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService);
+                threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService, clusterService);
             modules.add(actionModule);
 
             final RestController restController = actionModule.getRestController();
diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java
index 821f141b3eabc..a4d7a36f38670 100644
--- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.plugins;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 
 import java.util.Collection;
@@ -34,10 +33,9 @@ public interface SystemIndexPlugin extends ActionPlugin {
     /**
      * Returns a {@link Collection} of {@link SystemIndexDescriptor}s that describe this plugin's system indices, including
      * name, mapping, and settings.
-     * @param settings The node's settings
      * @return Descriptions of the system indices managed by this plugin.
      */
-    default Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    default Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.emptyList();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 101c3182fbb62..f2fd12bfa79c2 100644
--- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -183,57 +183,4 @@ protected Set<String> responseParams() {
         return Collections.emptySet();
     }
 
-    public static class Wrapper extends BaseRestHandler {
-
-        protected final BaseRestHandler delegate;
-
-        public Wrapper(BaseRestHandler delegate) {
-            this.delegate = delegate;
-        }
-
-        @Override
-        public String getName() {
-            return delegate.getName();
-        }
-
-        @Override
-        public List<Route> routes() {
-            return delegate.routes();
-        }
-
-        @Override
-        public List<DeprecatedRoute> deprecatedRoutes() {
-            return delegate.deprecatedRoutes();
-        }
-
-        @Override
-        public List<ReplacedRoute> replacedRoutes() {
-            return delegate.replacedRoutes();
-        }
-
-        @Override
-        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-            return delegate.prepareRequest(request, client);
-        }
-
-        @Override
-        protected Set<String> responseParams() {
-            return delegate.responseParams();
-        }
-
-        @Override
-        public boolean canTripCircuitBreaker() {
-            return delegate.canTripCircuitBreaker();
-        }
-
-        @Override
-        public boolean supportsContentStream() {
-            return delegate.supportsContentStream();
-        }
-
-        @Override
-        public boolean allowsUnsafeBuffers() {
-            return delegate.allowsUnsafeBuffers();
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 0537b4d40a044..bb015d6632e6b 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -31,8 +31,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.path.PathTrie;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -66,10 +64,6 @@ public class RestController implements HttpServerTransport.Dispatcher {
     private static final Logger logger = LogManager.getLogger(RestController.class);
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
-    // TODO once we are ready, this should default to true
-    public static final Setting<Boolean> RESTRICT_SYSTEM_INDICES =
-        Setting.boolSetting("rest.restrict_system_indices", false, Property.NodeScope);
-
     private final PathTrie<MethodHandlers> handlers = new PathTrie<>(RestUtils.REST_DECODER);
 
     private final UnaryOperator<RestHandler> handlerWrapper;
@@ -81,10 +75,9 @@ public class RestController implements HttpServerTransport.Dispatcher {
     /** Rest headers that are copied to internal requests made during a rest request. */
     private final Set<RestHeaderDefinition> headersToCopy;
     private final UsageService usageService;
-    private final boolean restrictSystemIndices;
 
     public RestController(Set<RestHeaderDefinition> headersToCopy, UnaryOperator<RestHandler> handlerWrapper,
-            NodeClient client, CircuitBreakerService circuitBreakerService, UsageService usageService, boolean restrictSystemIndices) {
+            NodeClient client, CircuitBreakerService circuitBreakerService, UsageService usageService) {
         this.headersToCopy = headersToCopy;
         this.usageService = usageService;
         if (handlerWrapper == null) {
@@ -93,7 +86,6 @@ public RestController(Set<RestHeaderDefinition> headersToCopy, UnaryOperator<Res
         this.handlerWrapper = handlerWrapper;
         this.client = client;
         this.circuitBreakerService = circuitBreakerService;
-        this.restrictSystemIndices = restrictSystemIndices;
     }
 
     /**
@@ -178,13 +170,6 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont
             handleFavicon(request.method(), request.uri(), channel);
             return;
         }
-
-        if (restrictSystemIndices) {
-            threadContext.disallowSystemIndexAccess();
-        } else {
-            assert threadContext.isSystemIndexAccessAllowed();
-        }
-
         try {
             tryAllHandlers(request, channel, threadContext);
         } catch (Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java
index 7d8e71502eb24..224c266bd2b18 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java
@@ -38,7 +38,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN;
@@ -86,41 +85,28 @@ static int getNumTasks(HttpChannel channel) {
     @Override
     public <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
         ActionType<Response> action, Request request, ActionListener<Response> listener) {
-        final AtomicBoolean created = new AtomicBoolean(false);
-        CloseListener closeListener = httpChannels.computeIfAbsent(httpChannel, channel -> {
-            created.set(true);
-            return new CloseListener();
-        });
+        CloseListener closeListener = httpChannels.computeIfAbsent(httpChannel, channel -> new CloseListener());
         TaskHolder taskHolder = new TaskHolder();
-        final Task task;
-        boolean success = false;
-        try {
-            task = client.executeLocally(action, request,
-                new ActionListener<>() {
-                    @Override
-                    public void onResponse(Response response) {
-                        try {
-                            closeListener.unregisterTask(taskHolder);
-                        } finally {
-                            listener.onResponse(response);
-                        }
+        Task task = client.executeLocally(action, request,
+            new ActionListener<>() {
+                @Override
+                public void onResponse(Response response) {
+                    try {
+                        closeListener.unregisterTask(taskHolder);
+                    } finally {
+                        listener.onResponse(response);
                     }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        try {
-                            closeListener.unregisterTask(taskHolder);
-                        } finally {
-                            listener.onFailure(e);
-                        }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    try {
+                        closeListener.unregisterTask(taskHolder);
+                    } finally {
+                        listener.onFailure(e);
                     }
-                });
-            success = true;
-        } finally {
-            if (success == false && created.get()) {
-                httpChannels.remove(httpChannel);
-            }
-        }
+                }
+            });
         final TaskId taskId = new TaskId(client.getLocalNodeId(), task.getId());
         closeListener.registerTask(taskHolder, taskId);
         closeListener.maybeRegisterChannel(httpChannel);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index f155b09aafb42..4a86ca118b176 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -33,7 +33,6 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -82,10 +81,10 @@ void validateOpType(String opType) {
 
     public static final class AutoIdHandler extends RestIndexAction {
 
-        private final Supplier<DiscoveryNodes> nodesInCluster;
+        private final ClusterService clusterService;
 
-        public AutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
-            this.nodesInCluster = nodesInCluster;
+        public AutoIdHandler(ClusterService clusterService) {
+            this.clusterService = clusterService;
         }
 
         @Override
@@ -101,7 +100,7 @@ public List<Route> routes() {
         @Override
         public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
             assert request.params().get("id") == null : "non-null id: " + request.params().get("id");
-            if (request.params().get("op_type") == null && nodesInCluster.get().getMinNodeVersion().onOrAfter(Version.V_7_5_0)) {
+            if (request.params().get("op_type") == null && clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_5_0)) {
                 // default to op_type create
                 request.params().put("op_type", "create");
             }
diff --git a/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java
index 7cb60a4326aa5..4116f88b14224 100644
--- a/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java
+++ b/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
@@ -103,14 +102,4 @@ public void close() throws IOException {
     public void reset() throws IOException {
         throw new UnsupportedOperationException();
     }
-
-    @Override
-    public Version getVersion() {
-        return stream.getVersion();
-    }
-
-    @Override
-    public void setVersion(Version version) {
-        stream.setVersion(version);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundMessage.java b/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
index 14db21bc86ec4..7aa0b1b6368f7 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
@@ -61,8 +61,7 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
             try (ThreadContext.StoredContext existing = threadContext.stashContext()) {
                 long requestId = streamInput.readLong();
                 byte status = streamInput.readByte();
-                final Version remoteVersion = Version.fromId(streamInput.readInt());
-                streamInput.setVersion(remoteVersion);
+                Version remoteVersion = Version.fromId(streamInput.readInt());
                 final boolean isHandshake = TransportStatus.isHandshake(status);
                 ensureVersionCompatibility(remoteVersion, version, isHandshake);
 
@@ -70,11 +69,10 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
                     // Consume the variable header size
                     streamInput.readInt();
                 } else {
-                    streamInput = decompressingStream(status, streamInput);
-                    assertRemoteVersion(streamInput, remoteVersion);
+                    streamInput = decompressingStream(status, remoteVersion, streamInput);
                 }
 
-                threadContext.readFrom(streamInput);
+                threadContext.readHeaders(streamInput);
 
                 InboundMessage message;
                 if (TransportStatus.isRequest(status)) {
@@ -85,18 +83,15 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
                     final String action = streamInput.readString();
 
                     if (remoteVersion.onOrAfter(TcpHeader.VERSION_WITH_HEADER_SIZE)) {
-                        streamInput = decompressingStream(status, streamInput);
-                        assertRemoteVersion(streamInput, remoteVersion);
+                        streamInput = decompressingStream(status, remoteVersion, streamInput);
                     }
-                    streamInput = namedWriteableStream(streamInput);
+                    streamInput = namedWriteableStream(streamInput, remoteVersion);
                     message = new Request(threadContext, remoteVersion, status, requestId, action, streamInput);
                 } else {
                     if (remoteVersion.onOrAfter(TcpHeader.VERSION_WITH_HEADER_SIZE)) {
-                        streamInput = decompressingStream(status, streamInput);
-                        assertRemoteVersion(streamInput, remoteVersion);
+                        streamInput = decompressingStream(status, remoteVersion, streamInput);
                     }
-                    streamInput = namedWriteableStream(streamInput);
-                    assertRemoteVersion(streamInput, remoteVersion);
+                    streamInput = namedWriteableStream(streamInput, remoteVersion);
                     message = new Response(threadContext, remoteVersion, status, requestId, streamInput);
                 }
                 success = true;
@@ -108,10 +103,12 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
             }
         }
 
-        static StreamInput decompressingStream(byte status, StreamInput streamInput) throws IOException {
+        static StreamInput decompressingStream(byte status, Version remoteVersion, StreamInput streamInput) throws IOException {
             if (TransportStatus.isCompress(status) && streamInput.available() > 0) {
                 try {
-                    return CompressorFactory.COMPRESSOR.streamInput(streamInput);
+                    StreamInput decompressor = CompressorFactory.COMPRESSOR.streamInput(streamInput);
+                    decompressor.setVersion(remoteVersion);
+                    return decompressor;
                 } catch (IllegalArgumentException e) {
                     throw new IllegalStateException("stream marked as compressed, but is missing deflate header");
                 }
@@ -120,12 +117,10 @@ static StreamInput decompressingStream(byte status, StreamInput streamInput) thr
             }
         }
 
-        private StreamInput namedWriteableStream(StreamInput delegate) {
-            return new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
-        }
-
-        static void assertRemoteVersion(StreamInput in, Version version) {
-            assert version.equals(in.getVersion()) : "Stream version [" + in.getVersion() + "] does not match version [" + version + "]";
+        private StreamInput namedWriteableStream(StreamInput delegate, Version remoteVersion) {
+            NamedWriteableAwareStreamInput streamInput = new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
+            streamInput.setVersion(remoteVersion);
+            return streamInput;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java b/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
index bae4cb0d982d5..16ba7a15a539c 100644
--- a/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
@@ -55,8 +55,7 @@ BytesReference serialize(BytesStreamOutput bytesStream) throws IOException {
         }
 
         try (CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bytesStream, TransportStatus.isCompress(status))) {
-            assert stream.getVersion().equals(version) :
-                "Stream version [" + stream.getVersion() + "] does not match version [" + version + "]";
+            stream.setVersion(version);
             if (variableHeaderLength == -1) {
                 writeVariableHeader(stream);
             }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportLogger.java b/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
index 7543ecfa304d4..310819969b718 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
@@ -28,8 +28,6 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.transport.InboundMessage.Reader.assertRemoteVersion;
-
 public final class TransportLogger {
 
     private static final Logger logger = LogManager.getLogger(TransportLogger.class);
@@ -77,8 +75,7 @@ private static String format(TcpChannel channel, BytesReference message, String
                 final byte status = streamInput.readByte();
                 final boolean isRequest = TransportStatus.isRequest(status);
                 final String type = isRequest ? "request" : "response";
-                final Version version = Version.fromId(streamInput.readInt());
-                streamInput.setVersion(version);
+                Version version = Version.fromId(streamInput.readInt());
                 sb.append(" [length: ").append(messageLengthWithHeader);
                 sb.append(", request id: ").append(requestId);
                 sb.append(", type: ").append(type);
@@ -87,18 +84,11 @@ private static String format(TcpChannel channel, BytesReference message, String
                 if (version.onOrAfter(TcpHeader.VERSION_WITH_HEADER_SIZE)) {
                     sb.append(", header size: ").append(streamInput.readInt()).append('B');
                 } else {
-                    streamInput = InboundMessage.Reader.decompressingStream(status, streamInput);
-                    assertRemoteVersion(streamInput, version);
+                    streamInput = InboundMessage.Reader.decompressingStream(status, version, streamInput);
                 }
 
-                // TODO (jaymode) Need a better way to deal with this. In one aspect,
-                // changes were made to ThreadContext to allocate less internally, yet we have this
-                // ugliness needed to move past the threadcontext data in the stream and discard it
-                // Could we have an alternative that essentially just seeks through the stream with
-                // minimal allocation?
-                // read and discard thread context data
+                // read and discard headers
                 ThreadContext.readHeadersFromStream(streamInput);
-                ThreadContext.readAllowedSystemIndices(streamInput);
 
                 if (isRequest) {
                     if (version.before(Version.V_8_0_0)) {
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 61118d31b3a2a..114abe0b2dcb7 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -109,7 +109,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() {
         UsageService usageService = new UsageService();
         ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                 settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
-                null, usageService);
+                null, usageService, null);
         actionModule.initRestHandlers(null);
         // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
         Exception e = expectThrows(IllegalArgumentException.class, () ->
@@ -141,7 +141,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             UsageService usageService = new UsageService();
             ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
-                    singletonList(dupsMainAction), null, null, usageService);
+                    singletonList(dupsMainAction), null, null, usageService, null);
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
             assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/] for method: GET"));
         } finally {
@@ -175,7 +175,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             UsageService usageService = new UsageService();
             ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
-                    singletonList(registersFakeHandler), null, null, usageService);
+                    singletonList(registersFakeHandler), null, null, usageService, null);
             actionModule.initRestHandlers(null);
             // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
             Exception e = expectThrows(IllegalArgumentException.class, () ->
diff --git a/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java b/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
index c02b4e460d28d..33d11aa23d890 100644
--- a/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
+++ b/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
@@ -21,13 +21,11 @@
 
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.TestUtil;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -390,12 +388,9 @@ private void doTest(byte bytes[]) throws IOException {
         StreamInput rawIn = new ByteBufferStreamInput(bb);
         Compressor c = compressor;
 
-        final Version version = VersionUtils.randomVersion(random());
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         OutputStreamStreamOutput rawOs = new OutputStreamStreamOutput(bos);
-        rawOs.setVersion(version);
         StreamOutput os = c.streamOutput(rawOs);
-        assertEquals(version, os.getVersion());
 
         Random r = random();
         int bufferSize = r.nextBoolean() ? 65535 : TestUtil.nextInt(random(), 1, 70000);
@@ -415,9 +410,7 @@ private void doTest(byte bytes[]) throws IOException {
         byte compressed[] = bos.toByteArray();
         ByteBuffer bb2 = ByteBuffer.wrap(compressed);
         StreamInput compressedIn = new ByteBufferStreamInput(bb2);
-        compressedIn.setVersion(version);
         StreamInput in = c.streamInput(compressedIn);
-        assertEquals(version, in.getVersion());
 
         // randomize constants again
         bufferSize = r.nextBoolean() ? 65535 : TestUtil.nextInt(random(), 1, 70000);
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index 5305b80c9828b..a2a456916e4d6 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -29,13 +29,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ThreadContextTests extends ESTestCase {
@@ -258,49 +254,6 @@ public void testSerialize() throws IOException {
             threadContext.addResponseHeader("Warning", "123456");
         }
         threadContext.addResponseHeader("Warning", "234567");
-        threadContext.disallowSystemIndexAccess();
-
-        BytesStreamOutput out = new BytesStreamOutput();
-        threadContext.writeTo(out);
-        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
-            assertNull(threadContext.getHeader("foo"));
-            assertNull(threadContext.getTransient("ctx.foo"));
-            assertTrue(threadContext.getResponseHeaders().isEmpty());
-            assertEquals("1", threadContext.getHeader("default"));
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-
-            threadContext.readFrom(out.bytes().streamInput());
-            assertEquals("bar", threadContext.getHeader("foo"));
-            assertNull(threadContext.getTransient("ctx.foo"));
-
-            final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
-            final List<String> warnings = responseHeaders.get("Warning");
-
-            assertThat(responseHeaders.keySet(), hasSize(1));
-            assertThat(warnings, hasSize(2));
-            assertThat(warnings, hasItem(equalTo("123456")));
-            assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertFalse(threadContext.isSystemIndexAccessAllowed());
-        }
-        assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
-        assertEquals("1", threadContext.getHeader("default"));
-        assertFalse(threadContext.isSystemIndexAccessAllowed());
-    }
-
-    public void testSerializeWithAllowedSystemIndexPatterns() throws IOException {
-        Settings build = Settings.builder().put("request.headers.default", "1").build();
-        ThreadContext threadContext = new ThreadContext(build);
-        threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", 1);
-        threadContext.addResponseHeader("Warning", "123456");
-        if (rarely()) {
-            threadContext.addResponseHeader("Warning", "123456");
-        }
-        threadContext.addResponseHeader("Warning", "234567");
-        final List<String> allowed = randomList(1, 8, () -> randomAlphaOfLengthBetween(2, 8));
-        threadContext.allowSystemIndexAccess(allowed);
 
         BytesStreamOutput out = new BytesStreamOutput();
         threadContext.writeTo(out);
@@ -309,9 +262,8 @@ public void testSerializeWithAllowedSystemIndexPatterns() throws IOException {
             assertNull(threadContext.getTransient("ctx.foo"));
             assertTrue(threadContext.getResponseHeaders().isEmpty());
             assertEquals("1", threadContext.getHeader("default"));
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
 
-            threadContext.readFrom(out.bytes().streamInput());
+            threadContext.readHeaders(out.bytes().streamInput());
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNull(threadContext.getTransient("ctx.foo"));
 
@@ -322,15 +274,10 @@ public void testSerializeWithAllowedSystemIndexPatterns() throws IOException {
             assertThat(warnings, hasSize(2));
             assertThat(warnings, hasItem(equalTo("123456")));
             assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
         }
         assertEquals("bar", threadContext.getHeader("foo"));
         assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
     }
 
     public void testSerializeInDifferentContext() throws IOException {
@@ -345,63 +292,17 @@ public void testSerializeInDifferentContext() throws IOException {
                 threadContext.addResponseHeader("Warning", "123456");
             }
             threadContext.addResponseHeader("Warning", "234567");
-            threadContext.disallowSystemIndexAccess();
-
-            assertEquals("bar", threadContext.getHeader("foo"));
-            assertNotNull(threadContext.getTransient("ctx.foo"));
-            assertEquals("1", threadContext.getHeader("default"));
-            assertThat(threadContext.getResponseHeaders().keySet(), hasSize(1));
-            assertFalse(threadContext.isSystemIndexAccessAllowed());
-            threadContext.writeTo(out);
-        }
-        {
-            Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
-            ThreadContext otherThreadContext = new ThreadContext(otherSettings);
-            otherThreadContext.readFrom(out.bytes().streamInput());
-
-            assertEquals("bar", otherThreadContext.getHeader("foo"));
-            assertNull(otherThreadContext.getTransient("ctx.foo"));
-            assertEquals("1", otherThreadContext.getHeader("default"));
-
-            final Map<String, List<String>> responseHeaders = otherThreadContext.getResponseHeaders();
-            final List<String> warnings = responseHeaders.get("Warning");
-
-            assertThat(responseHeaders.keySet(), hasSize(1));
-            assertThat(warnings, hasSize(2));
-            assertThat(warnings, hasItem(equalTo("123456")));
-            assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertFalse(otherThreadContext.isSystemIndexAccessAllowed());
-        }
-    }
-
-    public void testSerializeInDifferentContextWithAllowedSystemIndices() throws IOException {
-        final List<String> allowed = randomList(1, 8, () -> randomAlphaOfLengthBetween(2, 8));
-        BytesStreamOutput out = new BytesStreamOutput();
-        {
-            Settings build = Settings.builder().put("request.headers.default", "1").build();
-            ThreadContext threadContext = new ThreadContext(build);
-            threadContext.putHeader("foo", "bar");
-            threadContext.putTransient("ctx.foo", 1);
-            threadContext.addResponseHeader("Warning", "123456");
-            if (rarely()) {
-                threadContext.addResponseHeader("Warning", "123456");
-            }
-            threadContext.addResponseHeader("Warning", "234567");
-            threadContext.allowSystemIndexAccess(allowed);
 
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNotNull(threadContext.getTransient("ctx.foo"));
             assertEquals("1", threadContext.getHeader("default"));
             assertThat(threadContext.getResponseHeaders().keySet(), hasSize(1));
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
             threadContext.writeTo(out);
         }
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
             ThreadContext otherThreadContext = new ThreadContext(otherSettings);
-            otherThreadContext.readFrom(out.bytes().streamInput());
+            otherThreadContext.readHeaders(out.bytes().streamInput());
 
             assertEquals("bar", otherThreadContext.getHeader("foo"));
             assertNull(otherThreadContext.getTransient("ctx.foo"));
@@ -414,9 +315,6 @@ public void testSerializeInDifferentContextWithAllowedSystemIndices() throws IOE
             assertThat(warnings, hasSize(2));
             assertThat(warnings, hasItem(equalTo("123456")));
             assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertTrue(otherThreadContext.isSystemIndexAccessAllowed());
-            assertThat(otherThreadContext.allowedSystemIndexPatterns(), equalTo(allowed));
         }
     }
 
@@ -435,7 +333,7 @@ public void testSerializeInDifferentContextNoDefaults() throws IOException {
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
             ThreadContext otherhreadContext = new ThreadContext(otherSettings);
-            otherhreadContext.readFrom(out.bytes().streamInput());
+            otherhreadContext.readHeaders(out.bytes().streamInput());
 
             assertEquals("bar", otherhreadContext.getHeader("foo"));
             assertNull(otherhreadContext.getTransient("ctx.foo"));
@@ -715,31 +613,6 @@ public void testPutHeaders() {
         assertEquals("value for key [foo] already present", e.getMessage());
     }
 
-    public void testSystemIndexAccessAllowed() {
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), empty());
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            threadContext.disallowSystemIndexAccess();
-            assertFalse(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), nullValue());
-        }
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), empty());
-
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            threadContext.disallowSystemIndexAccess();
-            final List<String> allowed = randomList(1, 8, () -> randomAlphaOfLengthBetween(2, 8));
-            threadContext.allowSystemIndexAccess(allowed);
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), not(sameInstance(allowed)));
-            assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
-        }
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), empty());
-    }
-
     /**
      * Sometimes wraps a Runnable in an AbstractRunnable.
      */
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index dfcfd644dde34..415fe1bf713fa 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -91,7 +91,7 @@ public void setup() {
         inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
 
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, randomBoolean());
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (request, channel, client) -> channel.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
@@ -106,7 +106,7 @@ public void testApplyRelevantHeaders() throws Exception {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", true)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("true"));
         restHeaders.put("header.2", Collections.singletonList("true"));
@@ -142,7 +142,7 @@ public void testRequestWithDisallowedMultiValuedHeader() {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", false)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", List.of("foo", "bar"));
@@ -156,7 +156,7 @@ public void testRequestWithDisallowedMultiValuedHeaderButSameValues() {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", false)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", List.of("foo", "foo"));
@@ -210,7 +210,7 @@ public void testRegisterWithDeprecatedHandler() {
     }
 
     public void testRegisterSecondMethodWithDifferentNamedWildcard() {
-        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService);
 
         RestRequest.Method firstMethod = randomFrom(RestRequest.Method.values());
         RestRequest.Method secondMethod =
@@ -237,7 +237,7 @@ public void testRestHandlerWrapper() throws Exception {
                 h -> {
                     assertSame(handler, h);
                     return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
-                }, null, circuitBreakerService, usageService, randomBoolean());
+                }, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler);
         RestRequest request = testRestRequest("/wrapped", "{}", XContentType.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
@@ -300,7 +300,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() {
         String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
         RestRequest request = testRestRequest("/", content, null);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE);
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, randomBoolean());
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (r, c, client) -> c.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
@@ -603,25 +603,6 @@ public HttpRequest releaseAndCopy() {
         assertThat(channel.getRestResponse().getHeaders().get("Allow"), hasItem(equalTo(RestRequest.Method.GET.toString())));
     }
 
-    public void testDispatchRestrictSystemIndices() {
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, true);
-        restController.registerHandler(RestRequest.Method.GET, "/",
-            (request, channel, client) -> channel.sendResponse(
-                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
-        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
-            throw new IllegalArgumentException("test error");
-        });
-
-        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
-        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
-
-        assertFalse(channel.getSendResponseCalled());
-        ThreadContext context = new ThreadContext(Settings.EMPTY);
-        assertTrue(context.isSystemIndexAccessAllowed());
-        restController.dispatchRequest(fakeRestRequest, channel, context);
-        assertTrue(channel.getSendResponseCalled());
-        assertFalse(context.isSystemIndexAccessAllowed());
-    }
 
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
index 5b4183e72f3e2..6a4a8749397ab 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
@@ -89,7 +89,7 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception {
         final Settings settings = Settings.EMPTY;
         UsageService usageService = new UsageService();
         RestController restController = new RestController(Collections.emptySet(),
-                null, null, circuitBreakerService, usageService, randomBoolean());
+                null, null, circuitBreakerService, usageService);
 
         // A basic RestHandler handles requests to the endpoint
         RestHandler restHandler = new RestHandler() {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
index f7c060a48f527..4813e11e15bfc 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
@@ -58,7 +58,7 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase {
 
     private static UsageService usageService = new UsageService();
     private static RestController controller = new RestController(emptySet(), null, client,
-        new NoneCircuitBreakerService(), usageService, false);
+        new NoneCircuitBreakerService(), usageService);
     private static RestValidateQueryAction action = new RestValidateQueryAction();
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
index b62b9d7d0d3fd..258844f36b71c 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
@@ -56,7 +56,7 @@ public class RestCatRecoveryActionTests extends ESTestCase {
     public void testRestRecoveryAction() {
         final Settings settings = Settings.EMPTY;
         UsageService usageService = new UsageService();
-        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService, randomBoolean());
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService);
         final RestCatRecoveryAction action = new RestCatRecoveryAction();
         restController.registerHandler(action);
         final int totalShards = randomIntBetween(1, 32);
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index 060c2c545875f..aad58e50d69c9 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -127,8 +127,7 @@ public void testBuildTable() {
             }
         }
 
-        final RestController restController =
-            new RestController(Collections.emptySet(), null, null, null, new UsageService(), randomBoolean());
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, new UsageService());
         final RestIndicesAction action = new RestIndicesAction();
         restController.registerHandler(action);
         final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetaDatas);
diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
index bb549b64c724f..84bc5f3bef340 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
@@ -42,7 +43,9 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class RestIndexActionTests extends RestActionTestCase {
 
@@ -50,9 +53,11 @@ public class RestIndexActionTests extends RestActionTestCase {
 
     @Before
     public void setUpAction() {
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.state()).thenAnswer(invocationOnMock -> clusterStateSupplier.get());
         controller().registerHandler(new RestIndexAction());
         controller().registerHandler(new CreateHandler());
-        controller().registerHandler(new AutoIdHandler(() -> clusterStateSupplier.get().nodes()));
+        controller().registerHandler(new AutoIdHandler(clusterService));
     }
 
     public void testCreateOpTypeValidation() {
diff --git a/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java b/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java
index 90f251bbc5610..aeb92dac73479 100644
--- a/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -34,11 +33,7 @@ public class CompressibleBytesOutputStreamTests extends ESTestCase {
 
     public void testStreamWithoutCompression() throws IOException {
         BytesStream bStream = new ZeroOutOnCloseStream();
-        if (randomBoolean()) {
-            bStream.setVersion(VersionUtils.randomVersion(random()));
-        }
         CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bStream, false);
-        assertEquals(bStream.getVersion(), stream.getVersion());
 
         byte[] expectedBytes = randomBytes(randomInt(30));
         stream.write(expectedBytes);
@@ -66,11 +61,7 @@ public void testStreamWithoutCompression() throws IOException {
 
     public void testStreamWithCompression() throws IOException {
         BytesStream bStream = new ZeroOutOnCloseStream();
-        if (randomBoolean()) {
-            bStream.setVersion(VersionUtils.randomVersion(random()));
-        }
         CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bStream, true);
-        assertEquals(bStream.getVersion(), stream.getVersion());
 
         byte[] expectedBytes = randomBytes(randomInt(30));
         stream.write(expectedBytes);
@@ -97,11 +88,7 @@ public void testStreamWithCompression() throws IOException {
 
     public void testCompressionWithCallingMaterializeFails() throws IOException {
         BytesStream bStream = new ZeroOutOnCloseStream();
-        if (randomBoolean()) {
-            bStream.setVersion(VersionUtils.randomVersion(random()));
-        }
         CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bStream, true);
-        assertEquals(bStream.getVersion(), stream.getVersion());
 
         byte[] expectedBytes = randomBytes(between(1, 30));
         stream.write(expectedBytes);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java
index 9ac1269a044df..a5d932a3d1a3d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java
@@ -47,7 +47,7 @@ public void setUpController() {
         controller = new RestController(Collections.emptySet(), null,
             nodeClient,
             new NoneCircuitBreakerService(),
-            new UsageService(), randomBoolean());
+            new UsageService());
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 0a89b2811b1a6..610e313fb0610 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -2378,15 +2378,15 @@ public String executor() {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(2, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(113, transportStats.getTxSize().getBytes());
+                assertEquals(111, transportStats.getTxSize().getBytes());
             });
             sendResponseLatch.countDown();
             responseLatch.await();
             stats = serviceC.transport.getStats(); // response has been received
             assertEquals(2, stats.getRxCount());
             assertEquals(2, stats.getTxCount());
-            assertEquals(52, stats.getRxSize().getBytes());
-            assertEquals(113, stats.getTxSize().getBytes());
+            assertEquals(50, stats.getRxSize().getBytes());
+            assertEquals(111, stats.getTxSize().getBytes());
         } finally {
             serviceC.close();
         }
@@ -2493,7 +2493,7 @@ public String executor() {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(2, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(113, transportStats.getTxSize().getBytes());
+                assertEquals(111, transportStats.getTxSize().getBytes());
             });
             sendResponseLatch.countDown();
             responseLatch.await();
@@ -2507,8 +2507,8 @@ public String executor() {
             String failedMessage = "Unexpected read bytes size. The transport exception that was received=" + exception;
             // 49 bytes are the non-exception message bytes that have been received. It should include the initial
             // handshake message and the header, version, etc bytes in the exception message.
-            assertEquals(failedMessage, 55 + streamOutput.bytes().length(), stats.getRxSize().getBytes());
-            assertEquals(113, stats.getTxSize().getBytes());
+            assertEquals(failedMessage, 53 + streamOutput.bytes().length(), stats.getRxSize().getBytes());
+            assertEquals(111, stats.getTxSize().getBytes());
         } finally {
             serviceC.close();
         }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
index 30cce2da306da..6b472bec35a43 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
@@ -246,7 +246,7 @@ public List<Setting<?>> getSettings() {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(
             new SystemIndexDescriptor(ENRICH_INDEX_PATTERN, "Contains data to support enrich ingest processors.")
         );
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index 3d22402e89779..5b6378dab4e91 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
@@ -63,7 +62,7 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(
             new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, "Contains data for Logstash Central Management")
         );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 2a22a37841cd1..e17d06dbf9390 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -982,7 +982,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"),
             new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"),
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 1f62f6c341f87..2809aa0e6ec99 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -1088,7 +1088,7 @@ private synchronized NioGroupFactory getNioGroupFactory(Settings settings) {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"),
             new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index e616b07c3f54f..bcf795f874030 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -725,7 +725,7 @@ public void testAuthenticateTransportContextAndHeader() throws Exception {
                 threadContext2.writeTo(output);
                 StreamInput input = output.bytes().streamInput();
                 threadContext2 = new ThreadContext(Settings.EMPTY);
-                threadContext2.readFrom(input);
+                threadContext2.readHeaders(input);
                 header = threadContext2.getHeader(AuthenticationField.AUTHENTICATION_KEY);
             }
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
index f94237c9a5753..a58b3703c2974 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
@@ -393,7 +393,7 @@ public List<Entry> getNamedXContent() {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(
             new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, "Contains Transform configuration data")
         );
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index c22be9400da15..b9c44c22fd7a1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -690,7 +690,7 @@ public void reload(Settings settings) {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(Watch.INDEX, "Contains Watch definitions"),
             new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution")

From 34feb3cde197ec327cfce8d99688754035cb7b69 Mon Sep 17 00:00:00 2001
From: "C.J. Jameson" <cjcjameson@users.noreply.github.com>
Date: Mon, 23 Mar 2020 09:34:49 -0700
Subject: [PATCH 214/285] [DOCS] Clarify routing enforcement in docs (#53945)

Removes a mention of the `_doc` mapping type that's
no longer applicable now that mapping types are
removed/deprecated.
---
 docs/reference/mapping/fields/routing-field.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index 25c3571f08e8d..46a204ccddfde 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -92,7 +92,7 @@ PUT my_index2/_doc/1 <2>
 ------------------------------
 // TEST[catch:bad_request]
 
-<1> Routing is required for `_doc` documents.
+<1> Routing is required for all documents.
 <2> This index request throws a `routing_missing_exception`.
 
 ==== Unique IDs with custom routing
@@ -128,4 +128,4 @@ less than `index.number_of_shards`.
 Once enabled, the partitioned index will have the following limitations:
 
 *   Mappings with <<parent-join,`join` field>> relationships cannot be created within it.
-*   All mappings within the index must have the `_routing` field marked as required.
\ No newline at end of file
+*   All mappings within the index must have the `_routing` field marked as required.

From 78f473d979a75cf3005f8f98e4a04ff80cdd714b Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 23 Mar 2020 18:37:47 +0200
Subject: [PATCH 215/285] [ML] Fix typo in outlier detection timing stats
 (#53988)

The field holding the timing stats was mistakenly called
`timings_stats`.
---
 .../dataframe/stats/outlierdetection/OutlierDetectionStats.java | 2 +-
 .../dataframe/stats/outlierdetection/OutlierDetectionStats.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
index 15098e3770f3b..e3236dad0cd26 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -36,7 +36,7 @@ public class OutlierDetectionStats implements AnalysisStats {
 
     public static final ParseField TIMESTAMP = new ParseField("timestamp");
     public static final ParseField PARAMETERS = new ParseField("parameters");
-    public static final ParseField TIMING_STATS = new ParseField("timings_stats");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
 
     public static final ConstructingObjectParser<OutlierDetectionStats, Void> PARSER = new ConstructingObjectParser<>(
         NAME.getPreferredName(), true,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
index e1e24f36b87b9..6096496e792b0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -26,7 +26,7 @@ public class OutlierDetectionStats implements AnalysisStats {
     public static final String TYPE_VALUE = "outlier_detection_stats";
 
     public static final ParseField PARAMETERS = new ParseField("parameters");
-    public static final ParseField TIMING_STATS = new ParseField("timings_stats");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
 
     public static final ConstructingObjectParser<OutlierDetectionStats, Void> STRICT_PARSER = createParser(false);
     public static final ConstructingObjectParser<OutlierDetectionStats, Void> LENIENT_PARSER = createParser(true);

From 76659939dca3daf435c86d1a0e38ac4c2c485ac9 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 23 Mar 2020 18:39:16 +0200
Subject: [PATCH 216/285] [ML] Delete DF analytics stats upon job deletion
 (#53933)

Since a data frame analytics job may have associated docs
in the .ml-stats-* indices, when the job is deleted we
should delete those docs too.
---
 .../authz/store/ReservedRolesStore.java       |  2 +-
 ...ansportDeleteDataFrameAnalyticsAction.java | 52 +++++++++++++++--
 .../test/ml/data_frame_analytics_crud.yml     | 56 ++++++++++++++++++-
 3 files changed, 102 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index 4b6c35bb7ce85..0de81ccf15042 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -179,7 +179,7 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                 .put("machine_learning_admin", new RoleDescriptor("machine_learning_admin", new String[] { "manage_ml" },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".ml-anomalies*", ".ml-notifications*", ".ml-state*", ".ml-meta*")
+                                        .indices(".ml-anomalies*", ".ml-notifications*", ".ml-state*", ".ml-meta*", ".ml-stats-*")
                                         .privileges("view_index_metadata", "read").build(),
                                 RoleDescriptor.IndicesPrivileges.builder().indices(".ml-annotations*")
                                         .privileges("view_index_metadata", "read", "write").build()
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
index 47f2a216f2a81..422c690a5fc92 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -36,11 +37,13 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -138,20 +141,38 @@ private void normalDelete(ParentTaskAssigningClient parentTaskClient, ClusterSta
         // We clean up the memory tracker on delete because there is no stop; the task stops by itself
         memoryTracker.removeDataFrameAnalyticsJob(id);
 
-        // Step 3. Delete the config
+        // Step 4. Delete the config
+        ActionListener<BulkByScrollResponse> deleteStatsHandler = ActionListener.wrap(
+            bulkByScrollResponse -> {
+                if (bulkByScrollResponse.isTimedOut()) {
+                    logger.warn("[{}] DeleteByQuery for stats timed out", id);
+                }
+                if (bulkByScrollResponse.getBulkFailures().isEmpty() == false) {
+                    logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery for stats", id,
+                        bulkByScrollResponse.getBulkFailures().size(), bulkByScrollResponse.getVersionConflicts());
+                    for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) {
+                        logger.warn("[{}] DBQ failure: {}", id, failure);
+                    }
+                }
+                deleteConfig(parentTaskClient, id, listener);
+            },
+            listener::onFailure
+        );
+
+        // Step 3. Delete job docs from stats index
         ActionListener<BulkByScrollResponse> deleteStateHandler = ActionListener.wrap(
             bulkByScrollResponse -> {
                 if (bulkByScrollResponse.isTimedOut()) {
                     logger.warn("[{}] DeleteByQuery for state timed out", id);
                 }
                 if (bulkByScrollResponse.getBulkFailures().isEmpty() == false) {
-                    logger.warn("[{}] {} failures and {} conflicts encountered while runnint DeleteByQuery for state", id,
+                    logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery for state", id,
                         bulkByScrollResponse.getBulkFailures().size(), bulkByScrollResponse.getVersionConflicts());
                     for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) {
                         logger.warn("[{}] DBQ failure: {}", id, failure);
                     }
                 }
-                deleteConfig(parentTaskClient, id, listener);
+                deleteStats(parentTaskClient, id, deleteStatsHandler);
             },
             listener::onFailure
         );
@@ -193,8 +214,29 @@ private void deleteState(ParentTaskAssigningClient parentTaskClient,
         if (config.getAnalysis().persistsState()) {
             ids.add(config.getAnalysis().getStateDocId(config.getId()));
         }
-        DeleteByQueryRequest request = new DeleteByQueryRequest(AnomalyDetectorsIndex.jobStateIndexPattern());
-        request.setQuery(QueryBuilders.idsQuery().addIds(ids.toArray(String[]::new)));
+        executeDeleteByQuery(
+            parentTaskClient,
+            AnomalyDetectorsIndex.jobStateIndexPattern(),
+            QueryBuilders.idsQuery().addIds(ids.toArray(String[]::new)),
+            listener
+        );
+    }
+
+    private void deleteStats(ParentTaskAssigningClient parentTaskClient,
+                             String jobId,
+                             ActionListener<BulkByScrollResponse> listener) {
+        executeDeleteByQuery(
+            parentTaskClient,
+            MlStatsIndex.indexPattern(),
+            QueryBuilders.termQuery(Fields.JOB_ID.getPreferredName(), jobId),
+            listener
+        );
+    }
+
+    private void executeDeleteByQuery(ParentTaskAssigningClient parentTaskClient, String index, QueryBuilder query,
+                                      ActionListener<BulkByScrollResponse> listener) {
+        DeleteByQueryRequest request = new DeleteByQueryRequest(index);
+        request.setQuery(query);
         request.setIndicesOptions(MlIndicesUtils.addIgnoreUnavailable(IndicesOptions.lenientExpandOpen()));
         request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
         request.setAbortOnVersionConflict(false);
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index a5a99b30391ea..5ec3c978b73d6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -938,7 +938,7 @@ setup:
         id: "missing_config"
 
 ---
-"Test delete given config and state documents":
+"Test delete given config, state and stats documents":
   - skip:
       features: headers
   - do:
@@ -976,11 +976,33 @@ setup:
           {
           }
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      index:
+        index: .ml-stats-000001
+        id: "delete_foo_stats_1"
+        body: >
+          {
+            "job_id": "delete_foo"
+          }
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      index:
+        index: .ml-stats-000001
+        id: "delete_foo_stats_2"
+        body: >
+          {
+            "job_id": "delete_foo"
+          }
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: .ml-state*
+        index: .ml-stat*
 
   - do:
       search:
@@ -997,6 +1019,21 @@ setup:
 
   - match: { hits.total.value: 2 }
 
+  - do:
+      search:
+        index: .ml-stats-*
+        body:
+          size: 0
+          query:
+            bool:
+              filter:
+                ids:
+                  values:
+                    - "delete_foo_stats_1"
+                    - "delete_foo_stats_2"
+
+  - match: { hits.total.value: 2 }
+
   - do:
       ml.delete_data_frame_analytics:
         id: "delete_foo"
@@ -1016,6 +1053,21 @@ setup:
                     - "delete_foo_regression_state#1"
   - match: { hits.total.value: 0 }
 
+  - do:
+      search:
+        index: .ml-stats-*
+        body:
+          size: 0
+          query:
+            bool:
+              filter:
+                ids:
+                  values:
+                    - "delete_foo_stats_1"
+                    - "delete_foo_stats_2"
+
+  - match: { hits.total.value: 0 }
+
 ---
 "Test max model memory limit":
   - skip:

From bad7580f877288e1c5d5f046bd95d07e1c61c559 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Mon, 23 Mar 2020 09:40:25 -0700
Subject: [PATCH 217/285] [DOCS] Add generated_dest_index to preview transform
 API (#53905)

---
 .../transform/apis/preview-transform.asciidoc | 56 ++++++++++++++++---
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc
index 3ba1b945376cc..77effafa4aeea 100644
--- a/docs/reference/transform/apis/preview-transform.asciidoc
+++ b/docs/reference/transform/apis/preview-transform.asciidoc
@@ -30,7 +30,17 @@ on the source index for the {transform}. For more information, see
 This API generates a preview of the results that you will get when you run the
 <<put-transform,create {transforms} API>> with the same
 configuration. It returns a maximum of 100 results. The calculations are based
-on all the current data in the source index. 
+on all the current data in the source index.
+
+It also generates a list of mappings and settings for the destination index.
+If the destination index does not exist when you start a {transform}, these are
+the mappings and settings that are used. These values are determined based on
+the field types of the source index and the {transform} aggregations.
+
+TIP: There are some <<transform-aggresponse-limitations,limitations>> that
+might result in poor mappings. As a work-around, create the destination index
+or an index template with your preferred mappings before you start the
+{transform}.
 
 [[preview-transform-request-body]]
 ==== {api-request-body-title}
@@ -106,7 +116,16 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=sync-time-field]
 `preview`::
   (array) An array of documents. In particular, they are the JSON
   representation of the documents that would be created in the destination index
-  by the {transform}. 
+  by the {transform}.
+  
+`generated_dest_index`::
+  (object) Contains details about the destination index.
+    `mappings`:::
+    (object) The <<mapping,mappings>> for each document in the destination index. 
+    `settings`:::
+    (object) The <<index-modules-settings,index settings>> for the destination
+    index.
+    `aliases`::: The aliases for the destination index.
 
 ==== {api-examples-title}
 
@@ -156,15 +175,34 @@ The data that is returned for this example is as follows:
     }
     ...
   ],
-  "mappings": {
-    "properties": {
-      "max_price": {
-        "type": "double"
+  "generated_dest_index" : {
+    "mappings" : {
+      "_meta" : {
+        "_transform" : {
+          "transform" : "transform-preview",
+          "version" : {
+            "created" : "7.7.0"
+          },
+          "creation_date_in_millis" : 1584738236757
+        },
+        "created_by" : "transform"
       },
-      "customer_id": {
-        "type": "keyword"
+      "properties" : {
+        "max_price" : {
+          "type" : "half_float"
+        },
+        "customer_id" : {
+          "type" : "keyword"
+        }
       }
-    }
+    },
+    "settings" : {
+      "index" : {
+        "number_of_shards" : "1",
+        "auto_expand_replicas" : "0-1"
+      }
+    },
+    "aliases" : { }
   }
 }
 ----

From 0ea4324e22d150a0e5f336ee624508573f0d55cd Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Mon, 23 Mar 2020 09:48:00 -0700
Subject: [PATCH 218/285] [DOCS] Adds data nanos transform limitation (#53826)

---
 docs/reference/mapping/types/date_nanos.asciidoc | 4 ++--
 docs/reference/transform/limitations.asciidoc    | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc
index aadae3238cdf9..c5dc9481d3191 100644
--- a/docs/reference/mapping/types/date_nanos.asciidoc
+++ b/docs/reference/mapping/types/date_nanos.asciidoc
@@ -94,6 +94,6 @@ same mapping parameters than with the `date` field can be used.
 [[date-nanos-limitations]]
 ==== Limitations
 
-Aggregations are still on millisecond resolution, even when using a
-`date_nanos` field.
+Aggregations are still on millisecond resolution, even when using a `date_nanos`
+field. This limitation also affects <<transforms,{transforms}>>.
 
diff --git a/docs/reference/transform/limitations.asciidoc b/docs/reference/transform/limitations.asciidoc
index 5a66dd8190768..88430cf37db94 100644
--- a/docs/reference/transform/limitations.asciidoc
+++ b/docs/reference/transform/limitations.asciidoc
@@ -206,3 +206,10 @@ this entity will not be updated.
 If using a `sync.time.field` that represents the data ingest time and using a 
 zero second or very small `sync.time.delay`, then it is more likely that this 
 issue will occur.
+
+[[transform-date-nanos]]
+==== Support for date nanoseconds data type
+
+If your data uses the <<date_nanos,date nanosecond data type>>, aggregations
+are nonetheless on millisecond resolution. This limitation also affects the
+aggregations in your {transforms}.
\ No newline at end of file

From ea33795d98099a6690eeaabe04a3037b34e8db0c Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Mon, 23 Mar 2020 10:20:41 -0700
Subject: [PATCH 219/285] [DOCS] Fixes formatting in transform overview
 (#53900)

---
 docs/reference/transform/overview.asciidoc | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/docs/reference/transform/overview.asciidoc b/docs/reference/transform/overview.asciidoc
index 18a1db0f106ff..409928ff438c1 100644
--- a/docs/reference/transform/overview.asciidoc
+++ b/docs/reference/transform/overview.asciidoc
@@ -28,20 +28,18 @@ The second step is deciding how you want to aggregate the grouped data. When
 using aggregations, you practically ask questions about the index. There are 
 different types of aggregations, each with its own purpose and output. To learn 
 more about the supported aggregations and group-by fields, see 
-{ref}/transform-resource.html[{transform-cap} resources].
+<<put-transform>>.
 
 As an optional step, you can also add a query to further limit the scope of the
 aggregation.
 
 The {transform} performs a composite aggregation that paginates through all the 
 data defined by the source index query. The output of the aggregation is stored 
-in a destination index. Each time the {transform} queries the source index, it 
+in a _destination index_. Each time the {transform} queries the source index, it 
 creates a _checkpoint_. You can decide whether you want the {transform} to run 
-once (batch {transform}) or continuously ({ctransform}). A batch {transform} is a 
-single operation that has a single checkpoint. {ctransforms-cap} continually 
-increment and process checkpoints as new source data is ingested.
-
-.Example
+once or continuously. A _batch {transform}_ is a single operation that has a
+single checkpoint. _{ctransforms-cap}_ continually increment and process
+checkpoints as new source data is ingested.
 
 Imagine that you run a webshop that sells clothes. Every order creates a 
 document that contains a unique order ID, the name and the category of the 
@@ -72,7 +70,7 @@ indices then index the results into the destination index. Therefore, a
 aggregation that it performs and the indexing process.
 
 For better performance, make sure that your search aggregations and queries are 
-optimized, so they don't process unnecessary data.
+optimized and that your {transform} is processing only necessary data.
 
 NOTE: When you use <<search-aggregations-bucket-datehistogram-aggregation>>, the 
 queries are not considered optimal as they run through a significant amount of 

From de1229cc2bf20f4c56f9069e9cf5ff7e82075ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pawe=C5=82=20Krze=C5=9Bniak?= <pawel@elastic.co>
Date: Mon, 23 Mar 2020 18:28:43 +0100
Subject: [PATCH 220/285] [DOCS] link fix (#53973)

Fix bad link in top_metrics.
---
 .../aggregations/metrics/top-metrics-aggregation.asciidoc       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
index 6dbc8adfd3ff8..e0ab0afde2f3a 100644
--- a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
@@ -50,7 +50,7 @@ faster.
 
 The `sort` field in the metric request functions exactly the same as the `sort` field in the
 <<request-body-search-sort, search>> request except:
-* It can't be used on <<binary,binary>>, <<flattened,flattened>, <<ip,ip>>,
+* It can't be used on <<binary,binary>>, <<flattened,flattened>>, <<ip,ip>>,
 <<keyword,keyword>>, or <<text,text>> fields.
 * It only supports a single sort value so which document wins ties is not specified.
 

From b9c2604f04a1bd78c6f980b8f420883b8a1f12b6 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Mon, 23 Mar 2020 13:34:48 -0400
Subject: [PATCH 221/285] fix semantic merge conflicts

---
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../jackson-databind-2.8.11.6.jar.sha1        |  1 -
 .../metrics/MaxAggregatorTests.java           |  6 +-
 .../metrics/SumAggregatorTests.java           |  6 +-
 .../mapper/WildcardFieldMapperTests.java      | 86 +++++++++----------
 6 files changed, 51 insertions(+), 50 deletions(-)
 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1

diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/modules/ingest-geoip/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/discovery-ec2/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
deleted file mode 100644
index f491259db56bc..0000000000000
--- a/plugins/repository-s3/licenses/jackson-databind-2.8.11.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35753201d0cdb1dbe998ab289bca1180b68d4368
\ No newline at end of file
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
index 1e90a93180b51..c51fc8a3b164d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
@@ -91,7 +91,6 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
-import static java.util.Collections.singletonList;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -163,7 +162,10 @@ protected ScriptService getMockScriptService() {
 
     @Override
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
-        return singletonList(CoreValuesSourceType.NUMERIC);
+        return List.of(
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java
index ceeed769d3082..b23acf01db4d1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java
@@ -69,7 +69,6 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
-import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
@@ -404,7 +403,10 @@ private void testCase(Query query,
 
     @Override
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
-        return singletonList(CoreValuesSourceType.NUMERIC);
+        return List.of(
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN);
     }
 
     @Override
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
index f8c8ddc7f5eef..9d9f7eb4d1b9d 100644
--- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
@@ -68,21 +68,21 @@ public class WildcardFieldMapperTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         Builder builder = new WildcardFieldMapper.Builder(WILDCARD_FIELD_NAME);
-        builder.ignoreAbove(MAX_FIELD_LENGTH);        
+        builder.ignoreAbove(MAX_FIELD_LENGTH);
         wildcardFieldType = builder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));
-        
-        
+
+
         org.elasticsearch.index.mapper.KeywordFieldMapper.Builder kwBuilder = new KeywordFieldMapper.Builder(KEYWORD_FIELD_NAME);
-        keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));        
+        keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0)));
         super.setUp();
     }
-    
+
     public void testIllegalDocValuesArgument() {
         Builder ft = new WildcardFieldMapper.Builder("test");
         MapperParsingException e = expectThrows(MapperParsingException.class,
                 () -> ft.docValues(false));
         assertEquals("The field [test] cannot have doc values = false", e.getMessage());
-    }        
+    }
 
     public void testIllegalIndexedArgument() {
         Builder ft = new WildcardFieldMapper.Builder("test");
@@ -90,19 +90,19 @@ public void testIllegalIndexedArgument() {
                 () -> ft.index(false));
         assertEquals("The field [test] cannot have index = false", e.getMessage());
     }
-    
+
     public void testTooBigKeywordField() throws IOException {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
         iwc.setMergePolicy(newTieredMergePolicy(random()));
         RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-        
+
         // Create a string that is too large and will not be indexed
         String docContent = randomABString(MAX_FIELD_LENGTH + 1);
         Document doc = new Document();
-        ParseContext.Document parseDoc = new ParseContext.Document();                        
+        ParseContext.Document parseDoc = new ParseContext.Document();
         addFields(parseDoc, doc, docContent);
-        indexDoc(parseDoc, doc, iw);        
+        indexDoc(parseDoc, doc, iw);
 
         iw.forceMerge(1);
         DirectoryReader reader = iw.getReader();
@@ -112,24 +112,24 @@ public void testTooBigKeywordField() throws IOException {
         Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("*a*", null, null);
         TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
         assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
-                
+
         reader.close();
-        dir.close();        
+        dir.close();
     }
-    
+
     //Test long query strings don't cause exceptions
     public void testTooBigQueryField() throws IOException {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
         iwc.setMergePolicy(newTieredMergePolicy(random()));
         RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-        
+
         // Create a string that is too large and will not be indexed
         String docContent = randomABString(10);
         Document doc = new Document();
-        ParseContext.Document parseDoc = new ParseContext.Document();                        
+        ParseContext.Document parseDoc = new ParseContext.Document();
         addFields(parseDoc, doc, docContent);
-        indexDoc(parseDoc, doc, iw);        
+        indexDoc(parseDoc, doc, iw);
 
         iw.forceMerge(1);
         DirectoryReader reader = iw.getReader();
@@ -140,12 +140,12 @@ public void testTooBigQueryField() throws IOException {
         Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(queryString, null, null);
         TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
         assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L));
-                
+
         reader.close();
-        dir.close();        
-    }    
-    
-    
+        dir.close();
+    }
+
+
     public void testSearchResultsVersusKeywordField() throws IOException {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER);
@@ -156,22 +156,22 @@ public void testSearchResultsVersusKeywordField() throws IOException {
         HashSet<String> values = new HashSet<>();
         for (int i = 0; i < numDocs; i++) {
             Document doc = new Document();
-            ParseContext.Document parseDoc = new ParseContext.Document();                        
+            ParseContext.Document parseDoc = new ParseContext.Document();
             String docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
-            if (values.contains(docContent) == false) {                
+            if (values.contains(docContent) == false) {
                 addFields(parseDoc, doc, docContent);
                 values.add(docContent);
             }
             // Occasionally add a multi-value field
             if (randomBoolean()) {
                 docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1));
-                if (values.contains(docContent) == false) {                
+                if (values.contains(docContent) == false) {
                     addFields(parseDoc, doc, docContent);
                     values.add(docContent);
-                }                
+                }
             }
             indexDoc(parseDoc, doc, iw);
-            
+
         }
 
         iw.forceMerge(1);
@@ -200,38 +200,38 @@ public void testSearchResultsVersusKeywordField() throws IOException {
             }
             assertThat(expectedDocs.size(), equalTo(0));
         }
-        
-        
+
+
         //Test keyword and wildcard sort operations are also equivalent
         QueryShardContext shardContextMock = createMockShardContext();
-        
+
         FieldSortBuilder wildcardSortBuilder = new FieldSortBuilder(WILDCARD_FIELD_NAME);
-        SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field;        
+        SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field;
         ScoreDoc[] wildcardHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(wildcardSortField)).scoreDocs;
 
         FieldSortBuilder keywordSortBuilder = new FieldSortBuilder(KEYWORD_FIELD_NAME);
-        SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field;        
+        SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field;
         ScoreDoc[] keywordHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(keywordSortField)).scoreDocs;
-        
+
         assertThat(wildcardHits.length, equalTo(keywordHits.length));
         for (int i = 0; i < wildcardHits.length; i++) {
-            assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc));            
+            assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc));
         }
-        
+
         reader.close();
         dir.close();
     }
 
-    
-    
+
+
     protected MappedFieldType provideMappedFieldType(String name) {
         if (name.equals(WILDCARD_FIELD_NAME)) {
-            return wildcardFieldType.fieldType();            
+            return wildcardFieldType.fieldType();
         } else {
             return keywordFieldType.fieldType();
         }
-    }    
-    
+    }
+
     protected final QueryShardContext createMockShardContext() {
         Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index,
@@ -243,15 +243,15 @@ protected final QueryShardContext createMockShardContext() {
         };
         return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup,
                 null, null, null, xContentRegistry(), null, null, null,
-                () -> randomNonNegativeLong(), null, null, () -> true) {
+                () -> randomNonNegativeLong(), null, null, () -> true, null) {
 
             @Override
             public MappedFieldType fieldMapper(String name) {
                 return provideMappedFieldType(name);
             }
         };
-    }       
-    
+    }
+
     private void addFields(ParseContext.Document parseDoc, Document doc, String docContent) throws IOException {
         ArrayList<IndexableField> fields = new ArrayList<>();
         wildcardFieldType.createFields(docContent, parseDoc, fields);
@@ -260,7 +260,7 @@ private void addFields(ParseContext.Document parseDoc, Document doc, String docC
             doc.add(indexableField);
         }
         // Add keyword fields too
-        doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent)));        
+        doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent)));
         doc.add(new StringField(KEYWORD_FIELD_NAME, docContent, Field.Store.YES));
     }
 

From f783670464625a4bb6fa2f3c219d46f748773670 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= <przemyslaw.witek@elastic.co>
Date: Mon, 23 Mar 2020 18:37:54 +0100
Subject: [PATCH 222/285] Verify that the field is aggregatable before
 attempting cardinality aggregation (#53874)

---
 .../xpack/ml/integration/ClassificationIT.java     | 14 ++++++++++++++
 .../extractor/ExtractedFieldsDetectorFactory.java  | 13 +++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index a58418bbd35ef..b1e3fa32e965b 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -56,6 +56,7 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
     private static final String BOOLEAN_FIELD = "boolean-field";
     private static final String NUMERICAL_FIELD = "numerical-field";
     private static final String DISCRETE_NUMERICAL_FIELD = "discrete-numerical-field";
+    private static final String TEXT_FIELD = "text-field";
     private static final String KEYWORD_FIELD = "keyword-field";
     private static final String NESTED_FIELD = "outer-field.inner-field";
     private static final String ALIAS_TO_KEYWORD_FIELD = "alias-to-keyword-field";
@@ -260,6 +261,14 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableI
         assertThat(e.getMessage(), startsWith("invalid types [double] for required field [numerical-field];"));
     }
 
+    public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsText() throws Exception {
+        ElasticsearchStatusException e = expectThrows(
+            ElasticsearchStatusException.class,
+            () -> testWithOnlyTrainingRowsAndTrainingPercentIsFifty(
+                "classification_training_percent_is_50_text", TEXT_FIELD, KEYWORD_FIELD_VALUES, null));
+        assertThat(e.getMessage(), startsWith("field [text-field] of type [text] is non-aggregatable"));
+    }
+
     public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsBoolean() throws Exception {
         testWithOnlyTrainingRowsAndTrainingPercentIsFifty(
             "classification_training_percent_is_50_boolean", BOOLEAN_FIELD, BOOLEAN_FIELD_VALUES, "boolean");
@@ -517,6 +526,7 @@ private static void createIndex(String index) {
                 BOOLEAN_FIELD, "type=boolean",
                 NUMERICAL_FIELD, "type=double",
                 DISCRETE_NUMERICAL_FIELD, "type=integer",
+                TEXT_FIELD, "type=text",
                 KEYWORD_FIELD, "type=keyword",
                 NESTED_FIELD, "type=keyword",
                 ALIAS_TO_KEYWORD_FIELD, "type=alias,path=" + KEYWORD_FIELD,
@@ -532,6 +542,7 @@ private static void indexData(String sourceIndex, int numTrainingRows, int numNo
                 BOOLEAN_FIELD, BOOLEAN_FIELD_VALUES.get(i % BOOLEAN_FIELD_VALUES.size()),
                 NUMERICAL_FIELD, NUMERICAL_FIELD_VALUES.get(i % NUMERICAL_FIELD_VALUES.size()),
                 DISCRETE_NUMERICAL_FIELD, DISCRETE_NUMERICAL_FIELD_VALUES.get(i % DISCRETE_NUMERICAL_FIELD_VALUES.size()),
+                TEXT_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()),
                 KEYWORD_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()),
                 NESTED_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()));
             IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray());
@@ -549,6 +560,9 @@ private static void indexData(String sourceIndex, int numTrainingRows, int numNo
                 source.addAll(
                     List.of(DISCRETE_NUMERICAL_FIELD, DISCRETE_NUMERICAL_FIELD_VALUES.get(i % DISCRETE_NUMERICAL_FIELD_VALUES.size())));
             }
+            if (TEXT_FIELD.equals(dependentVariable) == false) {
+                source.addAll(List.of(TEXT_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size())));
+            }
             if (KEYWORD_FIELD.equals(dependentVariable) == false) {
                 source.addAll(List.of(KEYWORD_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size())));
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java
index b73f64e35283e..918999014d06c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
@@ -78,7 +79,7 @@ private void create(String[] index, DataFrameAnalyticsConfig config, ActionListe
         ActionListener<FieldCapabilitiesResponse> fieldCapabilitiesHandler = ActionListener.wrap(
             fieldCapabilitiesResponse -> {
                 fieldCapsResponseHolder.set(fieldCapabilitiesResponse);
-                getCardinalitiesForFieldsWithConstraints(index, config, fieldCardinalitiesHandler);
+                getCardinalitiesForFieldsWithConstraints(index, config, fieldCapabilitiesResponse, fieldCardinalitiesHandler);
             },
             listener::onFailure
         );
@@ -96,7 +97,9 @@ private void create(String[] index, DataFrameAnalyticsConfig config, ActionListe
         getDocValueFieldsLimit(index, docValueFieldsLimitListener);
     }
 
-    private void getCardinalitiesForFieldsWithConstraints(String[] index, DataFrameAnalyticsConfig config,
+    private void getCardinalitiesForFieldsWithConstraints(String[] index,
+                                                          DataFrameAnalyticsConfig config,
+                                                          FieldCapabilitiesResponse fieldCapabilitiesResponse,
                                                           ActionListener<Map<String, Long>> listener) {
         List<FieldCardinalityConstraint> fieldCardinalityConstraints = config.getAnalysis().getFieldCardinalityConstraints();
         if (fieldCardinalityConstraints.isEmpty()) {
@@ -111,6 +114,12 @@ private void getCardinalitiesForFieldsWithConstraints(String[] index, DataFrameA
 
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).query(config.getSource().getParsedQuery());
         for (FieldCardinalityConstraint constraint : fieldCardinalityConstraints) {
+            for (FieldCapabilities fieldCaps : fieldCapabilitiesResponse.getField(constraint.getField()).values()) {
+                if (fieldCaps.isAggregatable() == false) {
+                    throw ExceptionsHelper.badRequestException("field [{}] of type [{}] is non-aggregatable",
+                        fieldCaps.getName(), fieldCaps.getType());
+                }
+            }
             searchSourceBuilder.aggregation(
                 AggregationBuilders.cardinality(constraint.getField())
                     .field(constraint.getField())

From 20d861cefc91ddfb313376f1234cfd6a86861439 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova <mayya.sharipova@elastic.co>
Date: Mon, 23 Mar 2020 13:53:07 -0400
Subject: [PATCH 223/285] Add logging and enable testQueryRewrite (#53809)

This adds reenables IndicesRequestCacheIT.testQueryRewrite
and enables logging for it.

Relates to #32827
---
 .../java/org/elasticsearch/indices/IndicesRequestCache.java | 6 ++++--
 .../org/elasticsearch/indices/IndicesRequestCacheIT.java    | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 49c2d070c0310..898e451aa3bc6 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -124,7 +124,8 @@ BytesReference getOrCompute(CacheEntity cacheEntity, Supplier<BytesReference> lo
         if (cacheLoader.isLoaded()) {
             key.entity.onMiss();
             if (logger.isTraceEnabled()) {
-                logger.trace("Cache miss for reader version [{}] and request:\n {}", reader.getVersion(), cacheKeyRenderer.get());
+                logger.trace("Cache miss for reader version [{}], max_doc[{}] and request:\n {}",
+                    reader.getVersion(), reader.maxDoc(), cacheKeyRenderer.get());
             }
             // see if its the first time we see this reader, and make sure to register a cleanup key
             CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getReaderCacheHelper().getKey());
@@ -137,7 +138,8 @@ BytesReference getOrCompute(CacheEntity cacheEntity, Supplier<BytesReference> lo
         } else {
             key.entity.onHit();
             if (logger.isTraceEnabled()) {
-                logger.trace("Cache hit for reader version [{}] and request:\n {}", reader.getVersion(), cacheKeyRenderer.get());
+                logger.trace("Cache hit for reader version [{}], max_doc[{}] and request:\n {}",
+                    reader.getVersion(), reader.maxDoc(), cacheKeyRenderer.get());
             }
         }
         return value;
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index d06a7e1cecfa4..4274644217400 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 
 import java.time.ZoneId;
 import java.time.ZoneOffset;
@@ -97,7 +98,9 @@ public void testCacheAggs() throws Exception {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32827")
+    @TestIssueLogging(
+        value = "org.elasticsearch.indices.IndicesRequestCache:TRACE",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/32827")
     public void testQueryRewrite() throws Exception {
         Client client = client();
         assertAcked(client.admin().indices().prepareCreate("index").setMapping("s", "type=date")

From 04bd154037172d5c4a09250e1c60beb07d5b5c87 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 23 Mar 2020 19:06:32 +0100
Subject: [PATCH 224/285] Add heuristics to compute pre_filter_shard_size when
 unspecified (#53873)

This commit changes the pre_filter_shard_size default from 128 to unspecified.
This allows to apply heuristics based on the request and the target indices when deciding
whether the can match phase should run or not. When unspecified, this pr runs the can match phase
automatically if one of these conditions is met:
  * The request targets more than 128 shards.
  * The request contains read-only indices.
  * The primary sort of the query targets an indexed field.
Users can opt-out from this behavior by setting the `pre_filter_shard_size` to a static value.

Closes #39835
---
 .../client/RequestConverters.java             |   4 +-
 .../client/RequestConvertersTests.java        |   4 +-
 docs/reference/frozen-indices.asciidoc        |  13 +-
 docs/reference/search/multi-search.asciidoc   |  37 +++--
 docs/reference/search/search.asciidoc         | 146 +++++++++---------
 .../org/elasticsearch/search/CCSDuelIT.java   |   4 +-
 .../resources/rest-api-spec/api/msearch.json  |   3 +-
 .../resources/rest-api-spec/api/search.json   |   3 +-
 .../action/search/SearchRequest.java          |  42 +++--
 .../action/search/SearchRequestBuilder.java   |  11 +-
 .../action/search/TransportSearchAction.java  |  32 +++-
 .../action/search/RestMultiSearchAction.java  |  10 +-
 .../rest/action/search/RestSearchAction.java  |   4 +-
 .../search/TransportSearchActionTests.java    | 102 +++++++++++-
 14 files changed, 288 insertions(+), 127 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 0bb0eca83a389..d0cd3ea5a0091 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -410,7 +410,9 @@ static void addSearchRequestParams(Params params, SearchRequest searchRequest) {
         params.withIndicesOptions(searchRequest.indicesOptions());
         params.withSearchType(searchRequest.searchType().name().toLowerCase(Locale.ROOT));
         params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips()));
-        params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        if (searchRequest.getPreFilterShardSize() != null) {
+            params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        }
         params.withMaxConcurrentShardRequests(searchRequest.getMaxConcurrentShardRequests());
         if (searchRequest.requestCache() != null) {
             params.withRequestCache(searchRequest.requestCache());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index a7291b3dbbcf3..346a6310c5d75 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -1875,7 +1875,9 @@ private static void setRandomSearchParams(SearchRequest searchRequest,
         if (randomBoolean()) {
             searchRequest.setPreFilterShardSize(randomIntBetween(2, Integer.MAX_VALUE));
         }
-        expectedParams.put("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        if (searchRequest.getPreFilterShardSize() != null) {
+            expectedParams.put("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        }
     }
 
     public static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<IndicesOptions> getter,
diff --git a/docs/reference/frozen-indices.asciidoc b/docs/reference/frozen-indices.asciidoc
index b6cafad30f5dc..5a2ef125a8cc3 100644
--- a/docs/reference/frozen-indices.asciidoc
+++ b/docs/reference/frozen-indices.asciidoc
@@ -74,8 +74,8 @@ POST /twitter/_forcemerge?max_num_segments=1
 == Searching a frozen index
 
 Frozen indices are throttled in order to limit memory consumptions per node. The number of concurrently loaded frozen indices per node is
-limited by the number of threads in the <<search-throttled,search_throttled>> threadpool,  which is `1` by default. 
-Search requests will not be executed against frozen indices by default, even if a frozen index is named explicitly. This is 
+limited by the number of threads in the <<search-throttled,search_throttled>> threadpool,  which is `1` by default.
+Search requests will not be executed against frozen indices by default, even if a frozen index is named explicitly. This is
 to prevent accidental slowdowns by targeting a frozen index by mistake. To include frozen indices a search request must be executed with
 the query parameter `ignore_throttled=false`.
 
@@ -85,15 +85,6 @@ GET /twitter/_search?q=user:kimchy&ignore_throttled=false
 --------------------------------------------------
 // TEST[setup:twitter]
 
-[IMPORTANT]
-================================
-While frozen indices are slow to search, they can be pre-filtered efficiently. The request parameter `pre_filter_shard_size` specifies
-a threshold that, when exceeded, will enforce a round-trip to pre-filter search shards that cannot possibly match.
-This filter phase can limit the number of shards significantly. For instance, if a date range filter is applied, then all indices (frozen or unfrozen) that do not contain documents within the date range can be skipped efficiently.
-The default value for `pre_filter_shard_size` is `128` but it's recommended to set it to `1` when searching frozen indices. There is no
-significant overhead associated with this pre-filter phase.
-================================
-
 [role="xpack"]
 [testenv="basic"]
 [[monitoring_frozen_indices]]
diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc
index f24d963f41ac3..d3751af655df4 100644
--- a/docs/reference/search/multi-search.asciidoc
+++ b/docs/reference/search/multi-search.asciidoc
@@ -23,7 +23,7 @@ GET twitter/_msearch
 ==== {api-description-title}
 
 The multi search API executes several searches from a single API request.
-The format of the request is similar to the bulk API format and makes use 
+The format of the request is similar to the bulk API format and makes use
 of the newline delimited JSON (NDJSON) format.
 
 The structure is as follows:
@@ -85,7 +85,7 @@ Maximum number of concurrent searches the multi search API can execute.
 --
 (Optional, integer)
 Maximum number of concurrent shard requests that each sub-search request
-executes per node.  Defaults to `5`. 
+executes per node.  Defaults to `5`.
 
 You can use this parameter to prevent a request from overloading a cluster. For
 example, a default request hits all indices in a cluster. This could cause shard
@@ -103,8 +103,13 @@ Defines a threshold that enforces a pre-filter roundtrip to prefilter search
 shards based on query rewriting if the number of shards the search request
 expands to exceeds the threshold. This filter roundtrip can limit the number of
 shards significantly if for instance a shard can not match any documents based
-on it's rewrite method i.e., if date filters are mandatory to match but the
-shard bounds and the query are disjoint. Defaults to `128`.
+on its rewrite method i.e., if date filters are mandatory to match but the
+shard bounds and the query are disjoint.
+When unspecified, the pre-filter phase is executed if any of these
+conditions is met:
+  - The request targets more than `128` shards.
+  - The request targets one or more read-only index.
+  - The primary sort of the query targets an indexed field.
 
 `rest_total_hits_as_int`::
 (Optional, boolean)
@@ -121,7 +126,7 @@ to a specific shard.
 --
 (Optional, string)
 Indicates whether global term and document frequencies should be used when
-scoring returned documents. 
+scoring returned documents.
 
 Options are:
 
@@ -134,7 +139,7 @@ This is usually faster but less accurate.
 Documents are scored using global term and document frequencies across all
 shards. This is usually slower but more accurate.
 --
-  
+
 `typed_keys`::
 (Optional, boolean)
 Specifies whether aggregation and suggester names should be prefixed by their
@@ -196,7 +201,7 @@ to a specific shard.
 --
 (Optional, string)
 Indicates whether global term and document frequencies should be used when
-scoring returned documents. 
+scoring returned documents.
 
 Options are:
 
@@ -234,18 +239,18 @@ Number of hits to return. Defaults to `10`.
 ==== {api-response-body-title}
 
 `responses`::
-  (array) Includes the search response and status code for each search request 
-  matching its order in the original multi search request. If there was a  
-  complete failure for a specific search request, an object with `error` message 
-  and corresponding status code will be returned in place of the actual search 
+  (array) Includes the search response and status code for each search request
+  matching its order in the original multi search request. If there was a
+  complete failure for a specific search request, an object with `error` message
+  and corresponding status code will be returned in place of the actual search
   response.
 
 
 [[search-multi-search-api-example]]
 ==== {api-examples-title}
 
-The header part includes which index / indices to search on, the `search_type`, 
-`preference`, and `routing`. The body includes the typical search body request 
+The header part includes which index / indices to search on, the `search_type`,
+`preference`, and `routing`. The body includes the typical search body request
 (including the `query`, `aggregations`, `from`, `size`, and so on).
 
 [source,js]
@@ -308,7 +313,7 @@ See <<url-access-control>>
 ==== Template support
 
 Much like described in <<search-template>> for the _search resource, _msearch
-also provides support for templates. Submit them like follows for inline 
+also provides support for templates. Submit them like follows for inline
 templates:
 
 [source,console]
@@ -377,6 +382,6 @@ GET _msearch/template
 [[multi-search-partial-responses]]
 ==== Partial responses
 
-To ensure fast responses, the multi search API will respond with partial results 
-if one or more shards fail. See <<shard-failures, Shard failures>> for more 
+To ensure fast responses, the multi search API will respond with partial results
+if one or more shards fail. See <<shard-failures, Shard failures>> for more
 information.
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 1e8e874df2a9b..255cb0048be3c 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -25,7 +25,7 @@ GET /twitter/_search?q=tag:wow
 [[search-search-api-desc]]
 ==== {api-description-title}
 
-Allows you to execute a search query and get back search hits that match the 
+Allows you to execute a search query and get back search hits that match the
 query. The query can either be provided using a simple
 <<search-uri-request,query string as a parameter>>, or using a
 <<search-request-body,request body>>.
@@ -33,8 +33,8 @@ query. The query can either be provided using a simple
 [[search-partial-responses]]
 ===== Partial responses
 
-To ensure fast responses, the search API will respond with partial results if 
-one or more shards fail. See <<shard-failures, Shard failures>> for more 
+To ensure fast responses, the search API will respond with partial results if
+one or more shards fail. See <<shard-failures, Shard failures>> for more
 information.
 
 [[search-search-api-path-params]]
@@ -51,163 +51,167 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=allow-no-indices]
 Defaults to `true`.
 
 `allow_partial_search_results`::
-  (Optional, boolean) Indicates if an error should be returned if there is a 
+  (Optional, boolean) Indicates if an error should be returned if there is a
   partial search failure or timeout. Defaults to `true`.
 
 `analyzer`::
   (Optional, string) Defines the analyzer to use for the query string.
-  
+
 `analyze_wildcard`::
-  (Optional, boolean) If `true`, wildcard and prefix queries will also be 
+  (Optional, boolean) If `true`, wildcard and prefix queries will also be
   analyzed. Defaults to `false`.
-  
+
 `batched_reduce_size`::
-  (Optional, integer) The number of shard results that should be reduced at once 
-  on the coordinating node. This value should be used as a protection mechanism 
-  to reduce the memory overhead per search request if the potential number of 
+  (Optional, integer) The number of shard results that should be reduced at once
+  on the coordinating node. This value should be used as a protection mechanism
+  to reduce the memory overhead per search request if the potential number of
   shards in the request can be large. Defaults to `512`.
-  
+
 `ccs_minimize_roundtrips`::
-  (Optional, boolean) Indicates whether network round-trips should be minimized 
+  (Optional, boolean) Indicates whether network round-trips should be minimized
   as part of cross-cluster search requests execution. Defaults to `true`.
-  
+
 `default_operator`::
-  (Optional, string) The default operator for query string query (AND or OR). 
+  (Optional, string) The default operator for query string query (AND or OR).
   Defaults to `OR`.
-  
+
 `df`::
-  (Optional, string) Defines the field to use as default where no field prefix 
+  (Optional, string) Defines the field to use as default where no field prefix
   is given in the query string.
-  
+
 `docvalue_fields`::
-  (Optional, string) A comma-separated list of fields to return as the docvalue 
+  (Optional, string) A comma-separated list of fields to return as the docvalue
   representation of a field for each hit.
-  
+
 include::{docdir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 +
 Defaults to `open`.
-  
+
 `explain`::
-  (Optional, boolean) If `true`, returns detailed information about score 
+  (Optional, boolean) If `true`, returns detailed information about score
   computation as part of a hit. Defaults to `false`.
-  
+
 `from`::
   (Optional, integer) Defines the starting offset. Defaults to `0`.
 
 `ignore_throttled`::
-  (Optional, boolean) If `true`, concrete, expanded or aliased indices will be 
+  (Optional, boolean) If `true`, concrete, expanded or aliased indices will be
   ignored when throttled. Defaults to `false`.
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
-  
+
 `lenient`::
-  (Optional, boolean) If `true`, format-based query failures (such as 
+  (Optional, boolean) If `true`, format-based query failures (such as
   providing text to a numeric field) will be ignored. Defaults to `false`.
-  
+
 `max_concurrent_shard_requests`::
-  (Optional, integer) Defines the number of concurrent shard requests per node 
-  this search executes concurrently. This value should be used to limit the 
-  impact of the search on the cluster in order to limit the number of concurrent 
+  (Optional, integer) Defines the number of concurrent shard requests per node
+  this search executes concurrently. This value should be used to limit the
+  impact of the search on the cluster in order to limit the number of concurrent
   shard requests. Defaults to `5`.
-  
+
 `pre_filter_shard_size`::
-  (Optional, integer) Defines a threshold that enforces a pre-filter roundtrip 
-  to prefilter search shards based on query rewriting if the number of shards 
-  the search request expands to exceeds the threshold. This filter roundtrip can 
-  limit the number of shards significantly if for instance a shard can not match 
-  any documents based on it's rewrite method ie. if date filters are mandatory 
-  to match but the shard bounds and the query are disjoint. Defaults to `128`.
+  (Optional, integer) Defines a threshold that enforces a pre-filter roundtrip
+  to prefilter search shards based on query rewriting if the number of shards
+  the search request expands to exceeds the threshold. This filter roundtrip can
+  limit the number of shards significantly if for instance a shard can not match
+  any documents based on its rewrite method ie. if date filters are mandatory
+  to match but the shard bounds and the query are disjoint.
+  When unspecified, the pre-filter phase is executed if any of these conditions is met:
+    - The request targets more than `128` shards.
+    - The request targets one or more read-only index.
+    - The primary sort of the query targets an indexed field.
 
 `preference`::
-  (Optional, string) Specifies the node or shard the operation should be 
+  (Optional, string) Specifies the node or shard the operation should be
   performed on. Random by default.
-  
+
 `q`::
   (Optional, string) Query in the Lucene query string syntax.
 
 `request_cache`::
-  (Optional, boolean) If `true`, request cache will be used for this request. 
+  (Optional, boolean) If `true`, request cache will be used for this request.
   Defaults to index level settings.
-  
+
 `rest_total_hits_as_int`::
-  (Optional, boolean) Indicates whether hits.total should be rendered as an 
+  (Optional, boolean) Indicates whether hits.total should be rendered as an
   integer or an object in the rest search response. Defaults to `false`.
 
 `routing`::
-  (Optional, <<time-units, time units>>) Specifies how long a consistent view of 
+  (Optional, <<time-units, time units>>) Specifies how long a consistent view of
   the index should be maintained for scrolled search.
-  
+
 `search_type`::
-  (Optional, string) Defines the type of the search operation. Available 
+  (Optional, string) Defines the type of the search operation. Available
   options:
   * `query_then_fetch`
   * `dfs_query_then_fetch`
 
 `seq_no_primary_term`::
-  (Optional, boolean) If `true`, returns sequence number and primary term of the 
+  (Optional, boolean) If `true`, returns sequence number and primary term of the
   last modification of each hit.
 
 `size`::
   (Optional, integer) Defines the number of hits to return. Defaults to `10`.
-  
+
 `sort`::
   (Optional, string) A comma-separated list of <field>:<direction> pairs.
-  
+
 `_source`::
-  (Optional, string) True or false to return the `_source` field or not, or a 
+  (Optional, string) True or false to return the `_source` field or not, or a
   list of fields to return.
-  
+
 `_source_excludes`::
-  (Optional, string) A list of fields to exclude from the returned `_source` 
+  (Optional, string) A list of fields to exclude from the returned `_source`
   field.
-  
+
 `_source_includes`::
-  (Optional, string) A list of fields to extract and return from the `_source` 
+  (Optional, string) A list of fields to extract and return from the `_source`
   field.
-  
+
 `stats`::
-  (Optional, string) Specific `tag` of the request for logging and statistical 
+  (Optional, string) Specific `tag` of the request for logging and statistical
   purposes.
 
 `stored_fields`::
-  (Optional, string) A comma-separated list of stored fields to return as part 
+  (Optional, string) A comma-separated list of stored fields to return as part
   of a hit.
-  
+
 `suggest_field`::
   (Optional, string) Specifies which field to use for suggestions.
-  
+
 `suggest_mode`::
-  (Optional, string) Specifies suggest mode. Defaults to `missing`. Available 
+  (Optional, string) Specifies suggest mode. Defaults to `missing`. Available
   options:
   * `always`
   * `missing`
   * `popular`
-  
+
 `suggest_size`::
   (Optional, integer) Defines how many suggestions to return in response.
-  
+
 `suggest_text`::
-  (Optional, string) The source text for which the suggestions should be 
+  (Optional, string) The source text for which the suggestions should be
   returned.
-  
+
 `terminate_after`::
-  (Optional, integer) The maximum number of documents to collect for each shard, 
+  (Optional, integer) The maximum number of documents to collect for each shard,
   upon reaching which the query execution will terminate early.
-  
+
 include::{docdir}/rest-api/common-parms.asciidoc[tag=timeout]
 
 `track_scores`::
-  (Optional, boolean) If `true`, then calculates and returns scores even if they 
+  (Optional, boolean) If `true`, then calculates and returns scores even if they
   are not used for sorting.
-  
+
 `track_total_hits`::
-  (Optional, boolean) Indicates if the number of documents that match the query 
+  (Optional, boolean) Indicates if the number of documents that match the query
   should be tracked.
-  
+
 `typed_keys`::
-  (Optional, boolean) Specifies whether aggregation and suggester names should 
+  (Optional, boolean) Specifies whether aggregation and suggester names should
   be prefixed by their respective types in the response.
-  
+
 `version`::
   (Optional, boolean) If `true`, returns document version as part of a hit.
 
@@ -216,7 +220,7 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=timeout]
 ==== {api-request-body-title}
 
 `query`::
-  (Optional, <<query-dsl,query object>>) Defines the search definition using the 
+  (Optional, <<query-dsl,query object>>) Defines the search definition using the
   <<query-dsl,Query DSL>>.
 
 
diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
index a4e76fc7ec436..d2f876cd17848 100644
--- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
+++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
@@ -124,7 +124,7 @@ public class CCSDuelIT extends ESRestTestCase {
 
     private static final String INDEX_NAME = "ccs_duel_index";
     private static final String REMOTE_INDEX_NAME = "my_remote_cluster:" + INDEX_NAME;
-    private static final String[] TAGS = new String[]{"java", "xml", "sql", "html", "php", "ruby", "python", "perl"};
+    private static final String[] TAGS = new String[] {"java", "xml", "sql", "html", "php", "ruby", "python", "perl"};
 
     private static RestHighLevelClient restHighLevelClient;
 
@@ -435,6 +435,8 @@ public void testSortByField() throws Exception  {
     public void testSortByFieldOneClusterHasNoResults() throws Exception {
         assumeMultiClusterSetup();
         SearchRequest searchRequest = initSearchRequest();
+        // set to a value greater than the number of shards to avoid differences due to the skipping of shards
+        searchRequest.setPreFilterShardSize(128);
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
         boolean onlyRemote = randomBoolean();
         sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME));
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
index 8dd9da844520a..968ccfd8e718f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
@@ -50,8 +50,7 @@
       },
       "pre_filter_shard_size":{
         "type":"number",
-        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
-        "default":128
+        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint."
       },
       "max_concurrent_shard_requests":{
         "type":"number",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 1e0c232efa055..ac321acf8907b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -219,8 +219,7 @@
       },
       "pre_filter_shard_size":{
         "type":"number",
-        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
-        "default":128
+        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint."
       },
       "rest_total_hits_as_int":{
         "type":"boolean",
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 761a50bfa4168..a0be11eb0c2af 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -89,7 +89,7 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
 
     private int maxConcurrentShardRequests = 0;
 
-    private int preFilterShardSize = DEFAULT_PRE_FILTER_SHARD_SIZE;
+    private Integer preFilterShardSize;
 
     private boolean ccsMinimizeRoundtrips = true;
 
@@ -201,7 +201,11 @@ public SearchRequest(StreamInput in) throws IOException {
         requestCache = in.readOptionalBoolean();
         batchedReduceSize = in.readVInt();
         maxConcurrentShardRequests = in.readVInt();
-        preFilterShardSize = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            preFilterShardSize = in.readOptionalVInt();
+        } else {
+            preFilterShardSize = in.readVInt();
+        }
         allowPartialSearchResults = in.readOptionalBoolean();
         localClusterAlias = in.readOptionalString();
         if (localClusterAlias != null) {
@@ -231,7 +235,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalBoolean(requestCache);
         out.writeVInt(batchedReduceSize);
         out.writeVInt(maxConcurrentShardRequests);
-        out.writeVInt(preFilterShardSize);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalVInt(preFilterShardSize);
+        } else {
+            out.writeVInt(preFilterShardSize == null ? DEFAULT_BATCHED_REDUCE_SIZE : preFilterShardSize);
+        }
         out.writeOptionalBoolean(allowPartialSearchResults);
         out.writeOptionalString(localClusterAlias);
         if (localClusterAlias != null) {
@@ -531,8 +539,15 @@ public void setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
     /**
      * Sets a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
      * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
-     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
-     * bounds and the query are disjoint. The default is {@code 128}
+     * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint.
+     *
+     * When unspecified, the pre-filter phase is executed if any of these conditions is met:
+     * <ul>
+     * <li>The request targets more than 128 shards</li>
+     * <li>The request targets one or more read-only index</li>
+     * <li>The primary sort of the query targets an indexed field</li>
+     * </ul>
      */
     public void setPreFilterShardSize(int preFilterShardSize) {
         if (preFilterShardSize < 1) {
@@ -543,11 +558,20 @@ public void setPreFilterShardSize(int preFilterShardSize) {
 
     /**
      * Returns a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
-     * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
-     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
-     * bounds and the query are disjoint. The default is {@code 128}
+     * the search request expands to exceeds the threshold, or <code>null</code> if the threshold is unspecified.
+     * This filter roundtrip can limit the number of shards significantly if for
+     * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint.
+     *
+     * When unspecified, the pre-filter phase is executed if any of these conditions is met:
+     * <ul>
+     * <li>The request targets more than 128 shards</li>
+     * <li>The request targets one or more read-only index</li>
+     * <li>The primary sort of the query targets an indexed field</li>
+     * </ul>
      */
-    public int getPreFilterShardSize() {
+    @Nullable
+    public Integer getPreFilterShardSize() {
         return preFilterShardSize;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index b8b791360d30c..edb2f093b3ee0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -558,8 +558,15 @@ public SearchRequestBuilder setMaxConcurrentShardRequests(int maxConcurrentShard
     /**
      * Sets a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
      * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
-     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
-     * bounds and the query are disjoint. The default is {@code 128}
+     * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint.
+     *
+     * When unspecified, the pre-filter phase is executed if any of these conditions is met:
+     * <ul>
+     * <li>The request targets more than 128 shards</li>
+     * <li>The request targets one or more read-only index</li>
+     * <li>The primary sort of the query targets an indexed field</li>
+     * </ul>
      */
     public SearchRequestBuilder setPreFilterShardSize(int preFilterShardSize) {
         this.request.setPreFilterShardSize(preFilterShardSize);
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 61bbaa0a5097c..26d7324ebfbf6 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -55,7 +56,6 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
-import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterAware;
@@ -83,6 +83,7 @@
 
 import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
+import static org.elasticsearch.search.sort.FieldSortBuilder.hasPrimaryFieldSort;
 
 public class TransportSearchAction extends HandledTransportAction<SearchRequest, SearchResponse> {
 
@@ -512,7 +513,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         final DiscoveryNodes nodes = clusterState.nodes();
         BiFunction<String, String, Transport.Connection> connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(),
             nodes::get, remoteConnections, searchTransportService::getConnection);
-        boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators);
+        boolean preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, indices, shardIterators.size());
         searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
             Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start();
     }
@@ -539,12 +540,31 @@ static BiFunction<String, String, Transport.Connection> buildConnectionLookup(St
         };
     }
 
-    private static boolean shouldPreFilterSearchShards(SearchRequest searchRequest,
-                                                       GroupShardsIterator<SearchShardIterator> shardIterators) {
+    static boolean shouldPreFilterSearchShards(ClusterState clusterState,
+                                               SearchRequest searchRequest,
+                                               Index[] indices,
+                                               int numShards) {
         SearchSourceBuilder source = searchRequest.source();
+        Integer preFilterShardSize = searchRequest.getPreFilterShardSize();
+        if (preFilterShardSize == null
+                && (hasReadOnlyIndices(indices, clusterState) || hasPrimaryFieldSort(source))) {
+            preFilterShardSize = 1;
+        } else if (preFilterShardSize == null) {
+            preFilterShardSize = SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE;
+        }
         return searchRequest.searchType() == QUERY_THEN_FETCH // we can't do this for DFS it needs to fan out to all shards all the time
-                    && (SearchService.canRewriteToMatchNone(source) || FieldSortBuilder.hasPrimaryFieldSort(source))
-                    && searchRequest.getPreFilterShardSize() < shardIterators.size();
+                    && (SearchService.canRewriteToMatchNone(source) || hasPrimaryFieldSort(source))
+                    && preFilterShardSize < numShards;
+    }
+
+    private static boolean hasReadOnlyIndices(Index[] indices, ClusterState clusterState) {
+        for (Index index : indices) {
+            ClusterBlockException writeBlock = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, index.getName());
+            if (writeBlock != null) {
+                return true;
+            }
+        }
+        return false;
     }
 
     static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator,
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 4cef66b5da4b1..ee6cba31d26f1 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -94,7 +94,10 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
         }
 
-        int preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
+        Integer preFilterShardSize = null;
+        if (restRequest.hasParam("pre_filter_shard_size")) {
+            preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
+        }
 
         final Integer maxConcurrentShardRequests;
         if (restRequest.hasParam("max_concurrent_shard_requests")) {
@@ -111,10 +114,11 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a
             multiRequest.add(searchRequest);
         });
         List<SearchRequest> requests = multiRequest.requests();
-        preFilterShardSize = Math.max(1, preFilterShardSize / (requests.size()+1));
         for (SearchRequest request : requests) {
             // preserve if it's set on the request
-            request.setPreFilterShardSize(Math.min(preFilterShardSize, request.getPreFilterShardSize()));
+            if (preFilterShardSize != null && request.getPreFilterShardSize() == null) {
+                request.setPreFilterShardSize(preFilterShardSize);
+            }
             if (maxConcurrentShardRequests != null) {
                 request.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
             }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index b95aaf4b86018..37f674d70e57e 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -128,7 +128,9 @@ public static void parseSearchRequest(SearchRequest searchRequest, RestRequest r
 
         final int batchedReduceSize = request.paramAsInt("batched_reduce_size", searchRequest.getBatchedReduceSize());
         searchRequest.setBatchedReduceSize(batchedReduceSize);
-        searchRequest.setPreFilterShardSize(request.paramAsInt("pre_filter_shard_size", searchRequest.getPreFilterShardSize()));
+        if (request.hasParam("pre_filter_shard_size")) {
+            searchRequest.setPreFilterShardSize(request.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE));
+        }
 
         if (request.hasParam("max_concurrent_shard_requests")) {
             // only set if we have the parameter since we auto adjust the max concurrency on the coordinator
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 32bca4d0dc18c..895c75c3f98e4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -29,6 +29,10 @@
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.GroupShardsIteratorTests;
@@ -44,6 +48,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
@@ -58,6 +63,7 @@
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -535,7 +541,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 AtomicReference<SearchResponse> response = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, 
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
                         aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
@@ -838,6 +844,100 @@ public void testShouldMinimizeRoundtrips() throws Exception {
         }
     }
 
+    public void testShouldPreFilterSearchShards() {
+        int numIndices = randomIntBetween(1, 10);
+        Index[] indices = new Index[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            String indexName = randomAlphaOfLengthBetween(5, 10);
+            indices[i] = new Index(indexName, indexName + "-uuid");
+        }
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).build();
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(1, 127)));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(1, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp")));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(1, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp")))
+                .scroll("5m");
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(1, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+    }
+
+    public void testShouldPreFilterSearchShardsWithReadOnly() {
+        int numIndices = randomIntBetween(1, 10);
+        int numReadOnly = randomIntBetween(1, numIndices);
+        Index[] indices = new Index[numIndices];
+        ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder();
+        for (int i = 0; i < numIndices; i++) {
+            String indexName = randomAlphaOfLengthBetween(5, 10);
+            indices[i] = new Index(indexName, indexName + "-uuid");
+            if (--numReadOnly >= 0) {
+                if (randomBoolean()) {
+                    blocksBuilder.addIndexBlock(indexName, IndexMetaData.INDEX_WRITE_BLOCK);
+                } else {
+                    blocksBuilder.addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_BLOCK);
+                }
+            }
+        }
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).blocks(blocksBuilder).build();
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(1, 127)));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(1, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            searchRequest.scroll("5s");
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+    }
+
     private InternalAggregation.ReduceContextBuilder aggReduceContextBuilder() {
         return new InternalAggregation.ReduceContextBuilder() {
             @Override

From 8d4ff290cc6361cf23ae2b825c46bb83976e50f0 Mon Sep 17 00:00:00 2001
From: Namgyu Kim <kng0828@gmail.com>
Date: Tue, 24 Mar 2020 03:28:49 +0900
Subject: [PATCH 225/285] Add nori_number token filter in analysis-nori
 (#53583)

This change adds the `nori_number` token filter.
It also adds a `discard_punctuation` option in nori_tokenizer that should be used in conjunction with the new filter.
---
 docs/plugins/analysis-nori.asciidoc           | 109 ++++++++++++++++++
 .../analysis/NoriNumberFilterFactory.java     |  38 ++++++
 .../index/analysis/NoriTokenizerFactory.java  |   5 +-
 .../analysis/nori/AnalysisNoriPlugin.java     |   2 +
 .../analysis/AnalysisNoriFactoryTests.java    |   1 +
 .../index/analysis/NoriAnalysisTests.java     |  51 ++++++++
 .../test/analysis_nori/10_basic.yml           |  17 +++
 7 files changed, 222 insertions(+), 1 deletion(-)
 create mode 100644 plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java

diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc
index 7cc04c9c3de75..1e5f998a72da4 100644
--- a/docs/plugins/analysis-nori.asciidoc
+++ b/docs/plugins/analysis-nori.asciidoc
@@ -54,6 +54,10 @@ It can be set to:
     가곡역 => 가곡역, 가곡, 역
 --
 
+`discard_punctuation`::
+
+    Whether punctuation should be discarded from the output. Defaults to `true`.
+
 `user_dictionary`::
 +
 --
@@ -99,6 +103,7 @@ PUT nori_sample
           "nori_user_dict": {
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
+            "discard_punctuation": "false",
             "user_dictionary": "userdict_ko.txt"
           }
         },
@@ -434,3 +439,107 @@ Which responds with:
 --------------------------------------------------
 
 <1> The Hanja form is replaced by the Hangul translation.
+
+
+[[analysis-nori-number]]
+==== `nori_number` token filter
+
+The `nori_number` token filter normalizes Korean numbers
+to regular Arabic decimal numbers in half-width characters.
+
+Korean numbers are often written using a combination of Hangul and Arabic numbers with various kinds punctuation.
+For example, ３．２천 means 3200.
+This filter does this kind of normalization and allows a search for 3200 to match ３．２천 in text,
+but can also be used to make range facets based on the normalized numbers and so on.
+
+[NOTE]
+====
+Notice that this analyzer uses a token composition scheme and relies on punctuation tokens
+being found in the token stream.
+Please make sure your `nori_tokenizer` has `discard_punctuation` set to false.
+In case punctuation characters, such as U+FF0E(．), is removed from the token stream,
+this filter would find input tokens ３ and ２천 and give outputs 3 and 2000 instead of 3200,
+which is likely not the intended result.
+
+If you want to remove punctuation characters from your index that are not part of normalized numbers,
+add a `stop` token filter with the punctuation you wish to remove after `nori_number` in your analyzer chain.
+====
+Below are some examples of normalizations this filter supports.
+The input is untokenized text and the result is the single term attribute emitted for the input.
+
+- 영영칠 -> 7
+- 일영영영 -> 1000
+- 삼천2백2십삼 -> 3223
+- 조육백만오천일 -> 1000006005001
+- ３.２천 ->  3200
+- １.２만３４５.６７ -> 12345.67
+- 4,647.100 -> 4647.1
+- 15,7 -> 157 (be aware of this weakness)
+
+For example:
+
+[source,console]
+--------------------------------------------------
+PUT nori_sample
+{
+  "settings": {
+    "index": {
+      "analysis": {
+        "analyzer": {
+          "my_analyzer": {
+            "tokenizer": "tokenizer_discard_puncuation_false",
+            "filter": [
+              "part_of_speech_stop_sp", "nori_number"
+            ]
+          }
+        },
+        "tokenizer": {
+          "tokenizer_discard_puncuation_false": {
+            "type": "nori_tokenizer",
+            "discard_punctuation": "false"
+          }
+        },
+        "filter": {
+            "part_of_speech_stop_sp": {
+                "type": "nori_part_of_speech",
+                "stoptags": ["SP"]
+            }
+        }
+      }
+    }
+  }
+}
+
+GET nori_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "십만이천오백과 ３.２천"
+}
+--------------------------------------------------
+
+Which results in:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "tokens" : [{
+    "token" : "102500",
+    "start_offset" : 0,
+    "end_offset" : 6,
+    "type" : "word",
+    "position" : 0
+  }, {
+    "token" : "과",
+    "start_offset" : 6,
+    "end_offset" : 7,
+    "type" : "word",
+    "position" : 1
+  }, {
+    "token" : "3200",
+    "start_offset" : 8,
+    "end_offset" : 12,
+    "type" : "word",
+    "position" : 2
+  }]
+}
+--------------------------------------------------
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java
new file mode 100644
index 0000000000000..54a5ab9c2124e
--- /dev/null
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.ko.KoreanNumberFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+
+public class NoriNumberFilterFactory extends AbstractTokenFilterFactory {
+
+    public NoriNumberFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new KoreanNumberFilter(tokenStream);
+    }
+}
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
index bac5dd2a77065..9680d6fd5f80a 100644
--- a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
@@ -39,11 +39,13 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory {
 
     private final UserDictionary userDictionary;
     private final KoreanTokenizer.DecompoundMode decompoundMode;
+    private final boolean discardPunctuation;
 
     public NoriTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, settings, name);
         decompoundMode = getMode(settings);
         userDictionary = getUserDictionary(env, settings);
+        discardPunctuation = settings.getAsBoolean("discard_punctuation", true);
     }
 
     public static UserDictionary getUserDictionary(Environment env, Settings settings) {
@@ -77,7 +79,8 @@ public static KoreanTokenizer.DecompoundMode getMode(Settings settings) {
 
     @Override
     public Tokenizer create() {
-        return new KoreanTokenizer(KoreanTokenizer.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, decompoundMode, false);
+        return new KoreanTokenizer(KoreanTokenizer.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, decompoundMode, false,
+            discardPunctuation);
     }
 
 }
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java
index 6e9baa7acd26c..72097e2e83472 100644
--- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.NoriAnalyzerProvider;
+import org.elasticsearch.index.analysis.NoriNumberFilterFactory;
 import org.elasticsearch.index.analysis.NoriPartOfSpeechStopFilterFactory;
 import org.elasticsearch.index.analysis.NoriReadingFormFilterFactory;
 import org.elasticsearch.index.analysis.NoriTokenizerFactory;
@@ -42,6 +43,7 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         Map<String, AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
         extra.put("nori_part_of_speech", NoriPartOfSpeechStopFilterFactory::new);
         extra.put("nori_readingform", NoriReadingFormFilterFactory::new);
+        extra.put("nori_number", NoriNumberFilterFactory::new);
         return extra;
     }
 
diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java
index 1677ba94b8783..de70e26fc6f01 100644
--- a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java
+++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java
@@ -43,6 +43,7 @@ protected Map<String, Class<?>> getTokenFilters() {
         Map<String, Class<?>> filters = new HashMap<>(super.getTokenFilters());
         filters.put("koreanpartofspeechstop", NoriPartOfSpeechStopFilterFactory.class);
         filters.put("koreanreadingform", NoriReadingFormFilterFactory.class);
+        filters.put("koreannumber", NoriNumberFilterFactory.class);
         return filters;
     }
 }
diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java
index 051a2f3e4dc32..87c78c7f981b9 100644
--- a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java
+++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java
@@ -54,6 +54,9 @@ public void testDefaultsNoriAnalysis() throws IOException {
         filterFactory = analysis.tokenFilter.get("nori_readingform");
         assertThat(filterFactory, instanceOf(NoriReadingFormFilterFactory.class));
 
+        filterFactory = analysis.tokenFilter.get("nori_number");
+        assertThat(filterFactory, instanceOf(NoriNumberFilterFactory.class));
+
         IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
         NamedAnalyzer analyzer = indexAnalyzers.get("nori");
         assertThat(analyzer.analyzer(), instanceOf(KoreanAnalyzer.class));
@@ -130,6 +133,33 @@ public void testNoriTokenizer() throws Exception {
         assertTokenStreamContents(tokenizer, new String[] {"뿌리", "가", "깊", "은", "나무"});
         tokenizer.setReader(new StringReader("가늠표"));
         assertTokenStreamContents(tokenizer, new String[] {"가늠표", "가늠", "표"});
+        // discard_punctuation default(true)
+        tokenizer.setReader(new StringReader("3.2개"));
+        assertTokenStreamContents(tokenizer, new String[] {"3", "2", "개"});
+    }
+
+    public void testNoriTokenizerDiscardPunctuationOptionTrue() throws Exception {
+        Settings settings = createDiscardPunctuationOption("true");
+        TestAnalysis analysis = createTestAnalysis(settings);
+        Tokenizer tokenizer = analysis.tokenizer.get("my_tokenizer").create();
+        tokenizer.setReader(new StringReader("3.2개"));
+        assertTokenStreamContents(tokenizer, new String[] {"3", "2", "개"});
+    }
+
+    public void testNoriTokenizerDiscardPunctuationOptionFalse() throws Exception {
+        Settings settings = createDiscardPunctuationOption("false");
+        TestAnalysis analysis = createTestAnalysis(settings);
+        Tokenizer tokenizer = analysis.tokenizer.get("my_tokenizer").create();
+        tokenizer.setReader(new StringReader("3.2개"));
+        assertTokenStreamContents(tokenizer, new String[] {"3", ".", "2", "개"});
+    }
+
+    public void testNoriTokenizerInvalidDiscardPunctuationOption() {
+        String wrongOption = "wrong";
+        Settings settings = createDiscardPunctuationOption(wrongOption);
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings));
+        assertThat(exc.getMessage(), containsString("Failed to parse value [" + wrongOption
+            + "] as only [true] or [false] are allowed."));
     }
 
     public void testNoriPartOfSpeech() throws IOException {
@@ -159,6 +189,27 @@ public void testNoriReadingForm() throws IOException {
         assertTokenStreamContents(stream, new String[] {"향가"});
     }
 
+    public void testNoriNumber() throws IOException {
+        Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("index.analysis.filter.my_filter.type", "nori_number")
+            .build();
+        TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new AnalysisNoriPlugin());
+        TokenFilterFactory factory = analysis.tokenFilter.get("my_filter");
+        Tokenizer tokenizer = new KoreanTokenizer();
+        tokenizer.setReader(new StringReader("오늘 십만이천오백원짜리 와인 구입"));
+        TokenStream stream = factory.create(tokenizer);
+        assertTokenStreamContents(stream, new String[] {"오늘", "102500", "원", "짜리", "와인", "구입"});
+    }
+
+    private Settings createDiscardPunctuationOption(String option) {
+        return Settings.builder()
+            .put("index.analysis.tokenizer.my_tokenizer.type", "nori_tokenizer")
+            .put("index.analysis.tokenizer.my_tokenizer.discard_punctuation", option)
+            .build();
+    }
+
     private TestAnalysis createTestAnalysis(Settings analysisSettings) throws IOException {
         InputStream dict = NoriAnalysisTests.class.getResourceAsStream("user_dict.txt");
         Path home = createTempDir();
diff --git a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml
index a5aa9998da6ba..523874f5743bb 100644
--- a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml
+++ b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml
@@ -46,3 +46,20 @@
             filter:       [nori_readingform]
     - length: { tokens: 1 }
     - match:  { tokens.0.token: 향가 }
+---
+"Number filter":
+    - do:
+        indices.analyze:
+          body:
+            text:                   십만이천오백과 ３.２천
+            tokenizer:
+              type:                 nori_tokenizer
+              discard_punctuation:  false
+            filter:
+              - type:               nori_part_of_speech
+                stoptags:           ["SP"]
+              - type:               nori_number
+    - length: { tokens: 3 }
+    - match:  { tokens.0.token: "102500"}
+    - match:  { tokens.1.token: 과}
+    - match:  { tokens.2.token: "3200"}

From ffbb5582f58cef14c617239cbe430c0567f9a347 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 23 Mar 2020 11:32:41 -0700
Subject: [PATCH 226/285] Re-enable bwc tests disabled from #53912 (#54001)

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index da21449b30325..17cd5f6b93cd0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53912" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 569dffc88a55a80cd93273d5b55830a2108888bb Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Mar 2020 14:39:30 -0400
Subject: [PATCH 227/285] Move pipeline agg validation to coordinating node
 (#53669)

This moves the pipeline aggregation validation from the data node to the
coordinating node so that we, eventually, can stop sending pipeline
aggregations to the data nodes entirely. In fact, it moves it into the
"request validation" stage so multiple errors can be accumulated and
sent back to the requester for the entire request. We can't always take
advantage of that, but it'll be nice for folks not to have to play
whack-a-mole with validation.

This is implemented by replacing `PipelineAggretionBuilder#validate`
with:
```
protected abstract void validate(ValidationContext context);
```

The `ValidationContext` handles the accumulation of validation failures,
provides access to the aggregation's siblings, and implements a few
validation utility methods.
---
 .../action/search/SearchRequest.java          |   5 +
 .../aggregations/AggregatorFactories.java     |  51 +++++-
 .../PipelineAggregationBuilder.java           | 148 +++++++++++++++++-
 .../AbstractPipelineAggregationBuilder.java   |  42 -----
 ...cketMetricsPipelineAggregationBuilder.java |  30 ++--
 ...ucketScriptPipelineAggregationBuilder.java |   5 +
 ...ketSelectorPipelineAggregationBuilder.java |   5 +
 .../BucketSortPipelineAggregationBuilder.java |   9 +-
 ...mulativeSumPipelineAggregationBuilder.java |  13 +-
 .../DerivativePipelineAggregationBuilder.java |  13 +-
 ...StatsBucketPipelineAggregationBuilder.java |  15 +-
 .../MovFnPipelineAggregationBuilder.java      |  12 +-
 ...tilesBucketPipelineAggregationBuilder.java |  14 +-
 .../SerialDiffPipelineAggregationBuilder.java |  11 +-
 .../search/SearchModuleTests.java             |   3 +
 .../AbstractBucketMetricsTestCase.java        |   1 -
 .../aggregations/pipeline/AvgBucketTests.java |  25 ++-
 .../aggregations/pipeline/BucketSortIT.java   |  10 +-
 .../CumulativeSumAggregatorTests.java         |  33 ----
 .../pipeline/CumulativeSumTests.java          |  29 +++-
 .../pipeline/DerivativeTests.java             |  28 ++--
 .../pipeline/ExtendedStatsBucketTests.java    |  24 ++-
 .../aggregations/pipeline/MaxBucketTests.java |  25 ++-
 .../aggregations/pipeline/MinBucketTests.java |  25 ++-
 ...nitTests.java => MovFnAggrgatorTests.java} |  36 +----
 ...eAggregationBuilderSerializationTests.java |  42 +++--
 .../pipeline/PercentilesBucketTests.java      |  24 ++-
 .../PipelineAggregationHelperTests.java       |  47 ++----
 .../pipeline/SerialDifferenceTests.java       |  41 ++---
 .../pipeline/StatsBucketTests.java            |  25 ++-
 .../aggregations/pipeline/SumBucketTests.java |  25 ++-
 .../BasePipelineAggregationTestCase.java      |  42 ++++-
 ...CardinalityPipelineAggregationBuilder.java |  19 ++-
 .../analytics/StubAggregatorFactory.java      |  56 -------
 .../CumulativeCardinalityAggregatorTests.java |  66 --------
 .../CumulativeCardinalityTests.java           |  73 +++++++++
 36 files changed, 546 insertions(+), 526 deletions(-)
 rename server/src/test/java/org/elasticsearch/search/aggregations/pipeline/{MovFnUnitTests.java => MovFnAggrgatorTests.java} (79%)
 rename {server/src/test => test/framework/src/main}/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java (85%)
 delete mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java
 create mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index a0be11eb0c2af..eae980cd90f67 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -277,6 +277,11 @@ public ActionRequestValidationException validate() {
                     addValidationError("[request_cache] cannot be used in a scroll context", validationException);
             }
         }
+        if (source != null) {
+            if (source.aggregations() != null) {
+                validationException = source.aggregations().validate(validationException);
+            }
+        }
         return validationException;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 1998a53ad06b0..397c6ea762cf1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -283,8 +284,6 @@ public boolean mustVisitAllDocs() {
             return false;
         }
 
-
-
         public Builder addAggregator(AggregationBuilder factory) {
             if (!names.add(factory.name)) {
                 throw new IllegalArgumentException("Two sibling aggregations cannot have the same name: [" + factory.name + "]");
@@ -298,15 +297,51 @@ public Builder addPipelineAggregator(PipelineAggregationBuilder pipelineAggregat
             return this;
         }
 
+        /**
+         * Validate the root of the aggregation tree.
+         */
+        public ActionRequestValidationException validate(ActionRequestValidationException e) {
+            PipelineAggregationBuilder.ValidationContext context =
+                    PipelineAggregationBuilder.ValidationContext.forTreeRoot(aggregationBuilders, pipelineAggregatorBuilders, e);
+            validatePipelines(context);
+            return validateChildren(context.getValidationException());
+        }
+
+        /**
+         * Validate a the pipeline aggregations in this factory.
+         */
+        private void validatePipelines(PipelineAggregationBuilder.ValidationContext context) {
+            List<PipelineAggregationBuilder> orderedPipelineAggregators;
+            try {
+                orderedPipelineAggregators = resolvePipelineAggregatorOrder(pipelineAggregatorBuilders, aggregationBuilders);
+            } catch (IllegalArgumentException iae) {
+                context.addValidationError(iae.getMessage());
+                return;
+            }
+            for (PipelineAggregationBuilder builder : orderedPipelineAggregators) {
+                builder.validate(context);
+            }
+        }
+
+        /**
+         * Validate a the children of this factory.
+         */
+        private ActionRequestValidationException validateChildren(ActionRequestValidationException e) {
+            for (AggregationBuilder agg : aggregationBuilders) {
+                PipelineAggregationBuilder.ValidationContext context =
+                        PipelineAggregationBuilder.ValidationContext.forInsideTree(agg, e);
+                agg.factoriesBuilder.validatePipelines(context);
+                e = agg.factoriesBuilder.validateChildren(context.getValidationException());
+            }
+            return e;
+        }
+
         public AggregatorFactories build(QueryShardContext queryShardContext, AggregatorFactory parent) throws IOException {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
-            List<PipelineAggregationBuilder> orderedpipelineAggregators = null;
-            orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregationBuilders);
-            for (PipelineAggregationBuilder builder : orderedpipelineAggregators) {
-                builder.validate(parent, aggregationBuilders, pipelineAggregatorBuilders);
-            }
+            List<PipelineAggregationBuilder> orderedPipelineAggregators =
+                    resolvePipelineAggregatorOrder(pipelineAggregatorBuilders, aggregationBuilders);
             AggregatorFactory[] aggFactories = new AggregatorFactory[aggregationBuilders.size()];
 
             int i = 0;
@@ -314,7 +349,7 @@ public AggregatorFactories build(QueryShardContext queryShardContext, Aggregator
                 aggFactories[i] = agg.build(queryShardContext, parent);
                 ++i;
             }
-            return new AggregatorFactories(aggFactories, orderedpipelineAggregators);
+            return new AggregatorFactories(aggFactories, orderedPipelineAggregators);
         }
 
         private List<PipelineAggregationBuilder> resolvePipelineAggregatorOrder(
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
index 85b9a7dcb0ca5..aaffcfc440bce 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
@@ -18,14 +18,20 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * A factory that knows how to create an {@link PipelineAggregator} of a
@@ -64,11 +70,145 @@ public final String[] getBucketsPaths() {
     }
 
     /**
-     * Internal: Validates the state of this factory (makes sure the factory is properly
-     * configured)
+     * Makes sure this builder is properly configured.
      */
-    protected abstract void validate(AggregatorFactory parent, Collection<AggregationBuilder> aggregationBuilders,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders);
+    protected abstract void validate(ValidationContext context);
+    public abstract static class ValidationContext {
+        /**
+         * Build the context for the root of the aggregation tree.
+         */
+        public static ValidationContext forTreeRoot(Collection<AggregationBuilder> siblingAggregations,
+                Collection<PipelineAggregationBuilder> siblingPipelineAggregations,
+                ActionRequestValidationException validationFailuresSoFar) {
+            return new ForTreeRoot(siblingAggregations, siblingPipelineAggregations, validationFailuresSoFar);
+        }
+
+        /**
+         * Build the context for a node inside the aggregation tree.
+         */
+        public static ValidationContext forInsideTree(AggregationBuilder parent,
+                ActionRequestValidationException validationFailuresSoFar) {
+            return new ForInsideTree(parent, validationFailuresSoFar);
+        }
+
+
+        private ActionRequestValidationException e;
+
+        private ValidationContext(ActionRequestValidationException validationFailuresSoFar) {
+            this.e = validationFailuresSoFar;
+        }
+
+        private static class ForTreeRoot extends ValidationContext {
+            private final Collection<AggregationBuilder> siblingAggregations;
+            private final Collection<PipelineAggregationBuilder> siblingPipelineAggregations;
+
+            ForTreeRoot(Collection<AggregationBuilder> siblingAggregations,
+                    Collection<PipelineAggregationBuilder> siblingPipelineAggregations,
+                    ActionRequestValidationException validationFailuresSoFar) {
+                super(validationFailuresSoFar);
+                this.siblingAggregations = Objects.requireNonNull(siblingAggregations);
+                this.siblingPipelineAggregations = Objects.requireNonNull(siblingPipelineAggregations);
+            }
+
+            @Override
+            public Collection<AggregationBuilder> getSiblingAggregations() {
+                return siblingAggregations;
+            }
+
+            @Override
+            public Collection<PipelineAggregationBuilder> getSiblingPipelineAggregations() {
+                return siblingPipelineAggregations;
+            }
+
+            @Override
+            public void validateParentAggSequentiallyOrdered(String type, String name) {
+                addValidationError(type + " aggregation [" + name
+                        + "] must have a histogram, date_histogram or auto_date_histogram as parent but doesn't have a parent");
+            }
+        }
+
+        private static class ForInsideTree extends ValidationContext {
+            private final AggregationBuilder parent;
+
+            ForInsideTree(AggregationBuilder parent, ActionRequestValidationException validationFailuresSoFar) {
+                super(validationFailuresSoFar);
+                this.parent = Objects.requireNonNull(parent);
+            }
+
+            @Override
+            public Collection<AggregationBuilder> getSiblingAggregations() {
+                return parent.getSubAggregations();
+            }
+
+            @Override
+            public Collection<PipelineAggregationBuilder> getSiblingPipelineAggregations() {
+                return parent.getPipelineAggregations();
+            }
+
+            @Override
+            public void validateParentAggSequentiallyOrdered(String type, String name) {
+                if (parent instanceof HistogramAggregationBuilder) {
+                    HistogramAggregationBuilder histoParent = (HistogramAggregationBuilder) parent;
+                    if (histoParent.minDocCount() != 0) {
+                        addValidationError(
+                                "parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
+                    }
+                } else if (parent instanceof DateHistogramAggregationBuilder) {
+                    DateHistogramAggregationBuilder histoParent = (DateHistogramAggregationBuilder) parent;
+                    if (histoParent.minDocCount() != 0) {
+                        addValidationError(
+                                "parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
+                    }
+                } else if (parent instanceof AutoDateHistogramAggregationBuilder) {
+                    // Nothing to check
+                } else {
+                    addValidationError(
+                            type + " aggregation [" + name + "] must have a histogram, date_histogram or auto_date_histogram as parent");
+                }
+            }
+        }
+
+        /**
+         * Aggregations that are siblings to the aggregation being validated.
+         */
+        public abstract Collection<AggregationBuilder> getSiblingAggregations();
+
+        /**
+         * Pipeline aggregations that are siblings to the aggregation being validated.
+         */
+        public abstract Collection<PipelineAggregationBuilder> getSiblingPipelineAggregations();
+
+        /**
+         * Add a validation error to this context. All validation errors
+         * are accumulated in a list and, if there are any, the request
+         * is not executed and the entire list is returned as the error
+         * response.
+         */
+        public void addValidationError(String error) {
+            e = ValidateActions.addValidationError(error, e);
+        }
+
+        /**
+         * Add a validation error about the {@code buckets_path}.
+         */
+        public void addBucketPathValidationError(String error) {
+            addValidationError(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName() + ' ' + error);
+        }
+
+        /**
+         * Validates that the parent is sequentially ordered.
+         */
+        public abstract void validateParentAggSequentiallyOrdered(String type, String name);
+
+        /**
+         * The validation exception, if there is one. It'll be {@code null}
+         * if the context wasn't provided with any exception on creation
+         * and none were added.
+         */
+        public ActionRequestValidationException getValidationException() {
+            return e;
+        }
+    }
 
     /**
      * Creates the pipeline aggregator
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
index 35eab881bb643..c2bffc0450677 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
@@ -22,16 +22,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -79,16 +73,6 @@ public String type() {
         return type;
     }
 
-    /**
-     * Validates the state of this factory (makes sure the factory is properly
-     * configured)
-     */
-    @Override
-    public final void validate(AggregatorFactory parent, Collection<AggregationBuilder> factories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-        doValidate(parent, factories, pipelineAggregatorFactories);
-    }
-
     protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     /**
@@ -102,32 +86,6 @@ public final PipelineAggregator create() {
         return aggregator;
     }
 
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> factories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-    }
-
-    /**
-     * Validates pipeline aggregations that need sequentially ordered data.
-     */
-    public static void validateSequentiallyOrderedParentAggs(AggregatorFactory parent, String type, String name) {
-        if ((parent instanceof HistogramAggregatorFactory || parent instanceof DateHistogramAggregatorFactory
-                || parent instanceof AutoDateHistogramAggregatorFactory) == false) {
-            throw new IllegalStateException(
-                    type + " aggregation [" + name + "] must have a histogram, date_histogram or auto_date_histogram as parent");
-        }
-        if (parent instanceof HistogramAggregatorFactory) {
-            HistogramAggregatorFactory histoParent = (HistogramAggregatorFactory) parent;
-            if (histoParent.minDocCount() != 0) {
-                throw new IllegalStateException("parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
-            }
-        } else if (parent instanceof DateHistogramAggregatorFactory) {
-            DateHistogramAggregatorFactory histoParent = (DateHistogramAggregatorFactory) parent;
-            if (histoParent.minDocCount() != 0) {
-                throw new IllegalStateException("parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
-            }
-        }
-    }
-
     @SuppressWarnings("unchecked")
     @Override
     public PAB setMetaData(Map<String, Object> metaData) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
index 0c4aa6f1359bd..753f3ca38821c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
@@ -24,13 +24,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -107,28 +104,27 @@ public GapPolicy gapPolicy() {
     protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggBuilders,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                    + " must contain a single entry for aggregation [" + name + "]");
+            context.addBucketPathValidationError("must contain a single entry for aggregation [" + name + "]");
+            return;
         }
         // Need to find the first agg name in the buckets path to check its a
         // multi bucket agg: aggs are split with '>' and can optionally have a
         // metric name after them by using '.' so need to split on both to get
         // just the agg name
         final String firstAgg = bucketsPaths[0].split("[>\\.]")[0];
-        Optional<AggregationBuilder> aggBuilder = aggBuilders.stream().filter((builder) -> builder.getName().equals(firstAgg))
+        Optional<AggregationBuilder> aggBuilder = context.getSiblingAggregations().stream()
+                .filter(builder -> builder.getName().equals(firstAgg))
                 .findAny();
-        if (aggBuilder.isPresent()) {
-            if ((aggBuilder.get() instanceof MultiBucketAggregationBuilder) == false) {
-                throw new IllegalArgumentException("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                        + " must be a multi-bucket aggregation for aggregation [" + name + "] found :"
-                        + aggBuilder.get().getClass().getName() + " for buckets path: " + bucketsPaths[0]);
-            }
-        } else {
-            throw new IllegalArgumentException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                    + " aggregation does not exist for aggregation [" + name + "]: " + bucketsPaths[0]);
+        if (aggBuilder.isEmpty()) {
+            context.addBucketPathValidationError("aggregation does not exist for aggregation [" + name + "]: " + bucketsPaths[0]);
+            return;
+        }
+        if ((aggBuilder.get() instanceof MultiBucketAggregationBuilder) == false) {
+            context.addValidationError("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                    + " must be a multi-bucket aggregation for aggregation [" + name + "] found :"
+                    + aggBuilder.get().getClass().getName() + " for buckets path: " + bucketsPaths[0]);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
index 5fddd5f1a9e1c..fa6c6b7639413 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
@@ -198,6 +198,11 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
         return builder;
     }
 
+    @Override
+    protected void validate(ValidationContext context) {
+        // Nothing to check
+    }
+
     @Override
     protected boolean overrideBucketsPath() {
         return true;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
index 6bb83c60dea83..0981acbc9bafd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
@@ -193,6 +193,11 @@ public static BucketSelectorPipelineAggregationBuilder parse(String reducerName,
         return factory;
     }
 
+    @Override
+    protected void validate(ValidationContext context) {
+        // Nothing to check
+    }
+
     @Override
     protected boolean overrideBucketsPath() {
         return true;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
index 1e80c6f78c3c5..7f6b491975868 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
@@ -25,9 +25,6 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -36,7 +33,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -144,10 +140,9 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-                           Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
+    protected void validate(ValidationContext context) {
         if (sorts.isEmpty() && size == null && from == 0) {
-            throw new IllegalStateException("[" + name + "] is configured to perform nothing. Please set either of "
+            context.addValidationError("[" + name + "] is configured to perform nothing. Please set either of "
                     + Arrays.asList(SearchSourceBuilder.SORT_FIELD.getPreferredName(), SIZE.getPreferredName(), FROM.getPreferredName())
                     + " to use " + NAME);
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
index fc9b8caf72769..c1af597b4c275 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
@@ -25,13 +25,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -93,14 +89,11 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
-                    + " must contain a single entry for aggregation [" + name + "]");
+            context.addBucketPathValidationError("must contain a single entry for aggregation [" + name + "]");
         }
-        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
index 4aa4155ac063f..106a8a97f5728 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
@@ -28,16 +28,12 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -152,14 +148,13 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+            context.addValidationError(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
         }
-        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
index 89816a1fb22e8..97948d0e8638a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
@@ -22,12 +22,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -80,13 +76,10 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggBuilders,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-        super.doValidate(parent, aggBuilders, pipelineAggregatorFactories);
-
-        if (sigma < 0.0 ) {
-            throw new IllegalStateException(ExtendedStatsBucketParser.SIGMA.getPreferredName()
-                    + " must be a non-negative double");
+    protected void validate(ValidationContext context) {
+        super.validate(context);
+        if (sigma < 0.0) {
+            context.addValidationError(ExtendedStatsBucketParser.SIGMA.getPreferredName() + " must be a non-negative double");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
index d24710631d8fd..e5ec2780b0c14 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
@@ -30,13 +30,9 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -179,13 +175,11 @@ public void setShift(int shift) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-                           Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (window <= 0) {
-            throw new IllegalArgumentException("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
+            context.addValidationError("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
         }
-        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
index 06a940af89c77..a806b54772140 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
@@ -20,18 +20,15 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 import com.carrotsearch.hppc.DoubleArrayList;
+
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -109,14 +106,13 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-        super.doValidate(parent, aggFactories, pipelineAggregatorFactories);
-
+    protected void validate(ValidationContext context) {
+        super.validate(context);
         for (Double p : percents) {
             if (p == null || p < 0.0 || p > 100.0) {
-                throw new IllegalStateException(PERCENTS_FIELD.getPreferredName()
+                context.addValidationError(PERCENTS_FIELD.getPreferredName()
                         + " must only contain non-null doubles from 0.0-100.0 inclusive");
+                return;
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
index 3b02edf51579a..fe6bb921b9129 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
@@ -26,14 +26,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -138,11 +134,10 @@ protected DocValueFormat formatter() {
     protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new SerialDiffPipelineAggregator(name, bucketsPaths, formatter(), gapPolicy, lag, metaData);
     }
-    
+
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+    protected void validate(ValidationContext context) {
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index ff26f49a92686..48daf474d0ded 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -436,6 +436,9 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
         private static TestPipelineAggregationBuilder fromXContent(String name, XContentParser p) {
             return null;
         }
+
+        @Override
+        protected void validate(ValidationContext context) {}
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
index 8f268cecaedd2..d53a147261cb1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
@@ -40,5 +40,4 @@ protected final PAF createTestAggregatorFactory() {
     }
 
     protected abstract PAF doCreateTestAggregatorFactory(String name, String bucketsPath);
-
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java
index 0dc10cb7a7a13..2c5f0f9f343c3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class AvgBucketTests extends AbstractBucketMetricsTestCase<AvgBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final AvgBucketPipelineAggregationBuilder builder = new AvgBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new AvgBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        AvgBucketPipelineAggregationBuilder builder2 = new AvgBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new AvgBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
-
-        // Now try to point to a valid multi-bucket agg (no exception should be thrown)
-        AvgBucketPipelineAggregationBuilder builder3 = new AvgBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+                + " for buckets path: global>metric;"));
 
+        // Now try to point to a valid multi-bucket agg which is valid
+        assertThat(validate(aggBuilders, new AvgBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java
index d5e70aff07b69..ead4870731b1f 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -459,21 +459,21 @@ public void testEmptyBuckets() {
     }
 
     public void testInvalidPath() {
-        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
+        Exception e = expectThrows(ActionRequestValidationException.class,
                 () -> client().prepareSearch(INDEX)
                 .addAggregation(terms("foos").field(TERM_FIELD)
                         .subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("invalid")))))
                 .get());
-        assertThat(e.getCause().getMessage(), containsString("No aggregation found for path [invalid]"));
+        assertThat(e.getMessage(), containsString("No aggregation found for path [invalid]"));
     }
 
     public void testNeitherSortsNorSizeSpecifiedAndFromIsDefault_ShouldThrowValidation() {
-        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
+        Exception e = expectThrows(ActionRequestValidationException.class,
                 () -> client().prepareSearch(INDEX)
                         .addAggregation(terms("foos").field(TERM_FIELD)
                                 .subAggregation(bucketSort("bucketSort", Collections.emptyList())))
                         .get());
-        assertThat(e.getCause().getMessage(), containsString("[bucketSort] is configured to perform nothing." +
+        assertThat(e.getMessage(), containsString("[bucketSort] is configured to perform nothing." +
                 " Please set either of [sort, size, from] to use bucket_sort"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
index 9d27663d275f7..dd3e971f4400b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
@@ -38,8 +38,6 @@
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -52,10 +50,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -291,34 +286,6 @@ public void testNoBuckets() throws IOException {
             }
         });
     }
-    
-    /**
-     * The validation should verify the parent aggregation is allowed.
-     */
-    public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new CumulativeSumPipelineAggregationBuilder("cusum", "sum"));
-
-        final CumulativeSumPipelineAggregationBuilder builder = new CumulativeSumPipelineAggregationBuilder("name", "valid");
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
-    }
-
-    /**
-     * The validation should throw an IllegalArgumentException, since parent
-     * aggregation is not a type of HistogramAggregatorFactory,
-     * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory.
-     */
-    public void testValidateException() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new CumulativeSumPipelineAggregationBuilder("cusum", "sum"));
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
-
-        final CumulativeSumPipelineAggregationBuilder builder = new CumulativeSumPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("cumulative_sum aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
-    }
 
     private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer<InternalAggregation> verify) throws IOException {
         executeTestCase(query, aggBuilder, verify, indexWriter -> {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
index edf879ce77f68..f51b8ec479d8b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
@@ -19,10 +19,18 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 
-public class CumulativeSumTests extends BasePipelineAggregationTestCase<CumulativeSumPipelineAggregationBuilder> {
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
+public class CumulativeSumTests extends BasePipelineAggregationTestCase<CumulativeSumPipelineAggregationBuilder> {
     @Override
     protected CumulativeSumPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAlphaOfLengthBetween(3, 20);
@@ -34,4 +42,23 @@ protected CumulativeSumPipelineAggregationBuilder createTestAggregatorFactory()
         return factory;
     }
 
+    public void testValidate() throws IOException {
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new CumulativeSumPipelineAggregationBuilder("name", "valid")), nullValue());
+    }
+
+    public void testInvalidParent() throws IOException {
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
+
+        assertThat(validate(parent, new CumulativeSumPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: cumulative_sum aggregation [name] must have a histogram, date_histogram "
+                + "or auto_date_histogram as parent;"));
+    }
+
+    public void testNoParent() throws IOException {
+        assertThat(validate(emptyList(), new CumulativeSumPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: cumulative_sum aggregation [name] must have a histogram, date_histogram "
+                + "or auto_date_histogram as parent but doesn't have a parent;"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
index 7fd6e1e86266b..662778315f7e4 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
@@ -19,16 +19,20 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class DerivativeTests extends BasePipelineAggregationTestCase<DerivativePipelineAggregationBuilder> {
 
     @Override
@@ -51,16 +55,13 @@ protected DerivativePipelineAggregationBuilder createTestAggregatorFactory() {
         }
         return factory;
     }
-    
+
     /**
      * The validation should verify the parent aggregation is allowed.
      */
     public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new DerivativePipelineAggregationBuilder("deriv", "der"));
-
-        final DerivativePipelineAggregationBuilder builder = new DerivativePipelineAggregationBuilder("name", "valid");
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new DerivativePipelineAggregationBuilder("name", "valid")), nullValue());
     }
 
     /**
@@ -71,12 +72,11 @@ public void testValidate() throws IOException {
     public void testValidateException() throws IOException {
         final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
         aggBuilders.add(new DerivativePipelineAggregationBuilder("deriv", "der"));
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
 
-        final DerivativePipelineAggregationBuilder builder = new DerivativePipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("derivative aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
+        assertThat(validate(parent, new DerivativePipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: derivative aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent;"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java
index 9930541cb007e..90cc9be95e3be 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java
@@ -26,11 +26,11 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<ExtendedStatsBucketPipelineAggregationBuilder> {
 
@@ -68,23 +68,17 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final ExtendedStatsBucketPipelineAggregationBuilder builder = new ExtendedStatsBucketPipelineAggregationBuilder("name",
-                "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new ExtendedStatsBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        ExtendedStatsBucketPipelineAggregationBuilder builder2 = new ExtendedStatsBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new ExtendedStatsBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        ExtendedStatsBucketPipelineAggregationBuilder builder3 = new ExtendedStatsBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new ExtendedStatsBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java
index c55152c68c3a6..edbc1cda3eae2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class MaxBucketTests extends AbstractBucketMetricsTestCase<MaxBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final MaxBucketPipelineAggregationBuilder builder = new MaxBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new MaxBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        MaxBucketPipelineAggregationBuilder builder2 = new MaxBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new MaxBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        MaxBucketPipelineAggregationBuilder builder3 = new MaxBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new MaxBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java
index 317f1360c7845..057e074a90cfc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class MinBucketTests extends AbstractBucketMetricsTestCase<MinBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final MinBucketPipelineAggregationBuilder builder = new MinBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new MinBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        MinBucketPipelineAggregationBuilder builder2 = new MinBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new MinBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        MinBucketPipelineAggregationBuilder builder3 = new MinBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new MinBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java
similarity index 79%
rename from server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java
rename to server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java
index 9d3b10c39b1ff..e297e8a473f5f 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java
@@ -42,8 +42,6 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -53,16 +51,14 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class MovFnUnitTests extends AggregatorTestCase {
+public class MovFnAggrgatorTests extends AggregatorTestCase {
 
     private static final String DATE_FIELD = "date";
     private static final String INSTANT_FIELD = "instant";
@@ -172,34 +168,4 @@ private void executeTestCase(Query query,
     private static long asLong(String dateTime) {
         return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli();
     }
-    
-    /**
-     * The validation should verify the parent aggregation is allowed.
-     */
-    public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
-        aggBuilders.add(new MovFnPipelineAggregationBuilder("mov_fn", "avg", script, 3));
-
-        final MovFnPipelineAggregationBuilder builder = new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1);
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
-    }
-
-    /**
-     * The validation should throw an IllegalArgumentException, since parent
-     * aggregation is not a type of HistogramAggregatorFactory,
-     * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory.
-     */
-    public void testValidateException() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
-        aggBuilders.add(new MovFnPipelineAggregationBuilder("mov_fn", "avg", script, 3));
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
-
-        final MovFnPipelineAggregationBuilder builder = new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1);
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("moving_fn aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java
index cb1e2d5249b4a..17fbac3495c56 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java
@@ -19,17 +19,23 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 
-public class MovFnPipelineAggregationBuilderSerializationTests extends AbstractSerializingTestCase<MovFnPipelineAggregationBuilder> {
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
+public class MovFnPipelineAggregationBuilderSerializationTests extends BasePipelineAggregationTestCase<MovFnPipelineAggregationBuilder> {
     @Override
-    protected MovFnPipelineAggregationBuilder createTestInstance() {
+    protected MovFnPipelineAggregationBuilder createTestAggregatorFactory() {
         MovFnPipelineAggregationBuilder builder = new MovFnPipelineAggregationBuilder(
             randomAlphaOfLength(10),
             "foo",
@@ -40,19 +46,27 @@ protected MovFnPipelineAggregationBuilder createTestInstance() {
         return builder;
     }
 
-    @Override
-    protected Writeable.Reader<MovFnPipelineAggregationBuilder> instanceReader() {
-        return MovFnPipelineAggregationBuilder::new;
+    public void testValidParent() throws IOException {
+        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", emptyMap());
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new MovFnPipelineAggregationBuilder("mov_fn", "avg", script, 3)), nullValue());
     }
 
-    @Override
-    protected MovFnPipelineAggregationBuilder doParseInstance(XContentParser parser) throws IOException {
-        return MovFnPipelineAggregationBuilder.parse(parser);
+    public void testInvalidParent() throws IOException {
+        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
+
+        assertThat(validate(parent, new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1)), equalTo(
+                "Validation Failed: 1: moving_fn aggregation [name] must have a histogram, date_histogram"
+                + " or auto_date_histogram as parent;"));
     }
 
-    @Override
-    protected boolean supportsUnknownFields() {
-        return false;
+    public void testNoParent() throws IOException {
+        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
+                assertThat(validate(emptyList(), new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1)), equalTo(
+                "Validation Failed: 1: moving_fn aggregation [name] must have a histogram, date_histogram"
+                + " or auto_date_histogram as parent but doesn't have a parent;"));
     }
 }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java
index 165312a5bdef5..1918d26e1e1af 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java
@@ -26,11 +26,11 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<PercentilesBucketPipelineAggregationBuilder> {
 
@@ -72,23 +72,17 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final PercentilesBucketPipelineAggregationBuilder builder = new PercentilesBucketPipelineAggregationBuilder("name",
-                "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new PercentilesBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        PercentilesBucketPipelineAggregationBuilder builder2 = new PercentilesBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new PercentilesBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        PercentilesBucketPipelineAggregationBuilder builder3 = new PercentilesBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new PercentilesBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
index 9a2d5a411d4ad..2d67735468984 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
@@ -20,29 +20,20 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 
-import org.elasticsearch.common.Rounding;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalOrder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-
-import static org.mockito.Mockito.mock;
+import java.util.function.Function;
 
 /**
  * Provides helper methods and classes for use in PipelineAggregation tests,
@@ -155,28 +146,12 @@ public static double calculateMetric(double[] values, ValuesSourceAggregationBui
         return 0.0;
     }
 
-    static AggregatorFactory getRandomSequentiallyOrderedParentAgg() throws IOException {
-        AggregatorFactory factory = null;
-        switch (randomIntBetween(0, 2)) {
-            case 0:
-                factory = new HistogramAggregatorFactory("name", mock(ValuesSourceConfig.class), 0.0d, 0.0d,
-                    mock(InternalOrder.class), false, 0L, 0.0d, 1.0d, mock(QueryShardContext.class), null,
-                    new AggregatorFactories.Builder(), Collections.emptyMap());
-                break;
-            case 1:
-                factory = new DateHistogramAggregatorFactory("name", mock(ValuesSourceConfig.class),
-                    mock(InternalOrder.class), false, 0L, mock(Rounding.class), mock(Rounding.class),
-                    mock(ExtendedBounds.class), mock(QueryShardContext.class), mock(AggregatorFactory.class),
-                    new AggregatorFactories.Builder(), Collections.emptyMap());
-                break;
-            case 2:
-            default:
-                AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1];
-                factory = new AutoDateHistogramAggregatorFactory("name", mock(ValuesSourceConfig.class),
-                    1, roundings,
-                    mock(QueryShardContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap());
-        }
-
-        return factory;
+    static AggregationBuilder getRandomSequentiallyOrderedParentAgg() throws IOException {
+        @SuppressWarnings("unchecked")
+        Function<String, AggregationBuilder> builder = randomFrom(
+                HistogramAggregationBuilder::new,
+                DateHistogramAggregationBuilder::new,
+                AutoDateHistogramAggregationBuilder::new);
+        return builder.apply("name");
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
index ef1d13990a308..c5b09a7a5726c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
@@ -19,16 +19,21 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class SerialDifferenceTests extends BasePipelineAggregationTestCase<SerialDiffPipelineAggregationBuilder> {
 
     @Override
@@ -47,32 +52,28 @@ protected SerialDiffPipelineAggregationBuilder createTestAggregatorFactory() {
         }
         return factory;
     }
-    
+
     /**
      * The validation should verify the parent aggregation is allowed.
      */
     public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(createTestAggregatorFactory());
-
-        final SerialDiffPipelineAggregationBuilder builder = new SerialDiffPipelineAggregationBuilder("name", "valid");
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new SerialDiffPipelineAggregationBuilder("name", "valid")), nullValue());
     }
 
-    /**
-     * The validation should throw an IllegalArgumentException, since parent
-     * aggregation is not a type of HistogramAggregatorFactory,
-     * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory.
-     */
-    public void testValidateException() throws IOException {
+    public void testInvalidParent() throws IOException {
         final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
         aggBuilders.add(createTestAggregatorFactory());
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
+        assertThat(validate(parent, new SerialDiffPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: serial_diff aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent;"));
+    }
 
-        final SerialDiffPipelineAggregationBuilder builder = new SerialDiffPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("serial_diff aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
+    public void testNoParent() {
+        assertThat(validate(emptyList(), new SerialDiffPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: serial_diff aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent but doesn't have a parent;"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java
index bf2ef7615df66..aac81c19be6bd 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class StatsBucketTests extends AbstractBucketMetricsTestCase<StatsBucketPipelineAggregationBuilder> {
 
     @Override
@@ -44,23 +46,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final StatsBucketPipelineAggregationBuilder builder = new StatsBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new StatsBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        StatsBucketPipelineAggregationBuilder builder2 = new StatsBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new StatsBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        StatsBucketPipelineAggregationBuilder builder3 = new StatsBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new StatsBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java
index fdba878524146..0bcbf592458aa 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class SumBucketTests extends AbstractBucketMetricsTestCase<SumBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final SumBucketPipelineAggregationBuilder builder = new SumBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new SumBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        SumBucketPipelineAggregationBuilder builder2 = new SumBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new SumBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        SumBucketPipelineAggregationBuilder builder3 = new SumBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new SumBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
similarity index 85%
rename from server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 1d59ba476986d..1b533bdab83c5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -33,13 +33,16 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder.ValidationContext;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
 import static java.util.Collections.emptyList;
@@ -75,7 +78,7 @@ public void setUp() throws Exception {
             .put("node.name", AbstractQueryTestCase.class.toString())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             .build();
-        SearchModule searchModule = new SearchModule(settings, emptyList());
+        SearchModule searchModule = new SearchModule(settings, plugins());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(IndicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
@@ -89,6 +92,13 @@ public void setUp() throws Exception {
         }
     }
 
+    /**
+     * Plugins to add to the test.
+     */
+    protected List<SearchPlugin> plugins() {
+        return emptyList();
+    }
+
     /**
      * Generic test that creates new AggregatorFactory from the test
      * AggregatorFactory and checks both for equality and asserts equality on
@@ -196,4 +206,34 @@ public String randomNumericField() {
     protected NamedXContentRegistry xContentRegistry() {
         return xContentRegistry;
     }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(AggregationBuilder parent, AF builder) {
+        return validate(ValidationContext.forInsideTree(parent, null), builder);
+    }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(Collection<AggregationBuilder> siblingAggregations, AF builder) {
+        return validate(siblingAggregations, emptyList(), builder);
+    }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(Collection<AggregationBuilder> siblingAggregations,
+            Collection<PipelineAggregationBuilder> siblingPipelineAggregations, AF builder) {
+        return validate(ValidationContext.forTreeRoot(siblingAggregations, siblingPipelineAggregations, null), builder);
+    }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(ValidationContext context, AF builder) {
+        builder.validate(context);
+        return context.getValidationException() == null ? null : context.getValidationException().getMessage();
+    }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
index 844c661aadf7e..47480d98a00e2 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
@@ -10,20 +10,15 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 
 public class CumulativeCardinalityPipelineAggregationBuilder
@@ -90,14 +85,12 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-                           Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
-                + " must contain a single entry for aggregation [" + name + "]");
+            context.addBucketPathValidationError("must contain a single entry for aggregation [" + name + "]");
         }
 
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
@@ -105,6 +98,7 @@ protected final XContentBuilder internalXContent(XContentBuilder builder, Params
         if (format != null) {
             builder.field(BucketMetricsParser.FORMAT.getPreferredName(), format);
         }
+        builder.field(BUCKETS_PATH_FIELD.getPreferredName(), bucketsPaths[0]);
         return builder;
     }
 
@@ -126,4 +120,9 @@ public boolean equals(Object obj) {
     public String getWriteableName() {
         return NAME;
     }
+
+    @Override
+    protected boolean overrideBucketsPath() {
+        return true;
+    }
 }
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java
deleted file mode 100644
index ace86ddea6bb8..0000000000000
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.analytics;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.MockBigArrays;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test implementation for AggregatorFactory.
- */
-public class StubAggregatorFactory extends AggregatorFactory {
-
-    private final Aggregator aggregator;
-
-    private StubAggregatorFactory(QueryShardContext queryShardContext, Aggregator aggregator) throws IOException {
-        super("_name", queryShardContext, null, new AggregatorFactories.Builder(), Collections.emptyMap());
-        this.aggregator = aggregator;
-    }
-
-    @Override
-    protected Aggregator createInternal(SearchContext searchContext,
-                                            Aggregator parent,
-                                            boolean collectsFromSingleBucket,
-                                            List list, Map metaData) throws IOException {
-        return aggregator;
-    }
-
-    public static StubAggregatorFactory createInstance() throws IOException {
-        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
-        QueryShardContext queryShardContext = mock(QueryShardContext.class);
-        when(queryShardContext.bigArrays()).thenReturn(bigArrays);
-
-        Aggregator aggregator = mock(Aggregator.class);
-
-        return new StubAggregatorFactory(queryShardContext, aggregator);
-    }
-}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java
index e5b4b2ce0bf07..fd7adddd4234c 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java
@@ -16,46 +16,27 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.CheckedConsumer;
-import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalOrder;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.xpack.analytics.StubAggregatorFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Mockito.mock;
 
 public class CumulativeCardinalityAggregatorTests extends AggregatorTestCase {
 
@@ -115,53 +96,6 @@ public void testAllNull() throws IOException {
         });
     }
 
-    public void testParentValidations() throws IOException {
-        ValuesSourceConfig<ValuesSource> valuesSource = new ValuesSourceConfig<>(CoreValuesSourceType.NUMERIC);
-
-        // Histogram
-        Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        AggregatorFactory parent = new HistogramAggregatorFactory("name", valuesSource, 0.0d, 0.0d,
-            mock(InternalOrder.class), false, 0L, 0.0d, 1.0d, mock(QueryShardContext.class), null,
-            new AggregatorFactories.Builder(), Collections.emptyMap());
-        CumulativeCardinalityPipelineAggregationBuilder builder
-            = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid");
-        builder.validate(parent, Collections.emptySet(), aggBuilders);
-
-        // Date Histogram
-        aggBuilders.clear();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        parent = new DateHistogramAggregatorFactory("name", valuesSource,
-            mock(InternalOrder.class), false, 0L, mock(Rounding.class), mock(Rounding.class),
-            mock(ExtendedBounds.class), mock(QueryShardContext.class), mock(AggregatorFactory.class),
-            new AggregatorFactories.Builder(), Collections.emptyMap());
-        builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid");
-        builder.validate(parent, Collections.emptySet(), aggBuilders);
-
-        // Auto Date Histogram
-        ValuesSourceConfig<ValuesSource.Numeric> numericVS = new ValuesSourceConfig<>(CoreValuesSourceType.NUMERIC);
-        aggBuilders.clear();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1];
-        parent = new AutoDateHistogramAggregatorFactory("name", numericVS,
-            1, roundings,
-            mock(QueryShardContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap());
-        builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid");
-        builder.validate(parent, Collections.emptySet(), aggBuilders);
-
-        // Mocked "test" agg, should fail validation
-        aggBuilders.clear();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        StubAggregatorFactory parentFactory = StubAggregatorFactory.createInstance();
-
-        CumulativeCardinalityPipelineAggregationBuilder failBuilder
-            = new CumulativeCardinalityPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-            () -> failBuilder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("cumulative_cardinality aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-            ex.getMessage());
-    }
-
     public void testNonCardinalityAgg() {
         Query query = new MatchAllDocsQuery();
 
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java
new file mode 100644
index 0000000000000..d9ff43e7d06c8
--- /dev/null
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.analytics.cumulativecardinality;
+
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CumulativeCardinalityTests extends BasePipelineAggregationTestCase<CumulativeCardinalityPipelineAggregationBuilder> {
+    @Override
+    protected List<SearchPlugin> plugins() {
+        return singletonList(new SearchPlugin() {
+            @Override
+            public List<PipelineAggregationSpec> getPipelineAggregations() {
+                return singletonList(new PipelineAggregationSpec(
+                        CumulativeCardinalityPipelineAggregationBuilder.NAME,
+                        CumulativeCardinalityPipelineAggregationBuilder::new,
+                        CumulativeCardinalityPipelineAggregator::new,
+                        CumulativeCardinalityPipelineAggregationBuilder.PARSER));
+            }
+        });
+    }
+
+    @Override
+    protected CumulativeCardinalityPipelineAggregationBuilder createTestAggregatorFactory() {
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
+        CumulativeCardinalityPipelineAggregationBuilder builder =
+                new CumulativeCardinalityPipelineAggregationBuilder(name, bucketsPath);
+        if (randomBoolean()) {
+            builder.format(randomAlphaOfLengthBetween(1, 10));
+        }
+        return builder;
+    }
+
+
+    public void testParentValidations() throws IOException {
+        CumulativeCardinalityPipelineAggregationBuilder builder =
+                new CumulativeCardinalityPipelineAggregationBuilder("name", randomAlphaOfLength(5));
+
+        assertThat(validate(new HistogramAggregationBuilder("name"), builder), nullValue());
+        assertThat(validate(new DateHistogramAggregationBuilder("name"), builder), nullValue());
+        assertThat(validate(new AutoDateHistogramAggregationBuilder("name"), builder), nullValue());
+
+        // Mocked "test" agg, should fail validation
+        AggregationBuilder stubParent = mock(AggregationBuilder.class);
+        when(stubParent.getName()).thenReturn("name");
+        assertThat(validate(stubParent, builder), equalTo(
+                "Validation Failed: 1: cumulative_cardinality aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent;"));
+
+        assertThat(validate(emptyList(), builder), equalTo(
+                "Validation Failed: 1: cumulative_cardinality aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent but doesn't have a parent;"));
+    }
+}

From 2a99f4e50ead8f1bf3f404f31770924391f7f323 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Mon, 23 Mar 2020 14:41:49 -0400
Subject: [PATCH 228/285] [ML] adjusting feature importance mapping for
 multi-class support (#53821)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Feature importance storage format is changing to encompass multi-class.

Feature importance objects are now mapped as follows
(logistic) Regression:
```
{
   "feature_name": "feature_0",
   "importance": -1.3
}
```
Multi-class [class names are `foo`, `bar`, `baz`]
```
{
   “feature_name”: “feature_0”,
   “importance”: 2.0, // sum(abs()) of class importances
   “foo”: 1.0,
   “bar”: 0.5,
   “baz”: -0.5
},
```

This change adjusts the mapping creation for analytics so that the field is mapped as a `nested` type.

Native side change: https://github.com/elastic/ml-cpp/pull/1071
---
 .../ml/dataframe/analyses/Classification.java |  7 +--
 .../core/ml/dataframe/analyses/MapUtils.java  | 40 +++++++++++++++++
 .../ml/dataframe/analyses/Regression.java     |  7 ++-
 .../analyses/ClassificationTests.java         | 43 +++++++++++--------
 .../dataframe/analyses/RegressionTests.java   |  6 +--
 .../ml/integration/ClassificationIT.java      |  5 ++-
 .../xpack/ml/integration/RegressionIT.java    | 13 +++---
 7 files changed, 88 insertions(+), 33 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index ef0b7e19d100e..a61f91b629750 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -286,9 +286,11 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
     @SuppressWarnings("unchecked")
     @Override
     public Map<String, Object> getExplicitlyMappedFields(Map<String, Object> mappingsProperties, String resultsFieldName) {
+        Map<String, Object> additionalProperties = new HashMap<>();
+        additionalProperties.put(resultsFieldName + ".feature_importance", MapUtils.featureImportanceMapping());
         Object dependentVariableMapping = extractMapping(dependentVariable, mappingsProperties);
         if ((dependentVariableMapping instanceof Map) == false) {
-            return Collections.emptyMap();
+            return additionalProperties;
         }
         Map<String, Object> dependentVariableMappingAsMap = (Map) dependentVariableMapping;
         // If the source field is an alias, fetch the concrete field that the alias points to.
@@ -299,9 +301,8 @@ public Map<String, Object> getExplicitlyMappedFields(Map<String, Object> mapping
         // We may have updated the value of {@code dependentVariableMapping} in the "if" block above.
         // Hence, we need to check the "instanceof" condition again.
         if ((dependentVariableMapping instanceof Map) == false) {
-            return Collections.emptyMap();
+            return additionalProperties;
         }
-        Map<String, Object> additionalProperties = new HashMap<>();
         additionalProperties.put(resultsFieldName + "." + predictionFieldName, dependentVariableMapping);
         additionalProperties.put(resultsFieldName + ".top_classes.class_name", dependentVariableMapping);
         return additionalProperties;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java
new file mode 100644
index 0000000000000..5440bc850c258
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *//*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+final class MapUtils {
+
+    private static final Map<String, Object> FEATURE_IMPORTANCE_MAPPING;
+    static {
+        Map<String, Object> featureImportanceMappingProperties = new HashMap<>();
+        featureImportanceMappingProperties.put("feature_name", Collections.singletonMap("type", KeywordFieldMapper.CONTENT_TYPE));
+        featureImportanceMappingProperties.put("importance",
+            Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
+        Map<String, Object> featureImportanceMapping = new HashMap<>();
+        // TODO sorted indices don't support nested types
+        //featureImportanceMapping.put("dynamic", true);
+        //featureImportanceMapping.put("type", ObjectMapper.NESTED_CONTENT_TYPE);
+        featureImportanceMapping.put("properties", featureImportanceMappingProperties);
+        FEATURE_IMPORTANCE_MAPPING = Collections.unmodifiableMap(featureImportanceMapping);
+    }
+
+    static Map<String, Object> featureImportanceMapping() {
+        return FEATURE_IMPORTANCE_MAPPING;
+    }
+
+    private MapUtils() {}
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
index d8c490ddcb804..0d40ef6ba35df 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -187,9 +188,13 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
 
     @Override
     public Map<String, Object> getExplicitlyMappedFields(Map<String, Object> mappingsProperties, String resultsFieldName) {
+        Map<String, Object> additionalProperties = new HashMap<>();
+        additionalProperties.put(resultsFieldName + ".feature_importance", MapUtils.featureImportanceMapping());
         // Prediction field should be always mapped as "double" rather than "float" in order to increase precision in case of
         // high (over 10M) values of dependent variable.
-        return Collections.singletonMap(resultsFieldName + "." + predictionFieldName, Collections.singletonMap("type", "double"));
+        additionalProperties.put(resultsFieldName + "." + predictionFieldName,
+            Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
+        return additionalProperties;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 96961706aa4ca..2185260c01f99 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -26,7 +26,6 @@
 import java.util.Set;
 
 import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -244,31 +243,37 @@ public void testFieldCardinalityLimitsIsNonEmpty() {
     }
 
     public void testGetExplicitlyMappedFields() {
-        assertThat(new Classification("foo").getExplicitlyMappedFields(null, "results"), is(anEmptyMap()));
-        assertThat(new Classification("foo").getExplicitlyMappedFields(Collections.emptyMap(), "results"), is(anEmptyMap()));
+        assertThat(new Classification("foo").getExplicitlyMappedFields(null, "results"),
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
+        assertThat(new Classification("foo").getExplicitlyMappedFields(Collections.emptyMap(), "results"),
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
         assertThat(
             new Classification("foo").getExplicitlyMappedFields(Collections.singletonMap("foo", "not_a_map"), "results"),
-            is(anEmptyMap()));
-        assertThat(
-            new Classification("foo").getExplicitlyMappedFields(
-                Collections.singletonMap("foo", Collections.singletonMap("bar", "baz")),
-                "results"),
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
+        Map<String, Object> explicitlyMappedFields = new Classification("foo").getExplicitlyMappedFields(
+            Collections.singletonMap("foo", Collections.singletonMap("bar", "baz")),
+            "results");
+        assertThat(explicitlyMappedFields,
             allOf(
                 hasEntry("results.foo_prediction", Collections.singletonMap("bar", "baz")),
                 hasEntry("results.top_classes.class_name", Collections.singletonMap("bar", "baz"))));
-        assertThat(
-            new Classification("foo").getExplicitlyMappedFields(
-                new HashMap<>() {{
-                    put("foo", new HashMap<>() {{
-                        put("type", "alias");
-                        put("path", "bar");
-                    }});
-                    put("bar", Collections.singletonMap("type", "long"));
-                }},
-                "results"),
+        assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", MapUtils.featureImportanceMapping()));
+
+        explicitlyMappedFields = new Classification("foo").getExplicitlyMappedFields(
+            new HashMap<>() {{
+                put("foo", new HashMap<>() {{
+                    put("type", "alias");
+                    put("path", "bar");
+                }});
+                put("bar", Collections.singletonMap("type", "long"));
+            }},
+            "results");
+        assertThat(explicitlyMappedFields,
             allOf(
                 hasEntry("results.foo_prediction", Collections.singletonMap("type", "long")),
                 hasEntry("results.top_classes.class_name", Collections.singletonMap("type", "long"))));
+        assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", MapUtils.featureImportanceMapping()));
+
         assertThat(
             new Classification("foo").getExplicitlyMappedFields(
                 Collections.singletonMap("foo", new HashMap<>() {{
@@ -276,7 +281,7 @@ public void testGetExplicitlyMappedFields() {
                     put("path", "missing");
                 }}),
                 "results"),
-            is(anEmptyMap()));
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
     }
 
     public void testToXContent_GivenVersionBeforeRandomizeSeedWasIntroduced() throws IOException {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
index d171a2844f0d5..4a1b14e4db676 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
@@ -143,9 +143,9 @@ public void testFieldCardinalityLimitsIsEmpty() {
     }
 
     public void testGetExplicitlyMappedFields() {
-        assertThat(
-            new Regression("foo").getExplicitlyMappedFields(null, "results"),
-            hasEntry("results.foo_prediction", Collections.singletonMap("type", "double")));
+        Map<String, Object> explicitlyMappedFields = new Regression("foo").getExplicitlyMappedFields(null, "results");
+        assertThat(explicitlyMappedFields, hasEntry("results.foo_prediction", Collections.singletonMap("type", "double")));
+        assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", MapUtils.featureImportanceMapping()));
     }
 
     public void testGetStateDocId() {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index b1e3fa32e965b..9c4511527eeba 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -76,7 +76,6 @@ public void cleanup() {
         cleanUp();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
     public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws Exception {
         initialize("classification_single_numeric_feature_and_mixed_data_set");
         String predictedClassField = KEYWORD_FIELD + "_prediction";
@@ -108,7 +107,9 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
             assertThat(getFieldValue(resultsObject, predictedClassField), is(in(KEYWORD_FIELD_VALUES)));
             assertThat(getFieldValue(resultsObject, "is_training"), is(destDoc.containsKey(KEYWORD_FIELD)));
             assertTopClasses(resultsObject, 2, KEYWORD_FIELD, KEYWORD_FIELD_VALUES);
-            assertThat(resultsObject.keySet().stream().filter(k -> k.startsWith("feature_importance.")).findAny().isPresent(), is(true));
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> importanceArray = (List<Map<String, Object>>)resultsObject.get("feature_importance");
+            assertThat(importanceArray, hasSize(greaterThan(0)));
         }
 
         assertProgress(jobId, 100, 100, 100, 100);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index 5fc6d618da9ab..fc20bbb5afd7b 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -25,9 +25,11 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.test.hamcrest.OptionalMatchers.isPresent;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
 public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
@@ -48,7 +50,6 @@ public void cleanup() {
         cleanUp();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
     public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws Exception {
         initialize("regression_single_numeric_feature_and_mixed_data_set");
         String predictedClassField = DEPENDENT_VARIABLE_FIELD + "_prediction";
@@ -86,11 +87,13 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
             assertThat(resultsObject.containsKey(predictedClassField), is(true));
             assertThat(resultsObject.containsKey("is_training"), is(true));
             assertThat(resultsObject.get("is_training"), is(destDoc.containsKey(DEPENDENT_VARIABLE_FIELD)));
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> importanceArray = (List<Map<String, Object>>)resultsObject.get("feature_importance");
+            assertThat(importanceArray, hasSize(greaterThan(0)));
             assertThat(
-                resultsObject.toString(),
-                resultsObject.containsKey("feature_importance." + NUMERICAL_FEATURE_FIELD)
-                    || resultsObject.containsKey("feature_importance." + DISCRETE_NUMERICAL_FEATURE_FIELD),
-                is(true));
+                importanceArray.stream().filter(m -> NUMERICAL_FEATURE_FIELD.equals(m.get("feature_name"))
+                    || DISCRETE_NUMERICAL_FEATURE_FIELD.equals(m.get("feature_name"))).findAny(),
+                isPresent());
         }
 
         assertProgress(jobId, 100, 100, 100, 100);

From cdc377e8e74d3ca6c231c36dc5e80621aab47c69 Mon Sep 17 00:00:00 2001
From: Marios Trivyzas <matriv@gmail.com>
Date: Mon, 23 Mar 2020 19:49:32 +0100
Subject: [PATCH 229/285] Reduce performance impact of ExitableDirectoryReader
 (#53978)

Benchmarking showed that the effect of the ExitableDirectoryReader
is reduced considerably when checking every 8191 docs. Moreover,
set the cancellable task before calling QueryPhase#preProcess()
and make sure we don't wrap with an ExitableDirectoryReader at all
when lowLevelCancellation is set to false to avoid completely any
performance impact.

Follows: #52822
Follows: #53166
Follows: #53496
---
 .../search/DefaultSearchContext.java          | 11 ++++------
 .../elasticsearch/search/SearchService.java   | 16 ++++++--------
 .../search/internal/ContextIndexSearcher.java | 21 ++++++++----------
 .../internal/ExitableDirectoryReader.java     |  2 +-
 .../search/query/QueryPhase.java              |  8 +++----
 .../search/DefaultSearchContextTests.java     |  8 +++----
 .../search/SearchCancellationTests.java       | 14 ++++++------
 .../search/SearchServiceTests.java            | 22 +++++++++----------
 .../internal/ContextIndexSearcherTests.java   |  2 +-
 .../profile/query/QueryProfilerTests.java     |  4 ++--
 .../search/query/QueryPhaseTests.java         |  6 ++---
 .../aggregations/AggregatorTestCase.java      |  2 +-
 ...ityIndexReaderWrapperIntegrationTests.java | 10 +++++----
 13 files changed, 60 insertions(+), 66 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index bed38deb28a29..a4cdd540e3503 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -98,6 +98,7 @@ final class DefaultSearchContext extends SearchContext {
     private final QuerySearchResult queryResult;
     private final FetchSearchResult fetchResult;
     private final float queryBoost;
+    private final boolean lowLevelCancellation;
     private TimeValue timeout;
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
@@ -118,7 +119,6 @@ final class DefaultSearchContext extends SearchContext {
     private int trackTotalHitsUpTo = SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO;
     private FieldDoc searchAfter;
     private CollapseContext collapse;
-    private boolean lowLevelCancellation;
     // filter for sliced scroll
     private SliceBuilder sliceBuilder;
     private SearchShardTask task;
@@ -156,7 +156,7 @@ final class DefaultSearchContext extends SearchContext {
     DefaultSearchContext(SearchContextId id, ShardSearchRequest request, SearchShardTarget shardTarget,
                          Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService,
                          IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout,
-                         FetchPhase fetchPhase) throws IOException {
+                         FetchPhase fetchPhase, boolean lowLevelCancellation) throws IOException {
         this.id = id;
         this.request = request;
         this.fetchPhase = fetchPhase;
@@ -172,12 +172,13 @@ final class DefaultSearchContext extends SearchContext {
         this.indexService = indexService;
         this.clusterService = clusterService;
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
-            engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy());
+            engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation);
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
         queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher,
             request::nowInMillis, shardTarget.getClusterAlias());
         queryBoost = request.indexBoost();
+        this.lowLevelCancellation = lowLevelCancellation;
     }
 
     @Override
@@ -563,10 +564,6 @@ public boolean lowLevelCancellation() {
         return lowLevelCancellation;
     }
 
-    public void lowLevelCancellation(boolean lowLevelCancellation) {
-        this.lowLevelCancellation = lowLevelCancellation;
-    }
-
     @Override
     public FieldDoc searchAfter() {
         return searchAfter;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 451c906500449..9821f8f212d8c 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -339,10 +339,9 @@ public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, Ac
     }
 
     private DfsSearchResult executeDfsPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws IOException {
-        final SearchContext context = createAndPutContext(rewriteContext);
+        final SearchContext context = createAndPutContext(rewriteContext, task);
         context.incRef();
         try {
-            context.setTask(task);
             contextProcessing(context);
             dfsPhase.execute(context);
             contextProcessedSuccessfully(context);
@@ -422,11 +421,10 @@ private <T> void runAsync(SearchContextId contextId, Supplier<T> executable, Act
     }
 
     private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws Exception {
-        final SearchContext context = createAndPutContext(rewriteContext);
+        final SearchContext context = createAndPutContext(rewriteContext, task);
         final ShardSearchRequest request = rewriteContext.request;
         context.incRef();
         try {
-            context.setTask(task);
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 contextProcessing(context);
@@ -626,8 +624,8 @@ private SearchContext findContext(SearchContextId contextId, TransportRequest re
         }
     }
 
-    final SearchContext createAndPutContext(SearchRewriteContext rewriteContext) throws IOException {
-        SearchContext context = createContext(rewriteContext);
+    final SearchContext createAndPutContext(SearchRewriteContext rewriteContext, SearchShardTask task) throws IOException {
+        SearchContext context = createContext(rewriteContext, task);
         onNewContext(context);
         boolean success = false;
         try {
@@ -660,7 +658,7 @@ private void onNewContext(SearchContext context) {
         }
     }
 
-    final SearchContext createContext(SearchRewriteContext rewriteContext) throws IOException {
+    final SearchContext createContext(SearchRewriteContext rewriteContext, SearchShardTask searchTask) throws IOException {
         final DefaultSearchContext context = createSearchContext(rewriteContext, defaultSearchTimeout);
         try {
             final ShardSearchRequest request = rewriteContext.request;
@@ -684,6 +682,7 @@ final SearchContext createContext(SearchRewriteContext rewriteContext) throws IO
             if (context.size() == -1) {
                 context.size(DEFAULT_SIZE);
             }
+            context.setTask(searchTask);
 
             // pre process
             dfsPhase.preProcess(context);
@@ -696,7 +695,6 @@ final SearchContext createContext(SearchRewriteContext rewriteContext) throws IO
                 keepAlive = request.scroll().keepAlive().millis();
             }
             contextScrollKeepAlive(context, keepAlive);
-            context.lowLevelCancellation(lowLevelCancellation);
         } catch (Exception e) {
             context.close();
             throw e;
@@ -731,7 +729,7 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon
             DefaultSearchContext searchContext = new DefaultSearchContext(
                 new SearchContextId(readerId, idGenerator.incrementAndGet()),
                 request, shardTarget, searcher, clusterService, indexService, indexShard, bigArrays,
-                threadPool::relativeTimeInMillis, timeout, fetchPhase);
+                threadPool::relativeTimeInMillis, timeout, fetchPhase, lowLevelCancellation);
             success = true;
             return searchContext;
         } finally {
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index d3c4f5641dc23..ed41939b1cb9c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -82,23 +82,20 @@ public class ContextIndexSearcher extends IndexSearcher {
     private MutableQueryTimeout cancellable;
 
     public ContextIndexSearcher(IndexReader reader, Similarity similarity,
-                                QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException {
-        this(reader, similarity, queryCache, queryCachingPolicy, new MutableQueryTimeout());
+                                QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
+                                boolean wrapWithExitableDirectoryReader) throws IOException {
+        this(reader, similarity, queryCache, queryCachingPolicy, new MutableQueryTimeout(), wrapWithExitableDirectoryReader);
     }
 
-    // TODO: Make the 2nd constructor private so that the IndexReader is always wrapped.
-    // Some issues must be fixed:
-    //   - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and
-    //     the ShardSearcher sub-searchers.
-    //   - tests that use a MultiReader
-    public ContextIndexSearcher(IndexReader reader, Similarity similarity,
-                                QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
-                                MutableQueryTimeout cancellable) throws IOException {
-        super(cancellable != null ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader);
+    private ContextIndexSearcher(IndexReader reader, Similarity similarity,
+                                 QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
+                                 MutableQueryTimeout cancellable,
+                                 boolean wrapWithExitableDirectoryReader) throws IOException {
+        super(wrapWithExitableDirectoryReader ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader);
         setSimilarity(similarity);
         setQueryCache(queryCache);
         setQueryCachingPolicy(queryCachingPolicy);
-        this.cancellable = cancellable != null ? cancellable : new MutableQueryTimeout();
+        this.cancellable = cancellable;
     }
 
     public void setProfiler(QueryProfiler profiler) {
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
index 51a94f4023b65..0a2dd476cf3b1 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
@@ -245,7 +245,7 @@ public int getDocCount() {
 
     private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor {
 
-        private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 10) - 1; // 1023
+        private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 13) - 1; // 8191
 
         private final PointValues.IntersectVisitor in;
         private final QueryCancellation queryCancellation;
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 104dfdbee21a6..f1c416ca3b632 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -112,9 +112,9 @@ public QueryPhase() {
     public void preProcess(SearchContext context) {
         final Runnable cancellation;
         if (context.lowLevelCancellation()) {
-            SearchShardTask task = context.getTask();
             cancellation = context.searcher().addQueryCancellation(() -> {
-                if (task.isCancelled()) {
+                SearchShardTask task = context.getTask();
+                if (task != null && task.isCancelled()) {
                     throw new TaskCancelledException("cancelled");
                 }
             });
@@ -282,9 +282,9 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe
             }
 
             if (searchContext.lowLevelCancellation()) {
-                SearchShardTask task = searchContext.getTask();
                 searcher.addQueryCancellation(() -> {
-                    if (task.isCancelled()) {
+                    SearchShardTask task = searchContext.getTask();
+                    if (task != null && task.isCancelled()) {
                         throw new TaskCancelledException("cancelled");
                     }
                 });
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 0a68c54e75b47..75dec54b0b4fe 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -121,7 +121,7 @@ public void testPreProcess() throws Exception {
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
             DefaultSearchContext context1 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 1L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null, false);
             context1.from(300);
 
             // resultWindow greater than maxResultWindow and scrollContext is null
@@ -162,7 +162,7 @@ public void testPreProcess() throws Exception {
 
             // rescore is null but sliceBuilder is not null
             DefaultSearchContext context2 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 2L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null, false);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
             int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100);
@@ -179,7 +179,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
             DefaultSearchContext context3 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 3L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null, false);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
             assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query()));
@@ -189,7 +189,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
             DefaultSearchContext context4 = new DefaultSearchContext(new SearchContextId(UUIDs.randomBase64UUID(), 4L),
-                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null);
+                shardSearchRequest, target, searcher, null, indexService, indexShard, bigArrays, null, timeout, null, false);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
             context4.sliceBuilder(new SliceBuilder(0,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
index 3aafdcc575230..1ea831eb7750d 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
@@ -86,8 +86,8 @@ public static void cleanup() throws IOException {
     }
 
     public void testAddingCancellationActions() throws IOException {
-        ContextIndexSearcher searcher = new ContextIndexSearcher(reader,
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
         NullPointerException npe = expectThrows(NullPointerException.class, () -> searcher.addQueryCancellation(null));
         assertEquals("cancellation runnable should not be null", npe.getMessage());
 
@@ -100,8 +100,8 @@ public void testAddingCancellationActions() throws IOException {
     public void testCancellableCollector() throws IOException {
         TotalHitCountCollector collector1 = new TotalHitCountCollector();
         Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); };
-        ContextIndexSearcher searcher = new ContextIndexSearcher(reader,
-            IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
         searcher.search(new MatchAllDocsQuery(), collector1);
         assertThat(collector1.getTotalHits(), equalTo(reader.numDocs()));
@@ -116,14 +116,14 @@ public void testCancellableCollector() throws IOException {
         assertThat(collector2.getTotalHits(), equalTo(reader.numDocs()));
     }
 
-    public void testCancellableDirectoryReader() throws IOException {
+    public void testExitableDirectoryReader() throws IOException {
         AtomicBoolean cancelled = new AtomicBoolean(true);
         Runnable cancellation = () -> {
             if (cancelled.get()) {
                 throw new TaskCancelledException("cancelled");
         }};
-        ContextIndexSearcher searcher = new ContextIndexSearcher(reader,
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
         searcher.addQueryCancellation(cancellation);
         CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton());
 
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index e1e6003695d3a..7c5d573ce226c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -368,7 +368,7 @@ public void testTimeout() throws IOException {
             new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
                 new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
             indexShard);
-        final SearchContext contextWithDefaultTimeout = service.createContext(rewriteContext);
+        final SearchContext contextWithDefaultTimeout = service.createContext(rewriteContext, null);
         try {
             // the search context should inherit the default timeout
             assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
@@ -383,7 +383,7 @@ public void testTimeout() throws IOException {
             new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
                 new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
             indexShard);
-        final SearchContext context = service.createContext(rewriteContext);
+        final SearchContext context = service.createContext(rewriteContext, null);
         try {
             // the search context should inherit the query timeout
             assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
@@ -417,7 +417,7 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
 
         {
             SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
-            try (SearchContext context = service.createContext(rewriteContext)) {
+            try (SearchContext context = service.createContext(rewriteContext, null)) {
                 assertNotNull(context);
             }
         }
@@ -426,7 +426,7 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
             SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
             searchSourceBuilder.docValueField("one_field_too_much");
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> service.createContext(rewriteContext));
+                () -> service.createContext(rewriteContext, null));
             assertEquals(
                 "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [100] but was [101]. "
                     + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.", ex.getMessage());
@@ -458,7 +458,7 @@ public void testMaxScriptFieldsSearch() throws IOException {
 
         {
             SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
-            try (SearchContext context = service.createContext(rewriteContext)) {
+            try (SearchContext context = service.createContext(rewriteContext, null)) {
                 assertNotNull(context);
             }
         }
@@ -468,7 +468,7 @@ public void testMaxScriptFieldsSearch() throws IOException {
                 new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
             SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                    () -> service.createContext(rewriteContext));
+                    () -> service.createContext(rewriteContext, null));
             assertEquals(
                 "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
                     + (maxScriptFields + 1)
@@ -494,7 +494,7 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
             new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
                 new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
             indexShard);
-        try (SearchContext context = service.createContext(rewriteContext)) {
+        try (SearchContext context = service.createContext(rewriteContext, null)) {
             assertEquals(0, context.scriptFields().fields().size());
         }
     }
@@ -531,7 +531,7 @@ public void testMaxOpenScrollContexts() throws RuntimeException, IOException {
         SearchService.SearchRewriteContext rewriteContext =
             service.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutContext(rewriteContext));
+            () -> service.createAndPutContext(rewriteContext, null));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -557,7 +557,7 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                         SearchService.SearchRewriteContext rewriteContext =
                             searchService.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
                         try {
-                            searchService.createAndPutContext(rewriteContext);
+                            searchService.createAndPutContext(rewriteContext, null);
                         } catch (ElasticsearchException e) {
                             assertThat(e.getMessage(), equalTo(
                                 "Trying to create too many scroll contexts. Must be less than or equal to: " +
@@ -835,7 +835,7 @@ public SearchType searchType() {
                 }
             }, indexShard);
         NullPointerException e = expectThrows(NullPointerException.class,
-            () -> service.createContext(rewriteContext));
+            () -> service.createContext(rewriteContext, null));
         assertEquals("expected", e.getMessage());
         assertEquals("should have 2 store refs (IndexService + InternalEngine)", 2, indexService.getShard(0).store().refCount());
     }
@@ -988,7 +988,7 @@ OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
         int numContexts = randomIntBetween(1, 10);
         for (int i = 0; i < numContexts; i++) {
             SearchService.SearchRewriteContext rewriteContext = searchService.acquireSearcherAndRewrite(shardSearchRequest, indexShard);
-            final SearchContext searchContext = searchService.createContext(rewriteContext);
+            final SearchContext searchContext = searchService.createContext(rewriteContext, null);
             assertThat(searchContext.id().getId(), equalTo((long) (i + 1)));
             searchService.putContext(searchContext);
             contextIds.add(searchContext.id());
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
index 61a75a0716fd2..b2c3a4e85455b 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
@@ -240,7 +240,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
         DocumentSubsetDirectoryReader filteredReader = new DocumentSubsetDirectoryReader(reader, cache, roleQuery);
 
         ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
         // Assert wrapping
         assertEquals(ExitableDirectoryReader.class, searcher.getIndexReader().getClass());
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index f05684930a9c5..96b2dc57a86f5 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -43,8 +43,8 @@
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.profile.ProfileResult;
 import org.elasticsearch.test.ESTestCase;
@@ -83,7 +83,7 @@ public static void setup() throws IOException {
         reader = w.getReader();
         w.close();
         searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), ALWAYS_CACHE_POLICY);
+            IndexSearcher.getDefaultQueryCache(), ALWAYS_CACHE_POLICY, true);
     }
 
     @After
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 489285748efcf..c7c3fd856d587 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -912,12 +912,12 @@ public boolean lowLevelCancellation() {
 
     private static ContextIndexSearcher newContextSearcher(IndexReader reader) throws IOException {
         return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
     }
 
     private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size) throws IOException {
         return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) {
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true) {
 
             @Override
             public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
@@ -930,7 +930,7 @@ public void search(List<LeafReaderContext> leaves, Weight weight, Collector coll
     // used to check that numeric long or date sort optimization was run
     private static ContextIndexSearcher newOptimizedContextSearcher(IndexReader reader, int queryType) throws IOException {
         return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) {
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true) {
 
             @Override
             public void search(List<LeafReaderContext> leaves, Weight weight, CollectorManager manager,
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 40678a58d6e92..287935b035011 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -246,7 +246,7 @@ public boolean shouldCache(Query query) {
             }
         };
         ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(),
-            indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, null);
+            indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false);
 
         SearchContext searchContext = mock(SearchContext.class);
         when(searchContext.numberOfShards()).thenReturn(1);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
index a66be9616600c..422e5dd0128ba 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
@@ -161,8 +161,9 @@ protected IndicesAccessControl getIndicesAccessControl() {
             when(queryShardContext.toQuery(new TermsQueryBuilder("field", values[i]))).thenReturn(parsedQuery);
 
             DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
-            IndexSearcher indexSearcher = new ContextIndexSearcher(wrappedDirectoryReader,
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+            IndexSearcher indexSearcher = new ContextIndexSearcher(
+                    wrappedDirectoryReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                    IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
             int expectedHitCount = valuesHitCount[i];
             logger.info("Going to verify hit count with query [{}] with expected total hits [{}]", parsedQuery.query(), expectedHitCount);
@@ -270,8 +271,9 @@ protected IndicesAccessControl getIndicesAccessControl() {
 
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
         DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
-        IndexSearcher indexSearcher = new ContextIndexSearcher(wrappedDirectoryReader,
-            IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        IndexSearcher indexSearcher = new ContextIndexSearcher(
+                wrappedDirectoryReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
         ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs;
         Set<Integer> actualDocIds = new HashSet<>();

From 4af4eb99e18fdaadac879b1223e986227dd2ee71 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 14:49:42 -0400
Subject: [PATCH 230/285] Introduce aarch64 Docker image (#53936)

This commit introduces the infrastructure needed to build a Docker image
for aarch64.
---
 .../aarch64-docker-build-context/build.gradle | 11 +++
 .../docker/aarch64-docker-export/build.gradle |  2 +
 .../build.gradle                              | 11 +++
 .../aarch64-oss-docker-export/build.gradle    |  2 +
 distribution/docker/build.gradle              | 78 ++++++++++++-------
 .../docker/docker-build-context/build.gradle  |  2 +-
 .../oss-docker-build-context/build.gradle     |  2 +-
 distribution/docker/src/docker/Dockerfile     |  8 +-
 .../src/docker/config/tini-arm64.sha512       |  1 +
 9 files changed, 85 insertions(+), 32 deletions(-)
 create mode 100644 distribution/docker/aarch64-docker-build-context/build.gradle
 create mode 100644 distribution/docker/aarch64-docker-export/build.gradle
 create mode 100644 distribution/docker/aarch64-oss-docker-build-context/build.gradle
 create mode 100644 distribution/docker/aarch64-oss-docker-export/build.gradle
 create mode 100644 distribution/docker/src/docker/config/tini-arm64.sha512

diff --git a/distribution/docker/aarch64-docker-build-context/build.gradle b/distribution/docker/aarch64-docker-build-context/build.gradle
new file mode 100644
index 0000000000000..19b0bc3646c60
--- /dev/null
+++ b/distribution/docker/aarch64-docker-build-context/build.gradle
@@ -0,0 +1,11 @@
+apply plugin: 'base'
+
+task buildDockerBuildContext(type: Tar) {
+  extension = 'tar.gz'
+  compression = Compression.GZIP
+  archiveClassifier = "docker-build-context"
+  archiveBaseName = "elasticsearch-aarch64"
+  with dockerBuildContext("aarch64", false, false)
+}
+
+assemble.dependsOn buildDockerBuildContext
diff --git a/distribution/docker/aarch64-docker-export/build.gradle b/distribution/docker/aarch64-docker-export/build.gradle
new file mode 100644
index 0000000000000..537b5a093683e
--- /dev/null
+++ b/distribution/docker/aarch64-docker-export/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.
diff --git a/distribution/docker/aarch64-oss-docker-build-context/build.gradle b/distribution/docker/aarch64-oss-docker-build-context/build.gradle
new file mode 100644
index 0000000000000..bea7d156803fd
--- /dev/null
+++ b/distribution/docker/aarch64-oss-docker-build-context/build.gradle
@@ -0,0 +1,11 @@
+apply plugin: 'base'
+
+task buildOssDockerBuildContext(type: Tar) {
+  extension = 'tar.gz'
+  compression = Compression.GZIP
+  archiveClassifier = "docker-build-context"
+  archiveBaseName = "elasticsearch-aarch64-oss"
+  with dockerBuildContext("aarch64", true, false)
+}
+
+assemble.dependsOn buildOssDockerBuildContext
diff --git a/distribution/docker/aarch64-oss-docker-export/build.gradle b/distribution/docker/aarch64-oss-docker-export/build.gradle
new file mode 100644
index 0000000000000..537b5a093683e
--- /dev/null
+++ b/distribution/docker/aarch64-oss-docker-export/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index c33f74501da95..b9ccfe95ac401 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -1,3 +1,4 @@
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
@@ -13,38 +14,51 @@ testFixtures.useFixture()
 
 configurations {
   dockerPlugins
+  aarch64DockerSource
   dockerSource
+  aarch64OssDockerSource
   ossDockerSource
 }
 
 dependencies {
+  aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar")
   dockerSource project(path: ":distribution:archives:linux-tar")
+  aarch64OssDockerSource project(path: ":distribution:archives:oss-linux-aarch64-tar")
   ossDockerSource project(path: ":distribution:archives:oss-linux-tar")
 }
 
-ext.expansions = { oss, local ->
-  final String classifier = 'linux-x86_64'
+ext.expansions = { architecture, oss, local ->
+  switch (architecture) {
+    case "aarch64":
+    case "x64":
+      break;
+    default:
+      throw new IllegalArgumentException("unrecongized architecture [" + architecture + "], must be one of (aarch64|x64)")
+  }
+  final String classifier = "aarch64".equals(architecture) ? "linux-aarch64" : "linux-x86_64"
   final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz"
   return [
+    'base_image'          : "aarch64".equals(architecture) ? "arm64v8/centos:7" : "centos:7",
     'build_date'          : BuildParams.buildDate,
     'elasticsearch'       : elasticsearch,
     'git_revision'        : BuildParams.gitRevision,
     'license'             : oss ? 'Apache-2.0' : 'Elastic-License',
     'source_elasticsearch': local ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
+    'tini_suffix'         : "aarch64".equals(architecture) ? "-arm64" : "",
     'version'             : VersionProperties.elasticsearch
   ]
 }
 
-private static String buildPath(final boolean oss) {
-  return "build/${oss ? 'oss-' : ''}docker"
+private static String buildPath(final String architecture, final boolean oss) {
+  return "build/${"aarch64".equals(architecture) ? 'aarch64-' : ''}${oss ? 'oss-' : ''}docker"
 }
 
-private static String taskName(final String prefix, final boolean oss, final String suffix) {
-  return "${prefix}${oss ? 'Oss' : ''}${suffix}"
+private static String taskName(final String prefix, final String architecture, final boolean oss, final String suffix) {
+  return "${prefix}${"aarch64".equals(architecture) ? 'Aarch64' : ''}${oss ? 'Oss' : ''}${suffix}"
 }
 
 project.ext {
-  dockerBuildContext = { boolean oss, boolean local ->
+  dockerBuildContext = { String architecture, boolean oss, boolean local ->
     copySpec {
       into('bin') {
         from project.projectDir.toPath().resolve("src/docker/bin")
@@ -62,25 +76,33 @@ project.ext {
       }
 
       from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
-        expand(expansions(oss, local))
+        expand(expansions(architecture, oss, local))
       }
     }
   }
 }
 
-void addCopyDockerContextTask(final boolean oss) {
-  task(taskName("copy", oss, "DockerContext"), type: Sync) {
-    expansions(oss, true).findAll { it.key != 'build_date' }.each { k, v ->
+void addCopyDockerContextTask(final String architecture, final boolean oss) {
+  task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) {
+    expansions(architecture, oss, true).findAll { it.key != 'build_date' }.each { k, v ->
       inputs.property(k, { v.toString() })
     }
-    into buildPath(oss)
+    into buildPath(architecture, oss)
 
-    with dockerBuildContext(oss, true)
+    with dockerBuildContext(architecture, oss, true)
 
-    if (oss) {
-      from configurations.ossDockerSource
+    if ("aarch64".equals(architecture)) {
+      if (oss) {
+        from configurations.aarch64OssDockerSource
+      } else {
+        from configurations.aarch64DockerSource
+      }
     } else {
-      from configurations.dockerSource
+      if (oss) {
+        from configurations.ossDockerSource
+      } else {
+        from configurations.dockerSource
+      }
     }
 
     from configurations.dockerPlugins
@@ -149,9 +171,9 @@ task integTest(type: Test) {
 
 check.dependsOn integTest
 
-void addBuildDockerImage(final boolean oss) {
-  final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: DockerBuildTask) {
-    TaskProvider<Sync> copyContextTask = tasks.named(taskName("copy", oss, "DockerContext"))
+void addBuildDockerImage(final String architecture, final boolean oss) {
+  final Task buildDockerImageTask = task(taskName("build", architecture, oss, "DockerImage"), type: DockerBuildTask) {
+    TaskProvider<Sync> copyContextTask = tasks.named(taskName("copy", architecture, oss, "DockerContext"))
     dependsOn(copyContextTask)
     dockerContext.fileProvider(copyContextTask.map { it.destinationDir })
 
@@ -170,11 +192,14 @@ void addBuildDockerImage(final boolean oss) {
     }
   }
   assemble.dependsOn(buildDockerImageTask)
+  buildDockerImageTask.enabled = Architecture.current().name().toLowerCase().equals(architecture)
 }
 
-for (final boolean oss : [false, true]) {
-  addCopyDockerContextTask(oss)
-  addBuildDockerImage(oss)
+for (final String architecture : ["aarch64", "x64"]) {
+  for (final boolean oss : [false, true]) {
+    addCopyDockerContextTask(architecture, oss)
+    addBuildDockerImage(architecture, oss)
+  }
 }
 
 // We build the images used in compose locally, but the pull command insists on using a repository
@@ -192,11 +217,12 @@ subprojects { Project subProject ->
   if (subProject.name.contains('docker-export')) {
     apply plugin: 'distribution'
 
+    final String architecture = subProject.name.contains('aarch64-') ? 'aarch64' : 'x64'
     final boolean oss = subProject.name.contains('oss-')
 
-    def exportTaskName = taskName("export", oss, "DockerImage")
-    def buildTaskName = taskName("build", oss, "DockerImage")
-    def tarFile = "${parent.projectDir}/build/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
+    def exportTaskName = taskName("export", architecture, oss, "DockerImage")
+    def buildTaskName = taskName("build", architecture, oss, "DockerImage")
+    def tarFile = "${parent.projectDir}/build/elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
 
     final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) {
       inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker")
@@ -212,7 +238,7 @@ subprojects { Project subProject ->
 
     artifacts.add('default', file(tarFile)) {
       type 'tar'
-      name "elasticsearch${oss ? '-oss' : ''}"
+      name "elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}"
       builtBy exportTaskName
     }
 
diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle
index 50be407e566bc..2dd28329d7ba5 100644
--- a/distribution/docker/docker-build-context/build.gradle
+++ b/distribution/docker/docker-build-context/build.gradle
@@ -5,7 +5,7 @@ task buildDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch"
-  with dockerBuildContext(false, false)
+  with dockerBuildContext("x64", false, false)
 }
 
 assemble.dependsOn buildDockerBuildContext
diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle
index b69f7dc620f53..0a29c2a2b7274 100644
--- a/distribution/docker/oss-docker-build-context/build.gradle
+++ b/distribution/docker/oss-docker-build-context/build.gradle
@@ -5,7 +5,7 @@ task buildOssDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch-oss"
-  with dockerBuildContext(true, false)
+  with dockerBuildContext("x64", true, false)
 }
 
 assemble.dependsOn buildOssDockerBuildContext
diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
index b12a3c7518265..7eb62b7d8ad1b 100644
--- a/distribution/docker/src/docker/Dockerfile
+++ b/distribution/docker/src/docker/Dockerfile
@@ -11,7 +11,7 @@
 # Set gid=0 and make group perms==owner perms
 ################################################################################
 
-FROM centos:7 AS builder
+FROM ${base_image} AS builder
 
 RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
     yum install --setopt=tsflags=nodocs -y gzip shadow-utils tar && \
@@ -42,8 +42,8 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 # gpg, but the keyservers are slow to return the key and this can fail the
 # build. Instead, we check the binary against a checksum that we have
 # computed.
-ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /tini
-COPY config/tini.sha512 /tini.sha512
+ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini${tini_suffix} /tini
+COPY config/tini${tini_suffix}.sha512 /tini.sha512
 RUN sha512sum -c /tini.sha512 && chmod +x /tini
 
 ################################################################################
@@ -52,7 +52,7 @@ RUN sha512sum -c /tini.sha512 && chmod +x /tini
 # Add entrypoint
 ################################################################################
 
-FROM centos:7
+FROM ${base_image}
 
 ENV ELASTIC_CONTAINER true
 
diff --git a/distribution/docker/src/docker/config/tini-arm64.sha512 b/distribution/docker/src/docker/config/tini-arm64.sha512
new file mode 100644
index 0000000000000..274eaa28cff08
--- /dev/null
+++ b/distribution/docker/src/docker/config/tini-arm64.sha512
@@ -0,0 +1 @@
+6ae5147e522e484b9d59b0caa04e6dadf0efe332b272039c7cf5951e39f5028e9852c3c4bcdd46b98977329108d555ee7ea55f9eca99765d05922ec7aff837d8  /tini

From 0d7b4ee3748e969dc10a8fa94fcefb23874cb3f6 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 15:04:25 -0400
Subject: [PATCH 231/285] Fix typo in jdk-download testKit build.gradle

This commit fixes a typo in the jdk-download testKit build.gradle file
where "architecture" was not spelled correctly.
---
 buildSrc/src/testKit/jdk-download/reuse/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildSrc/src/testKit/jdk-download/reuse/build.gradle b/buildSrc/src/testKit/jdk-download/reuse/build.gradle
index c8b08171091b1..39a5c3372d5f5 100644
--- a/buildSrc/src/testKit/jdk-download/reuse/build.gradle
+++ b/buildSrc/src/testKit/jdk-download/reuse/build.gradle
@@ -7,6 +7,6 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
-    architeecture = "x64"
+    architecture = "x64"
   }
 }

From 91462ee306eec09d0556056ea040b5dfe079fe5e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 15:06:47 -0400
Subject: [PATCH 232/285] Fix aarch64 OSS archive packaging

This commit fixes the OSS aarch64 packaging to use the aarch64 JDK.
---
 distribution/archives/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 17e9cd36d0212..ee274b93c270e 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -189,7 +189,7 @@ task buildLinuxTar(type: SymbolicLinkPreservingTar) {
 task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'aarch64', true, true)
 }
 
 task buildOssLinuxTar(type: SymbolicLinkPreservingTar) {

From 7161bd44cb91c301e2f886ebee7e22c7fabad1d0 Mon Sep 17 00:00:00 2001
From: Julie Tibshirani <julie.tibshirani@elastic.co>
Date: Mon, 23 Mar 2020 12:26:22 -0700
Subject: [PATCH 233/285] Remove the top-level 'mapping type' section. (#53374)

It seemed confusing for users that our top-level mapping page still had a
prominent section named 'Mapping Type'. This PR reworks the docs to remove this
reference and adds a note about types removal (similar to the note we added to
other APIs like put mapping).
---
 docs/reference/mapping.asciidoc               | 19 ++++++-------------
 docs/reference/mapping/fields.asciidoc        |  2 +-
 .../mapping/fields/type-field.asciidoc        |  6 +++---
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 3f89571ae5808..96153f5e2a355 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -13,29 +13,22 @@ are stored and indexed. For instance, use mappings to define:
 * custom rules to control the mapping for
   <<dynamic-mapping,dynamically added fields>>.
 
-[float]
-[[mapping-type]]
-== Mapping Type
-
-Each index has one _mapping type_ which determines how the document will be
-indexed.
-
-deprecated::[6.0.0,See <<removal-of-types>>]
-
-A mapping type has:
+A mapping definition has:
 
 <<mapping-fields,Meta-fields>>::
 
 Meta-fields are used to customize how a document's metadata associated is
 treated. Examples of meta-fields include the document's
-<<mapping-index-field,`_index`>>, <<mapping-type-field,`_type`>>,
-<<mapping-id-field,`_id`>>,  and <<mapping-source-field,`_source`>> fields.
+<<mapping-index-field,`_index`>>, <<mapping-id-field,`_id`>>, and
+<<mapping-source-field,`_source`>> fields.
 
 <<mapping-types,Fields>> or _properties_::
 
-A mapping type contains a list of fields or `properties` pertinent to the
+A mapping contains a list of fields or `properties` pertinent to the
 document.
 
+NOTE: Before 7.0.0, the 'mappings' definition used to include a type name.
+For more details, please see <<removal-of-types>>.
 
 [float]
 [[field-datatypes]]
diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc
index f6d5f00a9b5e0..0ea4b77441c42 100644
--- a/docs/reference/mapping/fields.asciidoc
+++ b/docs/reference/mapping/fields.asciidoc
@@ -15,7 +15,7 @@ can be customised when a mapping type is created.
 
 <<mapping-type-field,`_type`>>::
 
-    The document's <<mapping-type,mapping type>>.
+    The document's mapping type.
 
 <<mapping-id-field,`_id`>>::
 
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index 044729b84eece..4e6126cde9c9d 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -3,9 +3,9 @@
 
 deprecated[6.0.0,See <<removal-of-types>>]
 
-Each document indexed is associated with a <<mapping-type-field,`_type`>> (see
-<<mapping-type>>) and an <<mapping-id-field,`_id`>>.  The `_type` field is
-indexed in order to make searching by type name fast.
+Each document indexed is associated with a <<mapping-type-field,`_type`>> and
+an <<mapping-id-field,`_id`>>.  The `_type` field is indexed in order to make
+searching by type name fast.
 
 The value of the `_type` field is accessible in queries, aggregations,
 scripts, and when sorting:

From e98b9c07804bcee6b371a43060db3560ad896e85 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Mon, 23 Mar 2020 15:31:38 -0400
Subject: [PATCH 234/285] Add scripting, unmapped, supported-type tests to
 Boxplot (#53435)

---
 .../boxplot/BoxplotAggregatorTests.java       | 127 ++++++++++++++++++
 1 file changed, 127 insertions(+)

diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java
index 31a532d5314be..d8af20e311628 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java
@@ -19,9 +19,16 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -30,15 +37,51 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.equalTo;
 
 public class BoxplotAggregatorTests extends AggregatorTestCase {
 
+    /** Script to return the {@code _value} provided by aggs framework. */
+    public static final String VALUE_SCRIPT = "_value";
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new BoxplotAggregationBuilder("foo").field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.HISTOGRAM);
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+        scripts.put(VALUE_SCRIPT, vars -> ((Number) vars.get("_value")).doubleValue() + 1);
+
+        MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME,
+            scripts,
+            Collections.emptyMap());
+        Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
+
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
+
+
     public void testNoMatchingField() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
@@ -122,6 +165,28 @@ public void testSomeMatchesNumericDocValues() throws IOException {
         });
     }
 
+    public void testMissingField() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot").field("number").missing(10L);
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("other", 2)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 2)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 3)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 4)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 5)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 0)));
+        },  (Consumer<InternalBoxplot>) boxplot -> {
+            assertEquals(0, boxplot.getMin(), 0);
+            assertEquals(10, boxplot.getMax(), 0);
+            assertEquals(10, boxplot.getQ1(), 0);
+            assertEquals(10, boxplot.getQ2(), 0);
+            assertEquals(10, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
     public void testUnmappedWithMissingField() throws IOException {
         BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
             .field("does_not_exist").missing(0L);
@@ -291,6 +356,68 @@ public void testGetProperty() throws IOException {
         }, fieldType);
     }
 
+    public void testValueScript() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
+            .field("number")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, (Consumer<InternalBoxplot>) boxplot -> {
+            assertEquals(2, boxplot.getMin(), 0);
+            assertEquals(8, boxplot.getMax(), 0);
+            assertEquals(2, boxplot.getQ1(), 0);
+            assertEquals(5, boxplot.getQ2(), 0);
+            assertEquals(8, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
+    public void testValueScriptUnmapped() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
+            .field("does_not_exist")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, (Consumer<InternalBoxplot>) boxplot -> {
+            assertEquals(Double.POSITIVE_INFINITY, boxplot.getMin(), 0);
+            assertEquals(Double.NEGATIVE_INFINITY, boxplot.getMax(), 0);
+            assertEquals(Double.NaN, boxplot.getQ1(), 0);
+            assertEquals(Double.NaN, boxplot.getQ2(), 0);
+            assertEquals(Double.NaN, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
+    public void testValueScriptUnmappedMissing() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
+            .field("does_not_exist")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()))
+            .missing(1.0);
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, (Consumer<InternalBoxplot>) boxplot -> {
+            // Note: the way scripts, missing and unmapped interact, these will be the missing value and the script is not invoked
+            assertEquals(1.0, boxplot.getMin(), 0);
+            assertEquals(1.0, boxplot.getMax(), 0);
+            assertEquals(1.0, boxplot.getQ1(), 0);
+            assertEquals(1.0, boxplot.getQ2(), 0);
+            assertEquals(1.0, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
     private void testCase(Query query,
                           CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
                           Consumer<InternalBoxplot> verify) throws IOException {

From da9273ab7be558ff99ddc5caf86bb66895b386aa Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Mon, 23 Mar 2020 20:32:19 +0100
Subject: [PATCH 235/285] =?UTF-8?q?[Transform][Rollup]=20add=20processing?=
 =?UTF-8?q?=20stats=20to=20record=20the=20time=20sp=E2=80=A6=20(#53770)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

add 2 additional stats: processing time and processing total which capture the
time spent for processing results and how often it ran. The 2 new stats
correspond to the existing indexing and search stats. Together with indexing
and search this now allows the user to see the full picture, all 3 stages.
---
 .../client/core/IndexerJobStats.java          |  33 ++++-
 .../client/rollup/GetRollupJobResponse.java   |  10 +-
 .../transforms/TransformIndexerStats.java     | 119 ++++++++++++++----
 .../org/elasticsearch/client/TransformIT.java |   2 +
 .../rollup/GetRollupJobResponseTests.java     |   7 +-
 .../TransformIndexerStatsTests.java           | 118 ++++++++++++-----
 .../hlrc/TransformIndexerStatsTests.java      |  36 ++++--
 .../transforms/hlrc/TransformStatsTests.java  |  70 ++++++-----
 docs/reference/rollup/apis/get-job.asciidoc   |  14 ++-
 .../core/indexing/AsyncTwoPhaseIndexer.java   |   7 +-
 .../xpack/core/indexing/IndexerJobStats.java  |  42 ++++++-
 .../rollup/job/RollupIndexerJobStats.java     |  17 ++-
 .../transforms/TransformIndexerStats.java     |  86 ++++++++-----
 .../TransformInternalIndexConstants.java      |   2 +-
 .../job/JobWrapperSerializingTests.java       |   3 +-
 .../job/RollupIndexerJobStatsTests.java       |   3 +-
 .../TransformIndexerStatsTests.java           | 109 +++++++++++++---
 .../transforms/TransformStatsTests.java       |  30 ++++-
 .../xpack/rollup/job/IndexerUtilsTests.java   |  12 +-
 .../rest-api-spec/test/rollup/delete_job.yml  |   6 +
 .../rest-api-spec/test/rollup/get_jobs.yml    |   4 +
 .../rest-api-spec/test/rollup/put_job.yml     |   2 +
 .../integration/TransformUsageIT.java         |   5 -
 .../TransformInfoTransportAction.java         |  18 +--
 .../persistence/TransformInternalIndex.java   |   7 ++
 .../TransformInfoTransportActionTests.java    |  16 +--
 .../80_transform_jobs_crud.yml                |   6 +-
 27 files changed, 583 insertions(+), 201 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
index 5e59b4b19dbbe..dc332fa8a4ab0 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
@@ -31,8 +31,10 @@ public abstract class IndexerJobStats {
     public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
     public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
     public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
     public static ParseField INDEX_TOTAL = new ParseField("index_total");
     public static ParseField SEARCH_TOTAL = new ParseField("search_total");
+    public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
     public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     public static ParseField INDEX_FAILURES = new ParseField("index_failures");
 
@@ -44,11 +46,14 @@ public abstract class IndexerJobStats {
     protected final long indexTotal;
     protected final long searchTime;
     protected final long searchTotal;
+    protected final long processingTime;
+    protected final long processingTotal;
     protected final long indexFailures;
     protected final long searchFailures;
 
     public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations,
-                           long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) {
+                           long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal, long processingTotal,
+                           long indexFailures, long searchFailures) {
         this.numPages = numPages;
         this.numInputDocuments = numInputDocuments;
         this.numOuputDocuments = numOutputDocuments;
@@ -57,6 +62,8 @@ public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocu
         this.indexTotal = indexTotal;
         this.searchTime = searchTime;
         this.searchTotal = searchTotal;
+        this.processingTime = processingTime;
+        this.processingTotal = processingTotal;
         this.indexFailures = indexFailures;
         this.searchFailures = searchFailures;
     }
@@ -117,6 +124,13 @@ public long getSearchTime() {
         return searchTime;
     }
 
+    /**
+     * Returns the time spent processing (cumulative) in milliseconds
+     */
+    public long getProcessingTime() {
+        return processingTime;
+    }
+
     /**
      * Returns the total number of indexing requests that have been processed
      * (Note: this is not the number of _documents_ that have been indexed)
@@ -132,6 +146,14 @@ public long getSearchTotal() {
         return searchTotal;
     }
 
+    /**
+     * Returns the total number of processing runs that have been made
+     */
+    public long getProcessingTotal() {
+        return processingTotal;
+    }
+
+
     @Override
     public boolean equals(Object other) {
         if (this == other) {
@@ -149,16 +171,19 @@ public boolean equals(Object other) {
                 && Objects.equals(this.numInvocations, that.numInvocations)
                 && Objects.equals(this.indexTime, that.indexTime)
                 && Objects.equals(this.searchTime, that.searchTime)
+                && Objects.equals(this.processingTime, that.processingTime)
                 && Objects.equals(this.indexFailures, that.indexFailures)
                 && Objects.equals(this.searchFailures, that.searchFailures)
                 && Objects.equals(this.searchTotal, that.searchTotal)
+                && Objects.equals(this.processingTotal, that.processingTotal)
                 && Objects.equals(this.indexTotal, that.indexTotal);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-                indexTime, searchTime, indexFailures, searchFailures, searchTotal, indexTotal);
+                indexTime, searchTime, processingTime, indexFailures, searchFailures, searchTotal,
+                indexTotal, processingTotal);
     }
 
     @Override
@@ -172,6 +197,8 @@ public final String toString() {
                 + ", index_time_in_ms=" + indexTime
                 + ", index_total=" + indexTotal
                 + ", search_time_in_ms=" + searchTime
-                + ", search_total=" + searchTotal+ "}";
+                + ", search_total=" + searchTotal
+                + ", processing_time_in_ms=" + processingTime
+                + ", processing_total=" + processingTotal + "}";
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java
index e63daf5949002..9ded34aa05670 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java
@@ -177,16 +177,18 @@ public final String toString() {
     public static class RollupIndexerJobStats extends IndexerJobStats {
 
         RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
-                              long indexTime, long indexTotal, long searchTime, long searchTotal, long indexFailures, long searchFailures) {
+                              long indexTime, long indexTotal, long searchTime, long searchTotal, long processingTime,
+                              long processingTotal, long indexFailures, long searchFailures) {
             super(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-                    indexTime, searchTime, indexTotal, searchTotal, indexFailures, searchFailures);
+                    indexTime, searchTime, processingTime, indexTotal, searchTotal, processingTotal, indexFailures, searchFailures);
         }
 
         private static final ConstructingObjectParser<RollupIndexerJobStats, Void> PARSER = new ConstructingObjectParser<>(
                 STATS.getPreferredName(),
                 true,
                 args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3],
-                    (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9]));
+                    (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
+                    (long) args[10], (long) args[11]));
         static {
             PARSER.declareLong(constructorArg(), NUM_PAGES);
             PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
@@ -196,6 +198,8 @@ public static class RollupIndexerJobStats extends IndexerJobStats {
             PARSER.declareLong(constructorArg(), INDEX_TOTAL);
             PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
             PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
+            PARSER.declareLong(constructorArg(), PROCESSING_TIME_IN_MS);
+            PARSER.declareLong(constructorArg(), PROCESSING_TOTAL);
             PARSER.declareLong(constructorArg(), INDEX_FAILURES);
             PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
         }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
index 2a04c6ea45eb5..e3a0032e55b0d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
@@ -27,7 +27,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class TransformIndexerStats extends IndexerJobStats {
@@ -39,21 +38,38 @@ public class TransformIndexerStats extends IndexerJobStats {
     public static final ConstructingObjectParser<TransformIndexerStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
         NAME,
         true,
-        args -> new TransformIndexerStats((long) args[0], (long) args[1], (long) args[2],
-            (long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
-            (Double) args[10], (Double) args[11], (Double) args[12]));
+        args -> new TransformIndexerStats(
+            unboxSafe(args[0], 0L),
+            unboxSafe(args[1], 0L),
+            unboxSafe(args[2], 0L),
+            unboxSafe(args[3], 0L),
+            unboxSafe(args[4], 0L),
+            unboxSafe(args[5], 0L),
+            unboxSafe(args[6], 0L),
+            unboxSafe(args[7], 0L),
+            unboxSafe(args[8], 0L),
+            unboxSafe(args[9], 0L),
+            unboxSafe(args[10], 0L),
+            unboxSafe(args[11], 0L),
+            unboxSafe(args[12], 0.0),
+            unboxSafe(args[13], 0.0),
+            unboxSafe(args[14], 0.0)
+        )
+    );
 
     static {
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_FAILURES);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_INDEXED);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED);
@@ -67,16 +83,40 @@ public static TransformIndexerStats fromXContent(XContentParser parser) throws I
     private final double expAvgDocumentsIndexed;
     private final double expAvgDocumentsProcessed;
 
-    public TransformIndexerStats(long numPages, long numInputDocuments, long numOuputDocuments,
-                                 long numInvocations, long indexTime, long searchTime,
-                                 long indexTotal, long searchTotal, long indexFailures, long searchFailures,
-                                 Double expAvgCheckpointDurationMs, Double expAvgDocumentsIndexed,
-                                 Double expAvgDocumentsProcessed) {
-        super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime,
-                indexTotal, searchTotal, indexFailures, searchFailures);
-        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
-        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
-        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
+    public TransformIndexerStats(
+        long numPages,
+        long numInputDocuments,
+        long numOuputDocuments,
+        long numInvocations,
+        long indexTime,
+        long searchTime,
+        long processingTime,
+        long indexTotal,
+        long searchTotal,
+        long processingTotal,
+        long indexFailures,
+        long searchFailures,
+        double expAvgCheckpointDurationMs,
+        double expAvgDocumentsIndexed,
+        double expAvgDocumentsProcessed
+    ) {
+        super(
+            numPages,
+            numInputDocuments,
+            numOuputDocuments,
+            numInvocations,
+            indexTime,
+            searchTime,
+            processingTime,
+            indexTotal,
+            searchTotal,
+            processingTotal,
+            indexFailures,
+            searchFailures
+        );
+        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs;
+        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed;
+        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed;
     }
 
     public double getExpAvgCheckpointDurationMs() {
@@ -109,10 +149,12 @@ public boolean equals(Object other) {
             && Objects.equals(this.numInvocations, that.numInvocations)
             && Objects.equals(this.indexTime, that.indexTime)
             && Objects.equals(this.searchTime, that.searchTime)
+            && Objects.equals(this.processingTime, that.processingTime)
             && Objects.equals(this.indexFailures, that.indexFailures)
             && Objects.equals(this.searchFailures, that.searchFailures)
             && Objects.equals(this.indexTotal, that.indexTotal)
             && Objects.equals(this.searchTotal, that.searchTotal)
+            && Objects.equals(this.processingTotal, that.processingTotal)
             && Objects.equals(this.expAvgCheckpointDurationMs, that.expAvgCheckpointDurationMs)
             && Objects.equals(this.expAvgDocumentsIndexed, that.expAvgDocumentsIndexed)
             && Objects.equals(this.expAvgDocumentsProcessed, that.expAvgDocumentsProcessed);
@@ -120,8 +162,31 @@ public boolean equals(Object other) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-            indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal,
-            expAvgCheckpointDurationMs, expAvgDocumentsIndexed, expAvgDocumentsProcessed);
+        return Objects.hash(
+            numPages,
+            numInputDocuments,
+            numOuputDocuments,
+            numInvocations,
+            indexTime,
+            searchTime,
+            processingTime,
+            indexFailures,
+            searchFailures,
+            indexTotal,
+            searchTotal,
+            processingTotal,
+            expAvgCheckpointDurationMs,
+            expAvgDocumentsIndexed,
+            expAvgDocumentsProcessed
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T unboxSafe(Object l, T default_value) {
+        if (l == null) {
+            return default_value;
+        } else {
+            return (T) l;
+        }
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
index 3f93806aca779..94341c41685f0 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
@@ -441,6 +441,8 @@ public void testGetStats() throws Exception {
             0L,
             0L,
             0L,
+            0L,
+            0L,
             0.0,
             0.0,
             0.0);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java
index b866420a44c01..122f156986d71 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java
@@ -64,8 +64,9 @@ private GetRollupJobResponse createTestInstance() {
 
     private RollupIndexerJobStats randomStats() {
         return new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomNonNegativeLong());
     }
 
     private RollupJobStatus randomStatus() {
@@ -120,6 +121,8 @@ public void toXContent(RollupIndexerJobStats stats, XContentBuilder builder, ToX
         builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
         builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
         builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+        builder.field(IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+        builder.field(IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
         builder.endObject();
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
index 018cab89b0fc9..e06a7cddb93e9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
@@ -31,41 +31,103 @@ public class TransformIndexerStatsTests extends ESTestCase {
 
     public void testFromXContent() throws IOException {
         xContentTester(
-                this::createParser,
-                TransformIndexerStatsTests::randomStats,
-                TransformIndexerStatsTests::toXContent,
-                TransformIndexerStats::fromXContent)
-                .supportsUnknownFields(true)
-                .test();
+            this::createParser,
+            TransformIndexerStatsTests::randomStats,
+            TransformIndexerStatsTests::toXContent,
+            TransformIndexerStats::fromXContent
+        ).supportsUnknownFields(true).test();
     }
 
     public static TransformIndexerStats randomStats() {
-        return new TransformIndexerStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble());
+        return new TransformIndexerStats(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
     }
 
     public static void toXContent(TransformIndexerStats stats, XContentBuilder builder) throws IOException {
         builder.startObject();
-        builder.field(IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
-        builder.field(IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
-        builder.field(IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
-        builder.field(IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
-        builder.field(IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
-        builder.field(IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
-        builder.field(IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
-        builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
-        builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
-        builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
-        builder.field(TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
-            stats.getExpAvgCheckpointDurationMs());
-        builder.field(TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
-            stats.getExpAvgDocumentsIndexed());
-        builder.field(TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
-            stats.getExpAvgDocumentsProcessed());
+        if (randomBoolean()) {
+            builder.field(IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
+            builder.field(IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
+            builder.field(IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
+            builder.field(IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
+            builder.field(IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
+            builder.field(IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
+            builder.field(IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
+            builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
+            builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
+            builder.field(IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+            builder.field(IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
+            builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+            builder.field(
+                TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
+                stats.getExpAvgCheckpointDurationMs()
+            );
+            builder.field(TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(), stats.getExpAvgDocumentsIndexed());
+            builder.field(
+                TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
+                stats.getExpAvgDocumentsProcessed()
+            );
+        } else {
+            // a toXContent version which leaves out field with value 0 (simulating the case that an older version misses a field)
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
+            xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
+            xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
+            xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
+            xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
+            xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
+            xContentFieldIfNotZero(builder, IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+            xContentFieldIfNotZero(builder, IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
+            xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+            xContentFieldIfNotZero(
+                builder,
+                TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
+                stats.getExpAvgCheckpointDurationMs()
+            );
+            xContentFieldIfNotZero(
+                builder,
+                TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
+                stats.getExpAvgDocumentsIndexed()
+            );
+            xContentFieldIfNotZero(
+                builder,
+                TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
+                stats.getExpAvgDocumentsProcessed()
+            );
+        }
         builder.endObject();
     }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, long value) throws IOException {
+        if (value > 0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, double value) throws IOException {
+        if (value > 0.0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java
index 50c98bcd8f0b6..eb74164be5364 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java
@@ -30,17 +30,31 @@
 
 public class TransformIndexerStatsTests extends AbstractResponseTestCase<
     org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats,
-        TransformIndexerStats> {
+    TransformIndexerStats> {
+
+    public static org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats randomStats() {
+        return new org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats(
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
+    }
 
     @Override
     protected org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats createServerTestInstance(XContentType xContentType) {
-        return new org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats(randomLongBetween(10L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble());
+        return randomStats();
     }
 
     @Override
@@ -49,8 +63,10 @@ protected TransformIndexerStats doParseToClientInstance(XContentParser parser) t
     }
 
     @Override
-    protected void assertInstances(org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
-                                   TransformIndexerStats clientInstance) {
+    protected void assertInstances(
+        org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
+        TransformIndexerStats clientInstance
+    ) {
         assertTransformIndexerStats(serverTestInstance, clientInstance);
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
index 63f6bffdf6321..c732c425fe427 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
@@ -36,40 +36,36 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class TransformStatsTests extends AbstractResponseTestCase<org.elasticsearch.xpack.core.transform.transforms.TransformStats,
+public class TransformStatsTests extends AbstractResponseTestCase<
+    org.elasticsearch.xpack.core.transform.transforms.TransformStats,
     org.elasticsearch.client.transform.transforms.TransformStats> {
 
     public static org.elasticsearch.xpack.core.transform.transforms.NodeAttributes randomNodeAttributes() {
         int numberOfAttributes = randomIntBetween(1, 10);
         Map<String, String> attributes = new HashMap<>(numberOfAttributes);
-        for(int i = 0; i < numberOfAttributes; i++) {
+        for (int i = 0; i < numberOfAttributes; i++) {
             String val = randomAlphaOfLength(10);
-            attributes.put("key-"+i, val);
+            attributes.put("key-" + i, val);
         }
-        return new org.elasticsearch.xpack.core.transform.transforms.NodeAttributes(randomAlphaOfLength(10),
+        return new org.elasticsearch.xpack.core.transform.transforms.NodeAttributes(
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
-            attributes);
+            randomAlphaOfLength(10),
+            attributes
+        );
     }
 
-    public static org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats randomStats() {
-        return new org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats(randomLongBetween(10L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble());
-    }
     @Override
     protected org.elasticsearch.xpack.core.transform.transforms.TransformStats createServerTestInstance(XContentType xContentType) {
-        return new org.elasticsearch.xpack.core.transform.transforms.TransformStats(randomAlphaOfLength(10),
+        return new org.elasticsearch.xpack.core.transform.transforms.TransformStats(
+            randomAlphaOfLength(10),
             randomFrom(org.elasticsearch.xpack.core.transform.transforms.TransformStats.State.values()),
             randomBoolean() ? null : randomAlphaOfLength(100),
             randomBoolean() ? null : randomNodeAttributes(),
-            randomStats(),
-            TransformCheckpointingInfoTests.randomTransformCheckpointingInfo());
+            TransformIndexerStatsTests.randomStats(),
+            TransformCheckpointingInfoTests.randomTransformCheckpointingInfo()
+        );
     }
 
     @Override
@@ -78,8 +74,10 @@ protected TransformStats doParseToClientInstance(XContentParser parser) throws I
     }
 
     @Override
-    protected void assertInstances(org.elasticsearch.xpack.core.transform.transforms.TransformStats serverTestInstance,
-                                   TransformStats clientInstance) {
+    protected void assertInstances(
+        org.elasticsearch.xpack.core.transform.transforms.TransformStats serverTestInstance,
+        TransformStats clientInstance
+    ) {
         assertThat(serverTestInstance.getId(), equalTo(clientInstance.getId()));
         assertThat(serverTestInstance.getState().value(), equalTo(clientInstance.getState().value()));
         assertTransformIndexerStats(serverTestInstance.getIndexerStats(), clientInstance.getIndexerStats());
@@ -88,8 +86,10 @@ protected void assertInstances(org.elasticsearch.xpack.core.transform.transforms
         assertThat(serverTestInstance.getReason(), equalTo(clientInstance.getReason()));
     }
 
-    private void assertNodeAttributes(org.elasticsearch.xpack.core.transform.transforms.NodeAttributes serverTestInstance,
-                                      NodeAttributes clientInstance) {
+    private void assertNodeAttributes(
+        org.elasticsearch.xpack.core.transform.transforms.NodeAttributes serverTestInstance,
+        NodeAttributes clientInstance
+    ) {
         if (serverTestInstance == null || clientInstance == null) {
             assertNull(serverTestInstance);
             assertNull(clientInstance);
@@ -102,8 +102,10 @@ private void assertNodeAttributes(org.elasticsearch.xpack.core.transform.transfo
         assertThat(serverTestInstance.getTransportAddress(), equalTo(clientInstance.getTransportAddress()));
     }
 
-    public static void assertTransformProgress(org.elasticsearch.xpack.core.transform.transforms.TransformProgress serverTestInstance,
-                                         TransformProgress clientInstance) {
+    public static void assertTransformProgress(
+        org.elasticsearch.xpack.core.transform.transforms.TransformProgress serverTestInstance,
+        TransformProgress clientInstance
+    ) {
         if (serverTestInstance == null || clientInstance == null) {
             assertNull(serverTestInstance);
             assertNull(clientInstance);
@@ -115,16 +117,18 @@ public static void assertTransformProgress(org.elasticsearch.xpack.core.transfor
         assertThat(serverTestInstance.getDocumentsIndexed(), equalTo(clientInstance.getDocumentsIndexed()));
     }
 
-    public static void assertPosition(org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition serverTestInstance,
-                                TransformIndexerPosition clientInstance) {
+    public static void assertPosition(
+        org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition serverTestInstance,
+        TransformIndexerPosition clientInstance
+    ) {
         assertThat(serverTestInstance.getIndexerPosition(), equalTo(clientInstance.getIndexerPosition()));
         assertThat(serverTestInstance.getBucketsPosition(), equalTo(clientInstance.getBucketsPosition()));
     }
 
-
     public static void assertTransformCheckpointStats(
-            org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointStats serverTestInstance,
-            TransformCheckpointStats clientInstance) {
+        org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointStats serverTestInstance,
+        TransformCheckpointStats clientInstance
+    ) {
         assertTransformProgress(serverTestInstance.getCheckpointProgress(), clientInstance.getCheckpointProgress());
         assertThat(serverTestInstance.getCheckpoint(), equalTo(clientInstance.getCheckpoint()));
         assertPosition(serverTestInstance.getPosition(), clientInstance.getPosition());
@@ -133,8 +137,9 @@ public static void assertTransformCheckpointStats(
     }
 
     public static void assertTransformCheckpointInfo(
-            org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo serverTestInstance,
-            TransformCheckpointingInfo clientInstance) {
+        org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo serverTestInstance,
+        TransformCheckpointingInfo clientInstance
+    ) {
         assertTransformCheckpointStats(serverTestInstance.getNext(), clientInstance.getNext());
         assertTransformCheckpointStats(serverTestInstance.getLast(), clientInstance.getLast());
         assertThat(serverTestInstance.getChangesLastDetectedAt(), equalTo(clientInstance.getChangesLastDetectedAt()));
@@ -142,8 +147,9 @@ public static void assertTransformCheckpointInfo(
     }
 
     public static void assertTransformIndexerStats(
-            org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
-            TransformIndexerStats clientInstance) {
+        org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
+        TransformIndexerStats clientInstance
+    ) {
         assertThat(serverTestInstance.getExpAvgCheckpointDurationMs(), equalTo(clientInstance.getExpAvgCheckpointDurationMs()));
         assertThat(serverTestInstance.getExpAvgDocumentsProcessed(), equalTo(clientInstance.getExpAvgDocumentsProcessed()));
         assertThat(serverTestInstance.getExpAvgDocumentsIndexed(), equalTo(clientInstance.getExpAvgDocumentsIndexed()));
diff --git a/docs/reference/rollup/apis/get-job.asciidoc b/docs/reference/rollup/apis/get-job.asciidoc
index de45f96549f7f..a13ac77bbb884 100644
--- a/docs/reference/rollup/apis/get-job.asciidoc
+++ b/docs/reference/rollup/apis/get-job.asciidoc
@@ -142,14 +142,16 @@ The API yields the following response:
             "index_total": 0,
             "search_failures": 0,
             "search_time_in_ms": 0,
-            "search_total": 0
+            "search_total": 0,
+            "processing_time_in_ms": 0,
+            "processing_total": 0
           }
         }
     ]
 }
 ----
 
-The `jobs` array contains a single job (`id: sensor`) since we requested a single job in the endpoint's URL. 
+The `jobs` array contains a single job (`id: sensor`) since we requested a single job in the endpoint's URL.
 If we add another job, we can see how multi-job responses are handled:
 
 [source,console]
@@ -245,7 +247,9 @@ Which will yield the following response:
             "index_total": 0,
             "search_failures": 0,
             "search_time_in_ms": 0,
-            "search_total": 0
+            "search_total": 0,
+            "processing_time_in_ms": 0,
+            "processing_total": 0
           }
         },
         {
@@ -299,7 +303,9 @@ Which will yield the following response:
             "index_total": 0,
             "search_failures": 0,
             "search_time_in_ms": 0,
-            "search_total": 0
+            "search_total": 0,
+            "processing_time_in_ms": 0,
+            "processing_total": 0
           }
         }
     ]
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
index 52aa9304ce264..fef4592d75067 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
@@ -347,7 +347,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
 
             // allowPartialSearchResults is set to false, so we should never see shard failures here
             assert (searchResponse.getShardFailures().length == 0);
-
+            stats.markStartProcessing();
             stats.incrementNumPages(1);
             IterationResult<JobPosition> iterationResult = doProcess(searchResponse);
 
@@ -355,11 +355,11 @@ private void onSearchResponse(SearchResponse searchResponse) {
                 logger.debug("Finished indexing for job [" + getJobId() + "], saving state and shutting down.");
 
                 position.set(iterationResult.getPosition());
+                stats.markEndProcessing();
                 // execute finishing tasks
                 onFinish(ActionListener.wrap(
                         r -> doSaveState(finishAndSetState(), position.get(), () -> {}),
                         e -> doSaveState(finishAndSetState(), position.get(), () -> {})));
-
                 return;
             }
 
@@ -369,7 +369,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
             if (docs.isEmpty() == false) {
                 final BulkRequest bulkRequest = new BulkRequest();
                 docs.forEach(bulkRequest::add);
-
+                stats.markEndProcessing();
                 stats.markStartIndexing();
                 doNextBulk(bulkRequest, ActionListener.wrap(bulkResponse -> {
                     // TODO we should check items in the response and move after accordingly to
@@ -390,6 +390,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
                     onBulkResponse(bulkResponse, newPosition);
                 }, this::finishWithIndexingFailure));
             } else {
+                stats.markEndProcessing();
                 // no documents need to be indexed, continue with search
                 try {
                     JobPosition newPosition = iterationResult.getPosition();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
index 72134b0553b98..57aac7f839e4b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.indexing;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -31,26 +32,31 @@ public abstract class IndexerJobStats implements ToXContentObject, Writeable {
     protected long searchTime = 0;
     protected long indexTotal = 0;
     protected long searchTotal = 0;
+    protected long processingTime = 0;
+    protected long processingTotal = 0;
     protected long indexFailures = 0;
     protected long searchFailures = 0;
 
     private long startIndexTime;
     private long startSearchTime;
+    private long startProcessingTime;
 
     public IndexerJobStats() {
     }
 
     public IndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
-                           long indexTime, long searchTime, long indexTotal, long searchTotal,
-                           long indexFailures, long searchFailures) {
+                           long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal,
+                           long processingTotal, long indexFailures, long searchFailures) {
         this.numPages = numPages;
         this.numInputDocuments = numInputDocuments;
         this.numOuputDocuments = numOuputDocuments;
         this.numInvocations = numInvocations;
         this.indexTime = indexTime;
         this.searchTime = searchTime;
+        this.processingTime = processingTime;
         this.indexTotal = indexTotal;
         this.searchTotal = searchTotal;
+        this.processingTotal = processingTotal;
         this.indexFailures = indexFailures;
         this.searchFailures = searchFailures;
     }
@@ -66,6 +72,11 @@ public IndexerJobStats(StreamInput in) throws IOException {
         this.searchTotal = in.readVLong();
         this.indexFailures = in.readVLong();
         this.searchFailures = in.readVLong();
+
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo V_7_7_0
+            this.processingTime = in.readVLong();
+            this.processingTotal = in.readVLong();
+        }
     }
 
     public long getNumPages() {
@@ -100,6 +111,10 @@ public long getSearchTime() {
         return searchTime;
     }
 
+    public long getProcessingTime() {
+        return processingTime;
+    }
+
     public long getIndexTotal() {
         return indexTotal;
     }
@@ -108,6 +123,10 @@ public long getSearchTotal() {
         return searchTotal;
     }
 
+    public long getProcessingTotal() {
+        return processingTotal;
+    }
+
     public void incrementNumPages(long n) {
         assert(n >= 0);
         numPages += n;
@@ -154,6 +173,15 @@ public void markEndSearch() {
         searchTotal += 1;
     }
 
+    public void markStartProcessing() {
+        this.startProcessingTime = System.nanoTime();
+    }
+
+    public void markEndProcessing() {
+        processingTime += ((System.nanoTime() - startProcessingTime) / 1000000);
+        processingTotal += 1;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(numPages);
@@ -166,6 +194,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(searchTotal);
         out.writeVLong(indexFailures);
         out.writeVLong(searchFailures);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeVLong(processingTime);
+            out.writeVLong(processingTotal);
+        }
     }
 
     @Override
@@ -186,15 +218,17 @@ public boolean equals(Object other) {
             && Objects.equals(this.numInvocations, that.numInvocations)
             && Objects.equals(this.indexTime, that.indexTime)
             && Objects.equals(this.searchTime, that.searchTime)
+            && Objects.equals(this.processingTime, that.processingTime)
             && Objects.equals(this.indexFailures, that.indexFailures)
             && Objects.equals(this.searchFailures, that.searchFailures)
             && Objects.equals(this.indexTotal, that.indexTotal)
-            && Objects.equals(this.searchTotal, that.searchTotal);
+            && Objects.equals(this.searchTotal, that.searchTotal)
+            && Objects.equals(this.processingTotal, that.processingTotal);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-            indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal);
+            indexTime, searchTime, processingTime, indexFailures, searchFailures, indexTotal, searchTotal, processingTotal);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java
index f038228ae76a9..2843358ceef44 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java
@@ -27,15 +27,18 @@ public class RollupIndexerJobStats extends IndexerJobStats {
     private static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
     private static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
     private static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    private static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
     private static ParseField INDEX_TOTAL = new ParseField("index_total");
     private static ParseField SEARCH_TOTAL = new ParseField("search_total");
+    private static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
     private static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     private static ParseField INDEX_FAILURES = new ParseField("index_failures");
 
     public static final ConstructingObjectParser<RollupIndexerJobStats, Void> PARSER =
         new ConstructingObjectParser<>(NAME.getPreferredName(),
             args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3],
-                (long) args[4], (long) args[5], (long) args[6], (long) args[7],  (long) args[8], (long) args[9]));
+                (long) args[4], (long) args[5], (long) args[6], (long) args[7],  (long) args[8], (long) args[9],
+                (long) args[10], (long) args[11]));
 
     static {
         PARSER.declareLong(constructorArg(), NUM_PAGES);
@@ -44,8 +47,10 @@ public class RollupIndexerJobStats extends IndexerJobStats {
         PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
         PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
         PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
+        PARSER.declareLong(constructorArg(), PROCESSING_TIME_IN_MS);
         PARSER.declareLong(constructorArg(), INDEX_TOTAL);
         PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
+        PARSER.declareLong(constructorArg(), PROCESSING_TOTAL);
         PARSER.declareLong(constructorArg(), INDEX_FAILURES);
         PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
     }
@@ -55,10 +60,10 @@ public RollupIndexerJobStats() {
     }
 
     public RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
-                                 long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures,
-                                 long searchFailures) {
-        super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime,
-            indexTotal, searchTotal, indexFailures, searchFailures);
+                                 long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal,
+                                 long processingTotal, long indexFailures, long searchFailures) {
+        super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime, processingTime,
+            indexTotal, searchTotal, processingTotal, indexFailures, searchFailures);
     }
 
     public RollupIndexerJobStats(StreamInput in) throws IOException {
@@ -78,6 +83,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(SEARCH_TIME_IN_MS.getPreferredName(), searchTime);
         builder.field(SEARCH_TOTAL.getPreferredName(), searchTotal);
         builder.field(SEARCH_FAILURES.getPreferredName(), searchFailures);
+        builder.field(PROCESSING_TIME_IN_MS.getPreferredName(), processingTime);
+        builder.field(PROCESSING_TOTAL.getPreferredName(), processingTotal);
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
index 452ca90205671..2422f0636fffc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class TransformIndexerStats extends IndexerJobStats {
@@ -33,8 +32,10 @@ public class TransformIndexerStats extends IndexerJobStats {
     public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
     public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
     public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
     public static ParseField INDEX_TOTAL = new ParseField("index_total");
     public static ParseField SEARCH_TOTAL = new ParseField("search_total");
+    public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
     public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     public static ParseField INDEX_FAILURES = new ParseField("index_failures");
     public static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
@@ -50,33 +51,37 @@ public class TransformIndexerStats extends IndexerJobStats {
         NAME,
         true,
         args -> new TransformIndexerStats(
-            (long) args[0],
-            (long) args[1],
-            (long) args[2],
-            (long) args[3],
-            (long) args[4],
-            (long) args[5],
-            (long) args[6],
-            (long) args[7],
-            (long) args[8],
-            (long) args[9],
-            (Double) args[10],
-            (Double) args[11],
-            (Double) args[12]
+            unboxSafe(args[0], 0L),
+            unboxSafe(args[1], 0L),
+            unboxSafe(args[2], 0L),
+            unboxSafe(args[3], 0L),
+            unboxSafe(args[4], 0L),
+            unboxSafe(args[5], 0L),
+            unboxSafe(args[6], 0L),
+            unboxSafe(args[7], 0L),
+            unboxSafe(args[8], 0L),
+            unboxSafe(args[9], 0L),
+            unboxSafe(args[10], 0L),
+            unboxSafe(args[11], 0L),
+            unboxSafe(args[12], 0.0),
+            unboxSafe(args[13], 0.0),
+            unboxSafe(args[14], 0.0)
         )
     );
 
     static {
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_FAILURES);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_INDEXED);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED);
@@ -100,13 +105,15 @@ public TransformIndexerStats(
         long numInvocations,
         long indexTime,
         long searchTime,
+        long processingTime,
         long indexTotal,
         long searchTotal,
+        long processingTotal,
         long indexFailures,
         long searchFailures,
-        Double expAvgCheckpointDurationMs,
-        Double expAvgDocumentsIndexed,
-        Double expAvgDocumentsProcessed
+        double expAvgCheckpointDurationMs,
+        double expAvgDocumentsIndexed,
+        double expAvgDocumentsProcessed
     ) {
         super(
             numPages,
@@ -115,14 +122,16 @@ public TransformIndexerStats(
             numInvocations,
             indexTime,
             searchTime,
+            processingTime,
             indexTotal,
             searchTotal,
+            processingTotal,
             indexFailures,
             searchFailures
         );
-        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
-        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
-        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
+        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs;
+        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed;
+        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed;
     }
 
     public TransformIndexerStats(TransformIndexerStats other) {
@@ -133,8 +142,10 @@ public TransformIndexerStats(TransformIndexerStats other) {
             other.numInvocations,
             other.indexTime,
             other.searchTime,
+            other.processingTime,
             other.indexTotal,
             other.searchTotal,
+            other.processingTotal,
             other.indexFailures,
             other.searchFailures,
             other.expAvgCheckpointDurationMs,
@@ -181,6 +192,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(SEARCH_TIME_IN_MS.getPreferredName(), searchTime);
         builder.field(SEARCH_TOTAL.getPreferredName(), searchTotal);
         builder.field(SEARCH_FAILURES.getPreferredName(), searchFailures);
+        builder.field(PROCESSING_TIME_IN_MS.getPreferredName(), processingTime);
+        builder.field(PROCESSING_TOTAL.getPreferredName(), processingTotal);
         builder.field(EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), this.expAvgCheckpointDurationMs);
         builder.field(EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(), this.expAvgDocumentsIndexed);
         builder.field(EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(), this.expAvgDocumentsProcessed);
@@ -238,10 +251,12 @@ public boolean equals(Object other) {
             && Objects.equals(this.numInvocations, that.numInvocations)
             && Objects.equals(this.indexTime, that.indexTime)
             && Objects.equals(this.searchTime, that.searchTime)
+            && Objects.equals(this.processingTime, that.processingTime)
             && Objects.equals(this.indexFailures, that.indexFailures)
             && Objects.equals(this.searchFailures, that.searchFailures)
             && Objects.equals(this.indexTotal, that.indexTotal)
             && Objects.equals(this.searchTotal, that.searchTotal)
+            && Objects.equals(this.processingTotal, that.processingTotal)
             && Objects.equals(this.expAvgCheckpointDurationMs, that.expAvgCheckpointDurationMs)
             && Objects.equals(this.expAvgDocumentsIndexed, that.expAvgDocumentsIndexed)
             && Objects.equals(this.expAvgDocumentsProcessed, that.expAvgDocumentsProcessed);
@@ -256,10 +271,12 @@ public int hashCode() {
             numInvocations,
             indexTime,
             searchTime,
+            processingTime,
             indexFailures,
             searchFailures,
             indexTotal,
             searchTotal,
+            processingTotal,
             expAvgCheckpointDurationMs,
             expAvgDocumentsIndexed,
             expAvgDocumentsProcessed
@@ -278,4 +295,13 @@ public static TransformIndexerStats fromXContent(XContentParser parser) {
             throw new RuntimeException(e);
         }
     }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T unboxSafe(Object l, T default_value) {
+        if (l == null) {
+            return default_value;
+        } else {
+            return (T) l;
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
index 474943f0e8c26..59a80f6c9596f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
@@ -24,7 +24,7 @@ public final class TransformInternalIndexConstants {
     // internal index
 
     // version is not a rollover pattern, however padded because sort is string based
-    public static final String INDEX_VERSION = "004";
+    public static final String INDEX_VERSION = "005";
     public static final String INDEX_PATTERN = ".transform-internal-";
     public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION;
     public static final String LATEST_INDEX_NAME = LATEST_INDEX_VERSIONED_NAME;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
index 70105b48533bd..94536abfc86c5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
@@ -43,7 +43,8 @@ protected GetRollupJobsAction.JobWrapper createTestInstance() {
         return new GetRollupJobsAction.JobWrapper(ConfigTestHelpers.randomRollupJobConfig(random()),
                 new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(),
                     randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-                    randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()),
+                    randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                    randomNonNegativeLong(), randomNonNegativeLong()),
                 new RollupJobStatus(state, Collections.emptyMap()));
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
index 9cb57c3ecdc60..90d639c024787 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
@@ -29,7 +29,8 @@ protected RollupIndexerJobStats doParseInstance(XContentParser parser) {
     public static RollupIndexerJobStats randomStats() {
         return new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(),
             randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomNonNegativeLong(), randomNonNegativeLong());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
index f83f820e19433..89f026835bd89 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
@@ -7,14 +7,43 @@
 package org.elasticsearch.xpack.core.transform.transforms;
 
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 
+import java.io.IOException;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 
 public class TransformIndexerStatsTests extends AbstractSerializingTestCase<TransformIndexerStats> {
 
+    public static TransformIndexerStats randomStats() {
+        return new TransformIndexerStats(
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
     @Override
     protected TransformIndexerStats createTestInstance() {
         return randomStats();
@@ -30,24 +59,6 @@ protected TransformIndexerStats doParseInstance(XContentParser parser) {
         return TransformIndexerStats.fromXContent(parser);
     }
 
-    public static TransformIndexerStats randomStats() {
-        return new TransformIndexerStats(
-            randomLongBetween(10L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomBoolean() ? randomDouble() : null,
-            randomBoolean() ? randomDouble() : null,
-            randomBoolean() ? randomDouble() : null
-        );
-    }
-
     public void testExpAvgIncrement() {
         TransformIndexerStats stats = new TransformIndexerStats();
 
@@ -67,4 +78,66 @@ public void testExpAvgIncrement() {
         assertThat(stats.getExpAvgDocumentsIndexed(), closeTo(20.54545454, 0.0000001));
         assertThat(stats.getExpAvgDocumentsProcessed(), closeTo(59.0909090, 0.0000001));
     }
+
+    public void testXContentLeniencyForMissingFields() throws IOException {
+        // this is essentially the same test as done in the super class, but with the difference of a custom toXContent method that leaves
+        // out fields if the value is 0, this allow us to test successful parsing if fields are not available, e.g. on older versions
+        xContentTester(this::createParser, this::createTestInstance, TransformIndexerStatsTests::toXContentIfNotZero, this::doParseInstance)
+            .numberOfTestRuns(NUMBER_OF_TEST_RUNS)
+            .supportsUnknownFields(supportsUnknownFields())
+            .shuffleFieldsExceptions(getShuffleFieldsExceptions())
+            .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter())
+            .assertEqualsConsumer(this::assertEqualInstances)
+            .assertToXContentEquivalence(assertToXContentEquivalence())
+            .test();
+    }
+
+    public static void toXContentIfNotZero(TransformIndexerStats stats, XContentBuilder builder) throws IOException {
+        // a toXContent version which leaves out field with value 0
+        builder.startObject();
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+        xContentFieldIfNotZero(
+            builder,
+            TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
+            stats.getExpAvgCheckpointDurationMs()
+        );
+        xContentFieldIfNotZero(
+            builder,
+            TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
+            stats.getExpAvgDocumentsIndexed()
+        );
+        xContentFieldIfNotZero(
+            builder,
+            TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
+            stats.getExpAvgDocumentsProcessed()
+        );
+        builder.endObject();
+    }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, long value) throws IOException {
+        if (value > 0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, double value) throws IOException {
+        if (value > 0.0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
index ee4bd9a47b71b..cbaacf85277a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
@@ -69,7 +69,7 @@ public void testBwcWith73() throws IOException {
                 STARTED,
                 randomBoolean() ? null : randomAlphaOfLength(100),
                 randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(),
-                new TransformIndexerStats(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0.0, 0.0, 0.0),
+                new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 0.0, 0.0, 0.0),
                 new TransformCheckpointingInfo(
                     new TransformCheckpointStats(0, null, null, 10, 100),
                     new TransformCheckpointStats(0, null, null, 100, 1000),
@@ -89,4 +89,32 @@ public void testBwcWith73() throws IOException {
             }
         }
     }
+
+    public void testBwcWith76() throws IOException {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            TransformStats stats = new TransformStats(
+                "bwc-id",
+                STARTED,
+                randomBoolean() ? null : randomAlphaOfLength(100),
+                randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(),
+                new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 13.0, 14.0, 15.0),
+                new TransformCheckpointingInfo(
+                    new TransformCheckpointStats(0, null, null, 10, 100),
+                    new TransformCheckpointStats(0, null, null, 100, 1000),
+                    // changesLastDetectedAt aren't serialized back
+                    100,
+                    null
+                )
+            );
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                output.setVersion(Version.V_7_6_0);
+                stats.writeTo(output);
+                try (StreamInput in = output.bytes().streamInput()) {
+                    in.setVersion(Version.V_7_6_0);
+                    TransformStats statsFromOld = new TransformStats(in);
+                    assertThat(statsFromOld, equalTo(stats));
+                }
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
index 9f8e26d425f5d..556bc3b088065 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
@@ -67,7 +67,7 @@
 public class IndexerUtilsTests extends AggregatorTestCase {
     public void testMissingFields() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "the_histo";
         String valueField = "the_avg";
@@ -131,7 +131,7 @@ public void testMissingFields() throws IOException {
 
     public void testCorrectFields() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "the_histo";
         String valueField = "the_avg";
@@ -199,7 +199,7 @@ public void testCorrectFields() throws IOException {
 
     public void testNumericTerms() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats= new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats= new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String valueField = "the_avg";
 
@@ -255,7 +255,7 @@ public void testNumericTerms() throws IOException {
 
     public void testEmptyCounts() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "ts";
         String valueField = "the_avg";
@@ -444,7 +444,7 @@ public void testNullKeys() {
 
     public void testMissingBuckets() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String metricField = "metric_field";
         String valueField = "value_field";
@@ -517,7 +517,7 @@ public void testMissingBuckets() throws IOException {
 
     public void testTimezone() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "the_histo";
         String valueField = "the_avg";
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
index be7ace82c6887..b95f518158ed6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
@@ -75,6 +75,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
@@ -127,6 +129,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
@@ -179,6 +183,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
index 4ca935d7bf84b..cb81fe483c278 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
@@ -76,6 +76,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 ---
@@ -217,6 +219,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
index 530bf6414bbed..56d73acb6acd1 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
@@ -76,6 +76,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java
index 765feca863262..d339e24fb74ad 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java
@@ -100,11 +100,6 @@ public void testUsage() throws Exception {
             assertEquals(2, XContentMapValues.extractValue("transform.transforms.stopped", statsMap));
             assertEquals(1, XContentMapValues.extractValue("transform.transforms.started", statsMap));
             for (String statName : PROVIDED_STATS) {
-                if (statName.equals(TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName())
-                    || statName.equals(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName())) {
-                    continue;
-                }
-
                 // the trigger count can be higher if the scheduler kicked before usage has been called, therefore check for gte
                 if (statName.equals(TransformIndexerStats.NUM_INVOCATIONS.getPreferredName())) {
                     assertThat(
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
index 3dd12b74c93aa..847f37a5b9d4b 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
@@ -51,8 +51,10 @@ public class TransformInfoTransportAction extends XPackInfoFeatureTransportActio
         TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(),
         TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(),
         TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(),
+        TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(),
         TransformIndexerStats.INDEX_TOTAL.getPreferredName(),
         TransformIndexerStats.SEARCH_TOTAL.getPreferredName(),
+        TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(),
         TransformIndexerStats.INDEX_FAILURES.getPreferredName(),
         TransformIndexerStats.SEARCH_FAILURES.getPreferredName(),
         TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
@@ -105,13 +107,15 @@ static TransformIndexerStats parseSearchAggs(SearchResponse searchResponse) {
             statisticsList.get(3).longValue(),  // numInvocations
             statisticsList.get(4).longValue(),  // indexTime
             statisticsList.get(5).longValue(),  // searchTime
-            statisticsList.get(6).longValue(),  // indexTotal
-            statisticsList.get(7).longValue(),  // searchTotal
-            statisticsList.get(8).longValue(),  // indexFailures
-            statisticsList.get(9).longValue(), // searchFailures
-            statisticsList.get(10), // exponential_avg_checkpoint_duration_ms
-            statisticsList.get(11), // exponential_avg_documents_indexed
-            statisticsList.get(12)  // exponential_avg_documents_processed
+            statisticsList.get(6).longValue(),  // processingTime
+            statisticsList.get(7).longValue(),  // indexTotal
+            statisticsList.get(8).longValue(),  // searchTotal
+            statisticsList.get(9).longValue(),  // processingTotal
+            statisticsList.get(10).longValue(),  // indexFailures
+            statisticsList.get(11).longValue(), // searchFailures
+            statisticsList.get(12), // exponential_avg_checkpoint_duration_ms
+            statisticsList.get(13), // exponential_avg_documents_indexed
+            statisticsList.get(14)  // exponential_avg_documents_processed
         );
     }
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
index 9874dc1f2586a..decd3e6282e63 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
@@ -58,6 +58,7 @@ public final class TransformInternalIndex {
      * version 3 (7.5): rename to .transform-internal-xxx
      * version 4 (7.6): state::should_stop_at_checkpoint
      *                  checkpoint::checkpoint
+     * version 5 (7.7): stats::processing_time_in_ms, stats::processing_total
      */
 
     // constants for mappings
@@ -238,11 +239,17 @@ private static XContentBuilder addTransformStoredDocMappings(XContentBuilder bui
                      .startObject(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName())
                         .field(TYPE, LONG)
                     .endObject()
+                    .startObject(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName())
+                        .field(TYPE, LONG)
+                     .endObject()
                      .startObject(TransformIndexerStats.INDEX_TOTAL.getPreferredName())
                         .field(TYPE, LONG)
                     .endObject()
                      .startObject(TransformIndexerStats.SEARCH_TOTAL.getPreferredName())
                         .field(TYPE, LONG)
+                    .endObject()
+                    .startObject(TransformIndexerStats.PROCESSING_TOTAL.getPreferredName())
+                        .field(TYPE, LONG)
                     .endObject()
                      .startObject(TransformIndexerStats.SEARCH_FAILURES.getPreferredName())
                         .field(TYPE, LONG)
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java
index ceb30cf500792..7018c2d2ae2f1 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java
@@ -98,13 +98,15 @@ public void testParseSearchAggs() {
             4,  // numInvocations
             5,  // indexTime
             6,  // searchTime
-            7,  // indexTotal
-            8,  // searchTotal
-            9,  // indexFailures
-            10, // searchFailures
-            11.0,  // exponential_avg_checkpoint_duration_ms
-            12.0,  // exponential_avg_documents_indexed
-            13.0   // exponential_avg_documents_processed
+            7,  // processingTime
+            8,  // indexTotal
+            9,  // searchTotal
+            10, // processingTotal
+            11, // indexFailures
+            12, // searchFailures
+            13.0,  // exponential_avg_checkpoint_duration_ms
+            14.0,  // exponential_avg_documents_indexed
+            15.0   // exponential_avg_documents_processed
         );
 
         int currentStat = 1;
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
index d220c0b1a51ab..6056a243445cd 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
@@ -279,9 +279,9 @@ setup:
 
   - do:
       indices.get_mapping:
-        index: .transform-internal-004
-  - match: { \.transform-internal-004.mappings.dynamic: "false" }
-  - match: { \.transform-internal-004.mappings.properties.id.type: "keyword" }
+        index: .transform-internal-005
+  - match: { \.transform-internal-005.mappings.dynamic: "false" }
+  - match: { \.transform-internal-005.mappings.properties.id.type: "keyword" }
   - do:
       indices.get_mapping:
         index: .transform-notifications-000002

From ec4c699defbdd5481346ad43f2e8a51728dd8b0a Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Mon, 23 Mar 2020 15:33:17 -0400
Subject: [PATCH 236/285] Prevent SigTerms/SigText from running on fields they
 do not support (#52851)

* Prevent SigTerms/SigText from running on fields they do not support

SigTerms cannot run on fields that are not searchable, and SigText
cannot run on fields that do not have analyzers.  Both of these
situations fail today with an esoteric exception, so this just formalizes
the constraint by throwing an IllegalArgumentException up front.

In practice, the only affected field seems to be the `binary` field,
which is neither searchable or has a default analyzer (e.g. even numeric
and keyword fields have a default analyzer despite not being tokenized)

Adds supported-type tests, and makes some changes to the test itself
to allow testing sigtext (indexing _source).

Also a few tweaks to the test to avoid bad randomization (negative
numbers, etc).

* Tweak exception text
---
 .../index/mapper/NumberFieldMapper.java       |   1 -
 .../SignificantTermsAggregatorFactory.java    |  11 +-
 .../SignificantTextAggregatorFactory.java     |   8 ++
 .../SignificantTermsAggregatorTests.java      |  26 +++++
 .../SignificantTextAggregatorTests.java       |  27 +++++
 .../aggregations/AggregatorTestCase.java      | 103 ++++++++++++++----
 6 files changed, 152 insertions(+), 24 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index fa7055cadd2a9..ace6e6d6f28ec 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.lucene.document.DoublePoint;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatPoint;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 0687c81a64802..3de703099bcb2 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -88,7 +88,12 @@ public SignificantTermsAggregatorFactory(String name,
                                              Map<String, Object> metaData) throws IOException {
         super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData);
 
-        if (!config.unmapped()) {
+        if (config.unmapped() == false) {
+            if (config.fieldContext().fieldType().isSearchable() == false) {
+                throw new IllegalArgumentException("SignificantText aggregation requires fields to be searchable, but ["
+                    + config.fieldContext().fieldType().name() + "] is not");
+            }
+
             this.fieldType = config.fieldContext().fieldType();
             this.indexedFieldName = fieldType.name();
         }
@@ -129,6 +134,10 @@ private FilterableTermsEnum getTermsEnum(String field) throws IOException {
     }
 
     private long getBackgroundFrequency(String value) throws IOException {
+        // fieldType can be null if the field is unmapped, but theoretically this method should only be called
+        // when constructing buckets.  Assert to ensure this is the case
+        // TODO this is a bad setup and it should be refactored
+        assert fieldType != null;
         Query query = fieldType.termQuery(value, queryShardContext);
         if (query instanceof TermQuery) {
             // for types that use the inverted index, we prefer using a caching terms
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
index 4930c213443b8..950bde076908d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
@@ -83,6 +83,10 @@ public SignificantTextAggregatorFactory(String name,
         // Note that if the field is unmapped (its field type is null), we don't fail,
         // and just use the given field name as a placeholder.
         this.fieldType = queryShardContext.fieldMapper(fieldName);
+        if (fieldType != null && fieldType.indexAnalyzer() == null) {
+            throw new IllegalArgumentException("Field [" + fieldType.name() + "] has no analyzer, but SignificantText " +
+                "requires an analyzed field");
+        }
         this.indexedFieldName = fieldType != null ? fieldType.name() : fieldName;
         this.sourceFieldNames = sourceFieldNames == null
             ? new String[] { indexedFieldName }
@@ -124,6 +128,10 @@ private FilterableTermsEnum getTermsEnum(String field) throws IOException {
     }
 
     private long getBackgroundFrequency(String value) throws IOException {
+        // fieldType can be null if the field is unmapped, but theoretically this method should only be called
+        // when constructing buckets.  Assert to ensure this is the case
+        // TODO this is a bad setup and it should be refactored
+        assert fieldType != null;
         Query query = fieldType.termQuery(value, queryShardContext);
         if (query instanceof TermQuery) {
             // for types that use the inverted index, we prefer using a caching terms
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
index 6cb4826ead2c4..aa35aab052b2c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -46,10 +47,14 @@
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorFactory.ExecutionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -75,6 +80,27 @@ public void setUpTest() throws Exception {
         fieldType.setName("field");
     }
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new SignificantTermsAggregationBuilder("foo", ValueType.STRING).field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.BYTES);
+    }
+
+    @Override
+    protected List<String> unsupportedMappedFieldTypes() {
+        return List.of(
+            NumberFieldMapper.NumberType.DOUBLE.typeName(), // floating points are not supported at all
+            NumberFieldMapper.NumberType.FLOAT.typeName(),
+            NumberFieldMapper.NumberType.HALF_FLOAT.typeName(),
+            BinaryFieldMapper.CONTENT_TYPE // binary fields are not supported because they cannot be searched
+        );
+    }
+
     /**
      * For each provided field type, we also register an alias with name {@code <field>-alias}.
      */
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
index 123bb0f60427c..6a354b0a6a496 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
@@ -34,12 +34,17 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.BinaryFieldMapper;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -63,6 +68,28 @@ protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldT
             Function.identity()));
     }
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new SignificantTextAggregationBuilder("foo", fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        // TODO it is likely accidental that SigText supports anything other than Bytes, and then only text fields
+        return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.BYTES,
+            CoreValuesSourceType.RANGE,
+            CoreValuesSourceType.GEOPOINT);
+    }
+
+    @Override
+    protected List<String> unsupportedMappedFieldTypes() {
+        return List.of(
+            BinaryFieldMapper.CONTENT_TYPE, // binary fields are not supported because they do not have analyzers
+            GeoPointFieldMapper.CONTENT_TYPE // geopoint fields cannot use term queries
+        );
+    }
+
     /**
      * Uses the significant text aggregation to find the keywords in text fields
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 287935b035011..6033df8d9fa4e 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -18,12 +18,15 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.AssertingDirectoryReader;
 import org.apache.lucene.index.CompositeReaderContext;
 import org.apache.lucene.index.DirectoryReader;
@@ -49,6 +52,7 @@
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.util.BigArrays;
@@ -56,6 +60,10 @@
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache.Listener;
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
@@ -108,6 +116,7 @@
 import org.junit.After;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -119,7 +128,6 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.singleton;
 import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.anyString;
@@ -142,7 +150,6 @@ public abstract class AggregatorTestCase extends ESTestCase {
         ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects
         GeoShapeFieldMapper.CONTENT_TYPE, // Cannot aggregate geoshapes (yet)
 
-        TextFieldMapper.CONTENT_TYPE, // TODO Does not support doc values, but does support FD, needs a lot of mocking
         ObjectMapper.NESTED_CONTENT_TYPE, // TODO support for nested
         CompletionFieldMapper.CONTENT_TYPE, // TODO support completion
         FieldAliasMapper.CONTENT_TYPE // TODO support alias
@@ -598,6 +605,18 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy
             "createAggBuilderForTypeTest() must be implemented as well.");
     }
 
+    /**
+     * A method that allows implementors to specifically blacklist particular field types (based on their content_name).
+     * This is needed in some areas where the ValuesSourceType is not granular enough, for example integer values
+     * vs floating points, or `keyword` bytes vs `binary` bytes (which are not searchable)
+     *
+     * This is a blacklist instead of a whitelist because there are vastly more field types than ValuesSourceTypes,
+     * and it's expected that these unsupported cases are exceptional rather than common
+     */
+    protected List<String> unsupportedMappedFieldTypes() {
+        return Collections.emptyList();
+    }
+
     /**
      * This test will validate that an aggregator succeeds or fails to run against all the field types
      * that are registered in {@link IndicesModule} (e.g. all the core field types).  An aggregator
@@ -612,6 +631,7 @@ public final void testSupportedFieldTypes() throws IOException {
         Settings settings = Settings.builder().put("index.version.created", Version.CURRENT.id).build();
         String fieldName = "typeTestFieldName";
         List<ValuesSourceType> supportedVSTypes = getSupportedValuesSourceTypes();
+        List<String> unsupportedMappedFieldTypes = unsupportedMappedFieldTypes();
 
         if (supportedVSTypes.isEmpty()) {
             // If the test says it doesn't support any VStypes, it has not been converted yet so skip
@@ -630,7 +650,11 @@ public final void testSupportedFieldTypes() throws IOException {
 
             Map<String, Object> source = new HashMap<>();
             source.put("type", mappedType.getKey());
-            source.put("doc_values", "true");
+
+            // Text is the only field that doesn't support DVs, instead FD
+            if (mappedType.getKey().equals(TextFieldMapper.CONTENT_TYPE) == false) {
+                source.put("doc_values", "true");
+            }
 
             Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext());
             FieldMapper mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath()));
@@ -651,15 +675,16 @@ public final void testSupportedFieldTypes() throws IOException {
                     IndexSearcher indexSearcher = newIndexSearcher(indexReader);
                     AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName);
 
+                    ValuesSourceType vst = fieldType.getValuesSourceType();
                     // TODO in the future we can make this more explicit with expectThrows(), when the exceptions are standardized
                     try {
                         searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType);
-                        if (supportedVSTypes.contains(fieldType.getValuesSourceType()) == false) {
+                        if (supportedVSTypes.contains(vst) == false || unsupportedMappedFieldTypes.contains(fieldType.typeName())) {
                             fail("Aggregator [" + aggregationBuilder.getType() + "] should not support field type ["
-                                + fieldType.typeName() + "] but executing against the field did not throw an excetion");
+                                + fieldType.typeName() + "] but executing against the field did not throw an exception");
                         }
                     } catch (Exception e) {
-                        if (supportedVSTypes.contains(fieldType.getValuesSourceType())) {
+                        if (supportedVSTypes.contains(vst) && unsupportedMappedFieldTypes.contains(fieldType.typeName()) == false) {
                             fail("Aggregator [" + aggregationBuilder.getType() + "] supports field type ["
                                 + fieldType.typeName() + "] but executing against the field threw an exception: [" + e.getMessage() + "]");
                         }
@@ -679,34 +704,50 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd
 
         String typeName = fieldType.typeName();
         ValuesSourceType vst = fieldType.getValuesSourceType();
+        Document doc = new Document();
+        String json;
         
         if (vst.equals(CoreValuesSourceType.NUMERIC)) {
             // TODO note: once VS refactor adds DATE/BOOLEAN, this conditional will go away
+            long v;
             if (typeName.equals(DateFieldMapper.CONTENT_TYPE) || typeName.equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) {
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong())));
+                // positive integer because date_nanos gets unhappy with large longs
+                v = Math.abs(randomInt());
+                json = "{ \"" + fieldName + "\" : \"" + v + "\" }";
             } else if (typeName.equals(BooleanFieldMapper.CONTENT_TYPE)) {
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1)));
+                v = randomBoolean() ? 0 : 1;
+                json = "{ \"" + fieldName + "\" : \"" + (v == 0 ? "false" : "true") + "\" }";
             } else if (typeName.equals(NumberFieldMapper.NumberType.DOUBLE.typeName())) {
-                long encoded = NumericUtils.doubleToSortableLong(Math.abs(randomDouble()));
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, encoded)));
+                double d = Math.abs(randomDouble());
+                v = NumericUtils.doubleToSortableLong(d);
+                json = "{ \"" + fieldName + "\" : \"" + d + "\" }";
             } else if (typeName.equals(NumberFieldMapper.NumberType.FLOAT.typeName())) {
-                long encoded = NumericUtils.floatToSortableInt(Math.abs(randomFloat()));
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, encoded)));
+                float f = Math.abs(randomFloat());
+                v = NumericUtils.floatToSortableInt(f);
+                json = "{ \"" + fieldName + "\" : \"" + f + "\" }";
             } else if (typeName.equals(NumberFieldMapper.NumberType.HALF_FLOAT.typeName())) {
-                long encoded = HalfFloatPoint.halfFloatToSortableShort(Math.abs(randomFloat()));
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, encoded)));
+                float f = Math.abs(randomFloat());
+                v = HalfFloatPoint.halfFloatToSortableShort(f);
+                json = "{ \"" + fieldName + "\" : \"" + f + "\" }";
             } else {
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong())));
+                // smallest numeric is a byte so we select the smallest
+               v = Math.abs(randomByte());
+               json = "{ \"" + fieldName + "\" : \"" + v + "\" }";
             }
+            doc.add(new SortedNumericDocValuesField(fieldName, v));
+
         } else if (vst.equals(CoreValuesSourceType.BYTES)) {
             if (typeName.equals(BinaryFieldMapper.CONTENT_TYPE)) {
-                iw.addDocument(singleton(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes)));
+                doc.add(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes));
+                json = "{ \"" + fieldName + "\" : \"a\" }";
             } else if (typeName.equals(IpFieldMapper.CONTENT_TYPE)) {
                 // TODO note: once VS refactor adds IP, this conditional will go away
-                boolean v4 = randomBoolean();
-                iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4))))));
+                InetAddress ip = randomIp(randomBoolean());
+                json = "{ \"" + fieldName + "\" : \"" + NetworkAddress.format(ip) + "\" }";
+                doc.add(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(ip))));
             } else {
-                iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef("a"))));
+                doc.add(new SortedSetDocValuesField(fieldName, new BytesRef("a")));
+                json = "{ \"" + fieldName + "\" : \"a\" }";
             }
         } else if (vst.equals(CoreValuesSourceType.RANGE)) {
             Object start;
@@ -743,19 +784,37 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd
             }
 
             final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, start, end, true, true);
-            iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range)))));
-
+            doc.add(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range))));
+            json = "{ \"" + fieldName + "\" : { \n" +
+                "        \"gte\" : \"" + start + "\",\n" +
+                "        \"lte\" : \"" + end + "\"\n" +
+                "      }}";
         }  else if (vst.equals(CoreValuesSourceType.GEOPOINT)) {
-            iw.addDocument(singleton(new LatLonDocValuesField(fieldName, randomDouble(), randomDouble())));
+            double lat = randomDouble();
+            double lon = randomDouble();
+            doc.add(new LatLonDocValuesField(fieldName, lat, lon));
+            json = "{ \"" + fieldName + "\" : \"[" + lon + "," + lat + "]\" }";
         } else {
             throw new IllegalStateException("Unknown field type [" + typeName + "]");
         }
+
+        doc.add(new StoredField("_source", new BytesRef(json)));
+        iw.addDocument(doc);
     }
 
     private class MockParserContext extends Mapper.TypeParser.ParserContext {
         MockParserContext() {
             super(null, null, null, null, null);
         }
+
+        @Override
+        public IndexAnalyzers getIndexAnalyzers() {
+            NamedAnalyzer defaultAnalyzer = new NamedAnalyzer(AnalysisRegistry.DEFAULT_ANALYZER_NAME,
+                AnalyzerScope.GLOBAL, new StandardAnalyzer());
+            return new IndexAnalyzers(Map.of(AnalysisRegistry.DEFAULT_ANALYZER_NAME, defaultAnalyzer), Map.of(), Map.of());
+        }
+
+
     }
 
     @After

From 1016e20356417d1607ff1344fa023b0012bedf0a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Mar 2020 15:44:58 -0400
Subject: [PATCH 237/285] Prepare to backport #53793 (#54010)

Update version in wire protocol and disable BWC.
---
 build.gradle                                                  | 4 ++--
 .../org/elasticsearch/search/query/QuerySearchResult.java     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.gradle b/build.gradle
index 17cd5f6b93cd0..63c1d15eccbd4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53793" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 2863f45de85f2..e524e6354b473 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -320,7 +320,7 @@ public void readFromWithId(SearchContextId id, StreamInput in) throws IOExceptio
             }
         }
         setTopDocs(readTopDocs(in));
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getVersion().before(Version.V_7_7_0)) {
             InternalAggregations readAggs = null;
             if (hasAggs = in.readBoolean()) {
                 readAggs = new InternalAggregations(in);
@@ -398,7 +398,7 @@ public void writeToNoId(StreamOutput out) throws IOException {
             }
         } else {
             out.writeBoolean(true);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getVersion().before(Version.V_7_7_0)) {
                 InternalAggregations aggs = aggregations.get();
                 aggs.writeTo(out);
                 if (out.getVersion().before(Version.V_7_2_0)) {

From 6ded7f08350a739769d1f4e3cf99931c0dee5fbf Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Mon, 23 Mar 2020 15:57:10 -0400
Subject: [PATCH 238/285] Add VST for wildcard field

---
 .../wildcard/mapper/WildcardFieldMapper.java  | 143 +++++++++---------
 1 file changed, 75 insertions(+), 68 deletions(-)

diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index e489d8a35bb9f..1cbb932decba0 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -57,6 +57,8 @@
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -73,15 +75,15 @@
 public class WildcardFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "wildcard";
-    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; 
-    public static final int NGRAM_SIZE = 3;        
+    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10;
+    public static final int NGRAM_SIZE = 3;
     static final NamedAnalyzer WILDCARD_ANALYZER = new NamedAnalyzer("_wildcard", AnalyzerScope.GLOBAL, new Analyzer() {
         @Override
         public TokenStreamComponents createComponents(String fieldName) {
             Tokenizer tokenizer = new NGramTokenizer(NGRAM_SIZE, NGRAM_SIZE);
             return new TokenStreamComponents(tokenizer);
         }
-    });    
+    });
 
     public static class Defaults {
         public static final MappedFieldType FIELD_TYPE = new WildcardFieldType();
@@ -95,7 +97,7 @@ public static class Defaults {
             FIELD_TYPE.setOmitNorms(true);
             FIELD_TYPE.freeze();
         }
-        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;        
+        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;
     }
 
     public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
@@ -109,7 +111,7 @@ public Builder(String name) {
         @Override
         public Builder docValues(boolean docValues) {
             if (docValues == false) {
-                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");                
+                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");
             }
             return this;
         }
@@ -125,39 +127,39 @@ public Builder indexOptions(IndexOptions indexOptions) {
         @Override
         public Builder store(boolean store) {
             if (store) {
-                throw new MapperParsingException("The field [" + name + "] cannot have store = true");                
+                throw new MapperParsingException("The field [" + name + "] cannot have store = true");
             }
             return this;
         }
 
         @Override
         public Builder similarity(SimilarityProvider similarity) {
-            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");                
+            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");
         }
 
         @Override
         public Builder index(boolean index) {
             if (index == false) {
-                throw new MapperParsingException("The field [" + name + "] cannot have index = false");                
+                throw new MapperParsingException("The field [" + name + "] cannot have index = false");
             }
             return this;
         }
-        
+
         public Builder ignoreAbove(int ignoreAbove) {
             if (ignoreAbove < 0) {
                 throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
             }
             this.ignoreAbove = ignoreAbove;
             return this;
-        }        
-          
-        
+        }
+
+
         @Override
         protected void setupFieldType(BuilderContext context) {
             super.setupFieldType(context);
             fieldType().setHasDocValues(true);
             fieldType().setTokenized(false);
-            fieldType().setIndexOptions(IndexOptions.DOCS);                
+            fieldType().setIndexOptions(IndexOptions.DOCS);
         }
 
         @Override
@@ -167,9 +169,9 @@ public WildcardFieldType fieldType() {
 
         @Override
         public WildcardFieldMapper build(BuilderContext context) {
-            setupFieldType(context);            
+            setupFieldType(context);
             return new WildcardFieldMapper(
-                    name, fieldType, defaultFieldType, ignoreAbove, 
+                    name, fieldType, defaultFieldType, ignoreAbove,
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
         }
     }
@@ -180,7 +182,7 @@ public static class TypeParser implements Mapper.TypeParser {
                 throws MapperParsingException {
             WildcardFieldMapper.Builder builder = new WildcardFieldMapper.Builder(name);
             parseField(builder, name, node, parserContext);
-            
+
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String propName = entry.getKey();
@@ -189,19 +191,19 @@ public static class TypeParser implements Mapper.TypeParser {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
                 }
-            }            
-            
+            }
+
             return builder;
         }
     }
-    
-     public static final char TOKEN_START_OR_END_CHAR = 0;    
-    
+
+     public static final char TOKEN_START_OR_END_CHAR = 0;
+
      public static final class WildcardFieldType extends MappedFieldType {
 
-        public WildcardFieldType() {            
+        public WildcardFieldType() {
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
-            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);            
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
         }
 
         protected WildcardFieldType(WildcardFieldType ref) {
@@ -212,23 +214,23 @@ public WildcardFieldType clone() {
             WildcardFieldType result = new WildcardFieldType(this);
             return result;
         }
-        
-                
+
+
         // Holds parsed information about the wildcard pattern
         static class PatternStructure {
-            boolean openStart, openEnd, hasSymbols;            
+            boolean openStart, openEnd, hasSymbols;
             int lastGap =0;
             int wildcardCharCount, wildcardStringCount;
             String[] fragments;
             Integer []  precedingGapSizes;
             final String pattern;
-            
+
             @SuppressWarnings("fallthrough") // Intentionally uses fallthrough mirroring implementation in Lucene's WildcardQuery
             PatternStructure (String wildcardText) {
                 this.pattern = wildcardText;
                 ArrayList<String> fragmentList = new ArrayList<>();
                 ArrayList<Integer> precedingGapSizeList = new ArrayList<>();
-                StringBuilder sb = new StringBuilder();               
+                StringBuilder sb = new StringBuilder();
                 for (int i = 0; i < wildcardText.length();) {
                     final int c = wildcardText.codePointAt(i);
                     int length = Character.charCount(c);
@@ -237,10 +239,10 @@ static class PatternStructure {
                         if (i == 0) {
                             openStart = true;
                         }
-                        openEnd = true;                        
+                        openEnd = true;
                         hasSymbols = true;
                         wildcardStringCount++;
-                        
+
                         if (sb.length() > 0) {
                             precedingGapSizeList.add(lastGap);
                             fragmentList.add(sb.toString());
@@ -258,10 +260,10 @@ static class PatternStructure {
                         if (sb.length() > 0) {
                             precedingGapSizeList.add(lastGap);
                             fragmentList.add(sb.toString());
-                            sb = new StringBuilder();       
+                            sb = new StringBuilder();
                             lastGap = 0;
                         }
-                        
+
                         if (lastGap != Integer.MAX_VALUE) {
                             lastGap++;
                         }
@@ -288,19 +290,19 @@ static class PatternStructure {
                 }
                 fragments = fragmentList.toArray(new String[0]);
                 precedingGapSizes = precedingGapSizeList.toArray(new Integer[0]);
-                
+
             }
-            
+
             public boolean needsVerification() {
                 // Return true if term queries are not enough evidence
                 if (fragments.length == 1 && wildcardCharCount == 0) {
-                    // The one case where we don't need verification is when 
+                    // The one case where we don't need verification is when
                     // we have a single fragment and no ? characters
                     return false;
                 }
                 return true;
             }
-            
+
             // Returns number of positions for last gap (Integer.MAX means unlimited gap)
             public int getPrecedingGapSize(int fragmentNum) {
                 return precedingGapSizes[fragmentNum];
@@ -320,14 +322,14 @@ public boolean equals(Object obj) {
                 PatternStructure other = (PatternStructure) obj;
                 return pattern.equals(other.pattern);
             }
-            
-            
+
+
         }
-        
+
 
         @Override
         public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
-            PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
+            PatternStructure patternStructure = new PatternStructure(wildcardPattern);
             ArrayList<String> tokens = new ArrayList<>();
 
             for (int i = 0; i < patternStructure.fragments.length; i++) {
@@ -336,7 +338,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                 if (fLength == 0) {
                     continue;
                 }
-                
+
                 // Add any start/end of string character
                 if (i == 0 && patternStructure.openStart == false) {
                     // Start-of-string anchored (is not a leading wildcard)
@@ -349,7 +351,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                 if (fragment.codePointCount(0, fragment.length()) <= NGRAM_SIZE) {
                     tokens.add(fragment);
                 } else {
-                    // Break fragment into multiple Ngrams                
+                    // Break fragment into multiple Ngrams
                     TokenStream tokenizer = WILDCARD_ANALYZER.tokenStream(name(), fragment);
                     CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
                     String lastUnusedToken = null;
@@ -363,7 +365,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                             if (takeThis) {
                                 tokens.add(tokenValue);
                             } else {
-                                lastUnusedToken = tokenValue;                                
+                                lastUnusedToken = tokenValue;
                             }
                             // alternate
                             takeThis = !takeThis;
@@ -383,7 +385,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
 
             if (patternStructure.isMatchAll()) {
                 return new MatchAllDocsQuery();
-            } 
+            }
             BooleanQuery approximation = createApproximationQuery(tokens);
             if (approximation.clauses().size() > 1 || patternStructure.needsVerification()) {
                 BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder();
@@ -393,7 +395,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                 return verifyingBuilder.build();
             }
             return approximation;
-        }                
+        }
 
         private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
             BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
@@ -411,7 +413,7 @@ private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
             // TODO we can be smarter about pruning here. e.g.
             // * Avoid wildcard queries if there are sufficient numbers of other terms that are full 3grams that are cheaper term queries
             // * We can select terms on their scarcity rather than even spreads across the search string.
-            
+
             return bqBuilder.build();
         }
 
@@ -442,11 +444,11 @@ public Query existsQuery(QueryShardContext context) {
         public Query termQuery(Object value, QueryShardContext context) {
             return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, context);
         }
-        
+
         @Override
         public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
             return wildcardQuery(value + "*", method, context);
-        }        
+        }
 
         @Override
         public Query termsQuery(List<?> values, QueryShardContext context) {
@@ -455,8 +457,8 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
                 bq.add(termQuery(value, context), Occur.SHOULD);
             }
             return new ConstantScoreQuery(bq.build());
-        }     
-                
+        }
+
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
             failIfNoDocValues();
@@ -467,9 +469,14 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
                         CircuitBreakerService breakerService, MapperService mapperService) {
                     return new WildcardBytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
                 }};
-        }        
-    }
-     
+        }
+
+         @Override
+         public ValuesSourceType getValuesSourceType() {
+             return CoreValuesSourceType.BYTES;
+         }
+     }
+
     static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
 
         WildcardBytesBinaryDVIndexFieldData(Index index, String fieldName) {
@@ -482,7 +489,7 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested
                     sortMode, nested);
             return new SortField(getFieldName(), source, reverse);
         }
-    
+
     }
 
     private int ignoreAbove;
@@ -492,9 +499,9 @@ private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, Mapped
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         this.ignoreAbove = ignoreAbove;
         assert fieldType.indexOptions() == IndexOptions.DOCS;
-        
+
         ngramFieldType = fieldType.clone();
-        ngramFieldType.setTokenized(true);                    
+        ngramFieldType.setTokenized(true);
         ngramFieldType.freeze();
     }
 
@@ -503,8 +510,8 @@ private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, Mapped
     // pkg-private for testing
     int ignoreAbove() {
         return ignoreAbove;
-    }    
-    
+    }
+
     @Override
     protected WildcardFieldMapper clone() {
         return (WildcardFieldMapper) super.clone();
@@ -522,7 +529,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
             builder.field("ignore_above", ignoreAbove);
         }
     }
-    
+
     @Override
     protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         final String value;
@@ -537,13 +544,13 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
             }
         }
         ParseContext.Document parseDoc = context.doc();
-        
-        createFields(value, parseDoc, fields);        
-    }   
-    
+
+        createFields(value, parseDoc, fields);
+    }
+
     // For internal use by Lucene only - used to define ngram index
     final MappedFieldType ngramFieldType;
-    
+
     void createFields(String value, Document parseDoc, List<IndexableField>fields) {
         if (value == null || value.length() > ignoreAbove) {
             return;
@@ -551,25 +558,25 @@ void createFields(String value, Document parseDoc, List<IndexableField>fields) {
         String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
         Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
         fields.add(ngramField);
-        
+
         CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name());
         if (dvField == null) {
             dvField = new CustomBinaryDocValuesField(fieldType().name(), value.getBytes(StandardCharsets.UTF_8));
             parseDoc.addWithKey(fieldType().name(), dvField);
         } else {
             dvField.add(value.getBytes(StandardCharsets.UTF_8));
-        }        
+        }
     }
 
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
     }
-    
+
 
     @Override
     protected void doMerge(Mapper mergeWith) {
         super.doMerge(mergeWith);
         this.ignoreAbove = ((WildcardFieldMapper) mergeWith).ignoreAbove;
-    }    
+    }
 }

From 586f4b7a88af6e33160d996838cd6824d5e5b122 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 23 Mar 2020 13:31:19 -0700
Subject: [PATCH 239/285] Clarify IntelliJ import instructions

---
 CONTRIBUTING.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c072b7b3649ad..1398dd76979e9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -125,10 +125,13 @@ You can access Elasticsearch with:
 
 ### Importing the project into IntelliJ IDEA
 
-Elasticsearch builds using Java 13. Before importing into IntelliJ you will need
+Elasticsearch builds using Java 13. When importing into IntelliJ you will need
 to define an appropriate SDK. The convention is that **this SDK should be named
 "13"** so that the project import will detect it automatically. For more details
 on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
+SDK definitions are global, so you can add the JDK from any project, or after
+project import. Importing with a missing JDK will still work, IntelliJ will
+simply report a problem and will refuse to build until resolved.
 
 You can import the Elasticsearch project into IntelliJ IDEA via:
 

From ecdbd37f2e0f0447ed574b306adb64c19adc3ce1 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 23 Mar 2020 13:44:12 -0700
Subject: [PATCH 240/285] Refactor global build info plugin to leverage
 JavaInstallationRegistry (#53994)

This commit removes the configuration time vs execution time distinction
with regards to certain BuildParms properties. Because of the cost of
determining Java versions for configuration JDK locations we deferred
this until execution time. This had two main downsides. First, we had
to implement all this build logic in tasks, which required a bunch of
additional plumbing and complexity. Second, because some information
wasn't known during configuration time, we had to nest any build logic
that depended on this in awkward callbacks.

We now defer to the JavaInstallationRegistry recently added in Gradle.
This utility uses a much more efficient method for probing Java
installations vs our jrunscript implementation. This, combined with some
optimizations to avoid probing the current JVM as well as deferring
some evaluation via Providers when probing installations for BWC builds
we can maintain effectively the same configuration time performance
while removing a bunch of complexity and runtime cost (snapshotting
inputs for the GenerateGlobalBuildInfoTask was very expensive). The end
result should be a much more responsive build execution in almost all
scenarios.
---
 CONTRIBUTING.md                               |   3 -
 build.gradle                                  |   2 +-
 .../elasticsearch/gradle/BuildPlugin.groovy   |   8 +-
 .../gradle/precommit/PrecommitTasks.groovy    |  17 +-
 .../gradle/DistributionDownloadPlugin.java    |   6 +-
 .../gradle/JdkDownloadPlugin.java             |   4 +-
 .../info/GenerateGlobalBuildInfoTask.java     | 293 ------------------
 .../gradle/info/GlobalBuildInfoPlugin.java    | 229 ++++++++------
 .../gradle/info/GlobalInfoExtension.java      |  12 -
 .../gradle/info/PrintGlobalBuildInfoTask.java |  75 -----
 .../gradle/precommit/FilePermissionsTask.java |   4 +-
 .../precommit/TestingConventionsTasks.java    |  10 +-
 .../gradle/test/DistroTestPlugin.java         |   4 +-
 .../gradle/test/rest/CopyRestApiTask.java     |   4 +-
 .../gradle/test/rest/CopyRestTestsTask.java   |   4 +-
 .../testclusters/RestTestRunnerTask.java      |   4 +-
 .../testclusters/TestClustersPlugin.java      |   6 +-
 .../testfixtures/TestFixturesPlugin.java      |   4 +-
 .../GradleUtils.java}                         |   4 +-
 .../elasticsearch/gradle/{ => util}/Util.java |  27 +-
 .../gradle/vagrant/VagrantMachine.java        |   2 +-
 .../gradle/info/BuildParams.java              |  38 +--
 .../elasticsearch/gradle/info/JavaHome.java   |  18 +-
 gradle/ide.gradle                             |   2 +-
 .../reindex-tests-with-security/build.gradle  |   6 +-
 25 files changed, 220 insertions(+), 566 deletions(-)
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
 rename buildSrc/src/main/java/org/elasticsearch/gradle/{tool/Boilerplate.java => util/GradleUtils.java} (98%)
 rename buildSrc/src/main/java/org/elasticsearch/gradle/{ => util}/Util.java (60%)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1398dd76979e9..e6764b505d5f3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -103,9 +103,6 @@ be used to test against other JDKs as well, this is not only limited to JDK 11.
 > Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
 and `JAVA11_HOME`, and `JAVA12_HOME` available so that the tests can pass.
 
-> Warning: do not use `sdkman` for Java installations which do not have proper
-`jrunscript` for jdk distributions.
-
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
 script on Windows in the root of the repository. The examples below show the
diff --git a/build.gradle b/build.gradle
index 63c1d15eccbd4..4ae244c9ecf8f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -31,7 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
 
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 
 plugins {
   id 'lifecycle-base'
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 922afdbd770a0..7580671cc4cd6 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -32,7 +32,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
-import org.elasticsearch.gradle.tool.Boilerplate
+import org.elasticsearch.gradle.util.GradleUtils
 import org.gradle.api.Action
 import org.gradle.api.GradleException
 import org.gradle.api.InvalidUserDataException
@@ -82,7 +82,7 @@ import org.gradle.util.GradleVersion
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -148,7 +148,7 @@ class BuildPlugin implements Plugin<Project> {
             File securityPolicy = buildResources.copy("fips_java.policy")
             File bcfksKeystore = buildResources.copy("cacerts.bcfks")
             // This configuration can be removed once system modules are available
-            Boilerplate.maybeCreate(project.configurations, 'extraJars') {
+            GradleUtils.maybeCreate(project.configurations, 'extraJars') {
                 project.dependencies.add('extraJars', "org.bouncycastle:bc-fips:1.0.1")
                 project.dependencies.add('extraJars', "org.bouncycastle:bctls-fips:1.0.9")
             }
@@ -235,7 +235,7 @@ class BuildPlugin implements Plugin<Project> {
     static String getJavaHome(final Task task, final int version) {
         requireJavaHome(task, version)
         JavaHome java = BuildParams.javaVersions.find { it.version == version }
-        return java == null ? null : java.javaHome.absolutePath
+        return java == null ? null : java.javaHome.get().absolutePath
     }
 
     /**
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 52f98330ee50d..d95514f975a13 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -143,16 +143,13 @@ class PrecommitTasks {
         ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
         project.tasks.withType(CheckForbiddenApis).configureEach {
             dependsOn(buildResources)
-            doFirst {
-                // we need to defer this configuration since we don't know the runtime java version until execution time
-                targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
-                if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
-                    project.logger.warn(
-                            "Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
-                            BuildParams.runtimeJavaVersion
-                    )
-                    targetCompatibility = JavaVersion.VERSION_13.majorVersion
-                }
+            targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
+            if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
+                project.logger.warn(
+                        "Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
+                        BuildParams.runtimeJavaVersion
+                )
+                targetCompatibility = JavaVersion.VERSION_13.majorVersion
             }
             bundledSignatures = [
                     "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
index 0982fe0267cc5..048403d83f14d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
@@ -52,7 +52,7 @@
 import java.util.concurrent.Callable;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.gradle.Util.capitalize;
+import static org.elasticsearch.gradle.util.Util.capitalize;
 
 /**
  * A plugin to manage getting and extracting distributions of Elasticsearch.
@@ -77,7 +77,7 @@ public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
 
-        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+        Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index daaf1a0fa54e9..a2dca7c247c0d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -47,8 +47,8 @@
 import java.util.concurrent.Callable;
 import java.util.stream.StreamSupport;
 
-import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
+import static org.elasticsearch.gradle.util.GradleUtils.findByName;
+import static org.elasticsearch.gradle.util.GradleUtils.maybeCreate;
 
 public class JdkDownloadPlugin implements Plugin<Project> {
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
deleted file mode 100644
index 75ae735e8a9f1..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
+++ /dev/null
@@ -1,293 +0,0 @@
-package org.elasticsearch.gradle.info;
-
-import org.elasticsearch.gradle.OS;
-import org.gradle.api.DefaultTask;
-import org.gradle.api.GradleException;
-import org.gradle.api.JavaVersion;
-import org.gradle.api.file.RegularFileProperty;
-import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.tasks.CacheableTask;
-import org.gradle.api.tasks.Input;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.Nested;
-import org.gradle.api.tasks.OutputFile;
-import org.gradle.api.tasks.PathSensitive;
-import org.gradle.api.tasks.PathSensitivity;
-import org.gradle.api.tasks.TaskAction;
-import org.gradle.internal.jvm.Jvm;
-import org.gradle.process.ExecResult;
-
-import javax.inject.Inject;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.io.Writer;
-import java.nio.file.Files;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-@CacheableTask
-public class GenerateGlobalBuildInfoTask extends DefaultTask {
-    private JavaVersion minimumCompilerVersion;
-    private JavaVersion minimumRuntimeVersion;
-    private File compilerJavaHome;
-    private File runtimeJavaHome;
-    private List<JavaHome> javaVersions;
-    private final RegularFileProperty outputFile;
-    private final RegularFileProperty compilerVersionFile;
-    private final RegularFileProperty runtimeVersionFile;
-
-    @Inject
-    public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
-        this.outputFile = objectFactory.fileProperty();
-        this.compilerVersionFile = objectFactory.fileProperty();
-        this.runtimeVersionFile = objectFactory.fileProperty();
-    }
-
-    @Input
-    public JavaVersion getMinimumCompilerVersion() {
-        return minimumCompilerVersion;
-    }
-
-    public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) {
-        this.minimumCompilerVersion = minimumCompilerVersion;
-    }
-
-    @Input
-    public JavaVersion getMinimumRuntimeVersion() {
-        return minimumRuntimeVersion;
-    }
-
-    public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) {
-        this.minimumRuntimeVersion = minimumRuntimeVersion;
-    }
-
-    @InputDirectory
-    @PathSensitive(PathSensitivity.RELATIVE)
-    public File getCompilerJavaHome() {
-        return compilerJavaHome;
-    }
-
-    public void setCompilerJavaHome(File compilerJavaHome) {
-        this.compilerJavaHome = compilerJavaHome;
-    }
-
-    @InputDirectory
-    @PathSensitive(PathSensitivity.RELATIVE)
-    public File getRuntimeJavaHome() {
-        return runtimeJavaHome;
-    }
-
-    public void setRuntimeJavaHome(File runtimeJavaHome) {
-        this.runtimeJavaHome = runtimeJavaHome;
-    }
-
-    @Nested
-    public List<JavaHome> getJavaVersions() {
-        return javaVersions;
-    }
-
-    public void setJavaVersions(List<JavaHome> javaVersions) {
-        this.javaVersions = javaVersions;
-    }
-
-    @OutputFile
-    public RegularFileProperty getOutputFile() {
-        return outputFile;
-    }
-
-    @OutputFile
-    public RegularFileProperty getCompilerVersionFile() {
-        return compilerVersionFile;
-    }
-
-    @OutputFile
-    public RegularFileProperty getRuntimeVersionFile() {
-        return runtimeVersionFile;
-    }
-
-    @TaskAction
-    public void generate() {
-        String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
-        String gradleJavaVersion = System.getProperty("java.version");
-        String gradleJavaVersionDetails = javaVendorVersion
-            + " "
-            + gradleJavaVersion
-            + " ["
-            + System.getProperty("java.vm.name")
-            + " "
-            + System.getProperty("java.vm.version")
-            + "]";
-
-        String compilerJavaVersionDetails = gradleJavaVersionDetails;
-        JavaVersion compilerJavaVersionEnum = JavaVersion.current();
-        String runtimeJavaVersionDetails = gradleJavaVersionDetails;
-        JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
-        File gradleJavaHome = Jvm.current().getJavaHome();
-
-        try {
-            if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
-                if (compilerJavaHome.exists()) {
-                    compilerJavaVersionDetails = findJavaVersionDetails(compilerJavaHome);
-                    compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(compilerJavaHome));
-                } else {
-                    throw new RuntimeException("Compiler Java home path of '" + compilerJavaHome + "' does not exist");
-                }
-            }
-
-            if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
-                if (runtimeJavaHome.exists()) {
-                    runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
-                    runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
-                } else {
-                    throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
-                }
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-
-        try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile.getAsFile().get()))) {
-            final String osName = System.getProperty("os.name");
-            final String osVersion = System.getProperty("os.version");
-            final String osArch = System.getProperty("os.arch");
-            final JavaVersion parsedVersion = JavaVersion.toVersion(gradleJavaVersion);
-
-            writer.write("  Gradle Version        : " + getProject().getGradle().getGradleVersion() + "\n");
-            writer.write("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")\n");
-
-            if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false
-                || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) {
-                writer.write("  Compiler JDK Version  : " + compilerJavaVersionEnum + " (" + compilerJavaVersionDetails + ")\n");
-                writer.write("  Compiler java.home    : " + compilerJavaHome + "\n");
-                writer.write("  Runtime JDK Version   : " + runtimeJavaVersionEnum + " (" + runtimeJavaVersionDetails + ")\n");
-                writer.write("  Runtime java.home     : " + runtimeJavaHome + "\n");
-                writer.write("  Gradle JDK Version    : " + parsedVersion + " (" + gradleJavaVersionDetails + ")\n");
-                writer.write("  Gradle java.home      : " + gradleJavaHome);
-            } else {
-                writer.write("  JDK Version           : " + parsedVersion + " (" + gradleJavaVersionDetails + ")\n");
-                writer.write("  JAVA_HOME             : " + gradleJavaHome);
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-
-        // enforce Java version
-        if (compilerJavaVersionEnum.compareTo(minimumCompilerVersion) < 0) {
-            String message = String.format(
-                Locale.ROOT,
-                "The compiler java.home must be set to a JDK installation directory for Java %s but is [%s] " + "corresponding to [%s]",
-                minimumCompilerVersion,
-                compilerJavaHome,
-                compilerJavaVersionEnum
-            );
-            throw new GradleException(message);
-        }
-
-        if (runtimeJavaVersionEnum.compareTo(minimumRuntimeVersion) < 0) {
-            String message = String.format(
-                Locale.ROOT,
-                "The runtime java.home must be set to a JDK installation directory for Java %s but is [%s] " + "corresponding to [%s]",
-                minimumRuntimeVersion,
-                runtimeJavaHome,
-                runtimeJavaVersionEnum
-            );
-            throw new GradleException(message);
-        }
-
-        for (JavaHome javaVersion : javaVersions) {
-            File javaHome = javaVersion.getJavaHome();
-            if (javaHome == null) {
-                continue;
-            }
-            JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(javaHome));
-            JavaVersion expectedJavaVersionEnum;
-            int version = javaVersion.getVersion();
-            if (version < 9) {
-                expectedJavaVersionEnum = JavaVersion.toVersion("1." + version);
-            } else {
-                expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version));
-            }
-            if (javaVersionEnum != expectedJavaVersionEnum) {
-                String message = String.format(
-                    Locale.ROOT,
-                    "The environment variable JAVA%d_HOME must be set to a JDK installation directory for Java"
-                        + " %s but is [%s] corresponding to [%s]",
-                    version,
-                    expectedJavaVersionEnum,
-                    javaHome,
-                    javaVersionEnum
-                );
-                throw new GradleException(message);
-            }
-        }
-
-        writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
-        writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
-    }
-
-    private void writeToFile(File file, String content) {
-        try (Writer writer = new FileWriter(file)) {
-            writer.write(content);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    /**
-     * Finds printable java version of the given JAVA_HOME
-     */
-    private String findJavaVersionDetails(File javaHome) {
-        String versionInfoScript = "print("
-            + "java.lang.System.getProperty(\"java.vendor.version\", java.lang.System.getProperty(\"java.vendor\")) + \" \" + "
-            + "java.lang.System.getProperty(\"java.version\") + \" [\" + "
-            + "java.lang.System.getProperty(\"java.vm.name\") + \" \" + "
-            + "java.lang.System.getProperty(\"java.vm.version\") + \"]\");";
-        return runJavaAsScript(javaHome, versionInfoScript).trim();
-    }
-
-    /**
-     * Finds the parsable java specification version
-     */
-    private String findJavaSpecificationVersion(File javaHome) {
-        String versionScript = "print(java.lang.System.getProperty(\"java.specification.version\"));";
-        return runJavaAsScript(javaHome, versionScript);
-    }
-
-    /**
-     * Runs the given javascript using jjs from the jdk, and returns the output
-     */
-    private String runJavaAsScript(File javaHome, String script) {
-        ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-        ByteArrayOutputStream stderr = new ByteArrayOutputStream();
-        if (OS.current() == OS.WINDOWS) {
-            // gradle/groovy does not properly escape the double quote for windows
-            script = script.replace("\"", "\\\"");
-        }
-        File jrunscriptPath = new File(javaHome, "bin/jrunscript");
-        String finalScript = script;
-        ExecResult result = getProject().exec(spec -> {
-            spec.setExecutable(jrunscriptPath);
-            spec.args("-e", finalScript);
-            spec.setStandardOutput(stdout);
-            spec.setErrorOutput(stderr);
-            spec.setIgnoreExitValue(true);
-        });
-
-        if (result.getExitValue() != 0) {
-            getLogger().error("STDOUT:");
-            Arrays.stream(stdout.toString(UTF_8).split(System.getProperty("line.separator"))).forEach(getLogger()::error);
-            getLogger().error("STDERR:");
-            Arrays.stream(stderr.toString(UTF_8).split(System.getProperty("line.separator"))).forEach(getLogger()::error);
-            result.rethrowFailure();
-        }
-        return stdout.toString(UTF_8).trim();
-    }
-
-}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
index 431e66e9cde94..21642b30c0480 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
@@ -1,18 +1,27 @@
 package org.elasticsearch.gradle.info;
 
 import org.elasticsearch.gradle.OS;
+import org.elasticsearch.gradle.util.Util;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.internal.jvm.Jvm;
+import org.gradle.jvm.toolchain.JavaInstallation;
+import org.gradle.jvm.toolchain.JavaInstallationRegistry;
+import org.gradle.util.GradleVersion;
 
+import javax.inject.Inject;
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -33,102 +42,168 @@
 import java.util.stream.Stream;
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
-    private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
+    private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
     private static Integer _defaultParallel = null;
 
+    private final JavaInstallationRegistry javaInstallationRegistry;
+    private final ObjectFactory objects;
+    private final ProviderFactory providers;
+
+    @Inject
+    public GlobalBuildInfoPlugin(JavaInstallationRegistry javaInstallationRegistry, ObjectFactory objects, ProviderFactory providers) {
+        this.javaInstallationRegistry = javaInstallationRegistry;
+        this.objects = objects;
+        this.providers = providers;
+    }
+
     @Override
     public void apply(Project project) {
         if (project != project.getRootProject()) {
             throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
         }
 
-        GlobalInfoExtension extension = project.getExtensions().create(GLOBAL_INFO_EXTENSION_NAME, GlobalInfoExtension.class);
-
-        JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion"));
-        JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"));
+        JavaVersion minimumCompilerVersion = JavaVersion.toVersion(Util.getResourceContents("/minimumCompilerVersion"));
+        JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(Util.getResourceContents("/minimumRuntimeVersion"));
 
         File compilerJavaHome = findCompilerJavaHome();
         File runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome);
 
-        String testSeedProperty = System.getProperty("tests.seed");
-        final String testSeed;
-        if (testSeedProperty == null) {
-            long seed = new Random(System.currentTimeMillis()).nextLong();
-            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT);
-        } else {
-            testSeed = testSeedProperty;
-        }
-
-        final String buildSnapshotSystemProperty = System.getProperty("build.snapshot", "true");
-        final boolean isSnapshotBuild;
-        switch (buildSnapshotSystemProperty) {
-            case "true":
-                isSnapshotBuild = true;
-                break;
-            case "false":
-                isSnapshotBuild = false;
-                break;
-            default:
-                throw new IllegalArgumentException(
-                    "build.snapshot was set to [" + buildSnapshotSystemProperty + "] but can only be unset or [true|false]"
-                );
-        }
-        final List<JavaHome> javaVersions = new ArrayList<>();
-        for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); version++) {
-            if (System.getenv(getJavaHomeEnvVarName(Integer.toString(version))) != null) {
-                javaVersions.add(JavaHome.of(version, new File(findJavaHome(Integer.toString(version)))));
-            }
-        }
-
-        GenerateGlobalBuildInfoTask generateTask = project.getTasks()
-            .create("generateGlobalBuildInfo", GenerateGlobalBuildInfoTask.class, task -> {
-                task.setJavaVersions(javaVersions);
-                task.setMinimumCompilerVersion(minimumCompilerVersion);
-                task.setMinimumRuntimeVersion(minimumRuntimeVersion);
-                task.setCompilerJavaHome(compilerJavaHome);
-                task.setRuntimeJavaHome(runtimeJavaHome);
-                task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
-                task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
-                task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
-            });
-
-        PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
-            task.getBuildInfoFile().set(generateTask.getOutputFile());
-            task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
-            task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
-            task.setGlobalInfoListeners(extension.listeners);
-        });
-
         // Initialize global build parameters
         BuildParams.init(params -> {
             params.reset();
             params.setCompilerJavaHome(compilerJavaHome);
             params.setRuntimeJavaHome(runtimeJavaHome);
+            params.setCompilerJavaVersion(determineJavaVersion("compiler java.home", compilerJavaHome, minimumCompilerVersion));
+            params.setRuntimeJavaVersion(determineJavaVersion("runtime java.home", runtimeJavaHome, minimumRuntimeVersion));
             params.setIsRutimeJavaHomeSet(compilerJavaHome.equals(runtimeJavaHome) == false);
-            params.setJavaVersions(javaVersions);
+            params.setJavaVersions(getAvailableJavaVersions(minimumCompilerVersion));
             params.setMinimumCompilerVersion(minimumCompilerVersion);
             params.setMinimumRuntimeVersion(minimumRuntimeVersion);
             params.setGradleJavaVersion(Jvm.current().getJavaVersion());
             params.setGitRevision(gitRevision(project.getRootProject().getRootDir()));
             params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC));
-            params.setTestSeed(testSeed);
+            params.setTestSeed(getTestSeed());
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
             params.setDefaultParallel(findDefaultParallel(project));
-            params.setInFipsJvm(isInFipsJvm());
-            params.setIsSnapshotBuild(isSnapshotBuild);
+            params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
+            params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
         });
 
-        project.allprojects(
-            p -> {
-                // Make sure than any task execution generates and prints build info
-                p.getTasks().configureEach(task -> {
-                    if (task != generateTask && task != printTask) {
-                        task.dependsOn(printTask);
+        // Print global build info header just before task execution
+        project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo());
+    }
+
+    private void logGlobalBuildInfo() {
+        final String osName = System.getProperty("os.name");
+        final String osVersion = System.getProperty("os.version");
+        final String osArch = System.getProperty("os.arch");
+        final Jvm gradleJvm = Jvm.current();
+        final String gradleJvmDetails = getJavaInstallation(gradleJvm.getJavaHome()).getImplementationName();
+
+        LOGGER.quiet("=======================================");
+        LOGGER.quiet("Elasticsearch Build Hamster says Hello!");
+        LOGGER.quiet("  Gradle Version        : " + GradleVersion.current().getVersion());
+        LOGGER.quiet("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")");
+        if (Jvm.current().getJavaVersion().equals(BuildParams.getCompilerJavaVersion()) == false || BuildParams.getIsRuntimeJavaHomeSet()) {
+            String compilerJvmDetails = getJavaInstallation(BuildParams.getCompilerJavaHome()).getImplementationName();
+            String runtimeJvmDetails = getJavaInstallation(BuildParams.getRuntimeJavaHome()).getImplementationName();
+
+            LOGGER.quiet("  Compiler JDK Version  : " + BuildParams.getCompilerJavaVersion() + " (" + compilerJvmDetails + ")");
+            LOGGER.quiet("  Compiler java.home    : " + BuildParams.getCompilerJavaHome());
+            LOGGER.quiet("  Runtime JDK Version   : " + BuildParams.getRuntimeJavaVersion() + " (" + runtimeJvmDetails + ")");
+            LOGGER.quiet("  Runtime java.home     : " + BuildParams.getRuntimeJavaHome());
+            LOGGER.quiet("  Gradle JDK Version    : " + gradleJvm.getJavaVersion() + " (" + gradleJvmDetails + ")");
+            LOGGER.quiet("  Gradle java.home      : " + gradleJvm.getJavaHome());
+        } else {
+            LOGGER.quiet("  JDK Version           : " + gradleJvm.getJavaVersion() + " (" + gradleJvmDetails + ")");
+            LOGGER.quiet("  JAVA_HOME             : " + gradleJvm.getJavaHome());
+        }
+        LOGGER.quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
+        LOGGER.quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
+        LOGGER.quiet("=======================================");
+    }
+
+    private JavaVersion determineJavaVersion(String description, File javaHome, JavaVersion requiredVersion) {
+        JavaInstallation installation = getJavaInstallation(javaHome);
+        JavaVersion actualVersion = installation.getJavaVersion();
+        if (actualVersion.isCompatibleWith(requiredVersion) == false) {
+            throwInvalidJavaHomeException(
+                description,
+                javaHome,
+                Integer.parseInt(requiredVersion.getMajorVersion()),
+                Integer.parseInt(actualVersion.getMajorVersion())
+            );
+        }
+
+        return actualVersion;
+    }
+
+    private JavaInstallation getJavaInstallation(File javaHome) {
+        JavaInstallation installation;
+        if (isCurrentJavaHome(javaHome)) {
+            installation = javaInstallationRegistry.getInstallationForCurrentVirtualMachine().get();
+        } else {
+            installation = javaInstallationRegistry.installationForDirectory(objects.directoryProperty().fileValue(javaHome)).get();
+        }
+
+        return installation;
+    }
+
+    private List<JavaHome> getAvailableJavaVersions(JavaVersion minimumCompilerVersion) {
+        final List<JavaHome> javaVersions = new ArrayList<>();
+        for (int v = 8; v <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); v++) {
+            int version = v;
+            String javaHomeEnvVarName = getJavaHomeEnvVarName(Integer.toString(version));
+            if (System.getenv(javaHomeEnvVarName) != null) {
+                File javaHomeDirectory = new File(findJavaHome(Integer.toString(version)));
+                Provider<JavaInstallation> javaInstallationProvider = javaInstallationRegistry.installationForDirectory(
+                    objects.directoryProperty().fileValue(javaHomeDirectory)
+                );
+                JavaHome javaHome = JavaHome.of(version, providers.provider(() -> {
+                    int actualVersion = Integer.parseInt(javaInstallationProvider.get().getJavaVersion().getMajorVersion());
+                    if (actualVersion != version) {
+                        throwInvalidJavaHomeException("env variable " + javaHomeEnvVarName, javaHomeDirectory, version, actualVersion);
                     }
-                });
+                    return javaHomeDirectory;
+                }));
+                javaVersions.add(javaHome);
             }
+        }
+        return javaVersions;
+    }
+
+    private static boolean isCurrentJavaHome(File javaHome) {
+        try {
+            return Files.isSameFile(javaHome.toPath(), Jvm.current().getJavaHome().toPath());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static String getTestSeed() {
+        String testSeedProperty = System.getProperty("tests.seed");
+        final String testSeed;
+        if (testSeedProperty == null) {
+            long seed = new Random(System.currentTimeMillis()).nextLong();
+            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT);
+        } else {
+            testSeed = testSeedProperty;
+        }
+        return testSeed;
+    }
+
+    private static void throwInvalidJavaHomeException(String description, File javaHome, int expectedVersion, int actualVersion) {
+        String message = String.format(
+            Locale.ROOT,
+            "The %s must be set to a JDK installation directory for Java %d but is [%s] corresponding to [%s]",
+            description,
+            expectedVersion,
+            javaHome,
+            actualVersion
         );
+
+        throw new GradleException(message);
     }
 
     private static File findCompilerJavaHome() {
@@ -174,28 +249,6 @@ private static String getJavaHomeEnvVarName(String version) {
         return "JAVA" + version + "_HOME";
     }
 
-    private static boolean isInFipsJvm() {
-        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
-    }
-
-    private static String getResourceContents(String resourcePath) {
-        try (
-            BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))
-        ) {
-            StringBuilder b = new StringBuilder();
-            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
-                if (b.length() != 0) {
-                    b.append('\n');
-                }
-                b.append(line);
-            }
-
-            return b.toString();
-        } catch (IOException e) {
-            throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e);
-        }
-    }
-
     private static int findDefaultParallel(Project project) {
         // Since it costs IO to compute this, and is done at configuration time we want to cache this if possible
         // It's safe to store this in a static variable since it's just a primitive so leaking memory isn't an issue
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java
deleted file mode 100644
index a2daa4a5767c0..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.elasticsearch.gradle.info;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class GlobalInfoExtension {
-    final List<Runnable> listeners = new ArrayList<>();
-
-    public void ready(Runnable block) {
-        listeners.add(block);
-    }
-}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
deleted file mode 100644
index fdd79e0fc9cb3..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
+++ /dev/null
@@ -1,75 +0,0 @@
-package org.elasticsearch.gradle.info;
-
-import org.gradle.api.DefaultTask;
-import org.gradle.api.JavaVersion;
-import org.gradle.api.file.RegularFileProperty;
-import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.resources.TextResource;
-import org.gradle.api.tasks.InputFile;
-import org.gradle.api.tasks.TaskAction;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.List;
-
-public class PrintGlobalBuildInfoTask extends DefaultTask {
-    private final RegularFileProperty buildInfoFile;
-    private final RegularFileProperty compilerVersionFile;
-    private final RegularFileProperty runtimeVersionFile;
-    private List<Runnable> globalInfoListeners = new ArrayList<>();
-
-    @Inject
-    public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) {
-        this.buildInfoFile = objectFactory.fileProperty();
-        this.compilerVersionFile = objectFactory.fileProperty();
-        this.runtimeVersionFile = objectFactory.fileProperty();
-    }
-
-    @InputFile
-    public RegularFileProperty getBuildInfoFile() {
-        return buildInfoFile;
-    }
-
-    @InputFile
-    public RegularFileProperty getCompilerVersionFile() {
-        return compilerVersionFile;
-    }
-
-    @InputFile
-    public RegularFileProperty getRuntimeVersionFile() {
-        return runtimeVersionFile;
-    }
-
-    public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
-        this.globalInfoListeners = globalInfoListeners;
-    }
-
-    @TaskAction
-    public void print() {
-        getLogger().quiet("=======================================");
-        getLogger().quiet("Elasticsearch Build Hamster says Hello!");
-        getLogger().quiet(getFileText(getBuildInfoFile()).asString());
-        getLogger().quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
-        getLogger().quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
-        getLogger().quiet("=======================================");
-
-        setGlobalProperties();
-        globalInfoListeners.forEach(Runnable::run);
-
-        // Since all tasks depend on this task, and it always runs for every build, this makes sure that lifecycle tasks will still
-        // correctly report as UP-TO-DATE, since the convention is a lifecycle task (i.e. assemble, build, etc) will only be marked as
-        // UP-TO-DATE if all upstream tasks were also UP-TO-DATE.
-        setDidWork(false);
-    }
-
-    private TextResource getFileText(RegularFileProperty regularFileProperty) {
-        return getProject().getResources().getText().fromFile(regularFileProperty.getAsFile().get());
-    }
-
-    private void setGlobalProperties() {
-        BuildParams.init(params -> {
-            params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
-            params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
-        });
-    }
-}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java
index 68839f86c1035..35ad889b34125 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java
@@ -28,7 +28,7 @@
 import java.util.stream.Collectors;
 
 import org.apache.tools.ant.taskdefs.condition.Os;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.file.FileCollection;
@@ -80,7 +80,7 @@ private static boolean isExecutableFile(File file) {
     @InputFiles
     @SkipWhenEmpty
     public FileCollection getFiles() {
-        return Boilerplate.getJavaSourceSets(getProject())
+        return GradleUtils.getJavaSourceSets(getProject())
             .stream()
             .map(sourceSet -> sourceSet.getAllSource().matching(filesFilter))
             .reduce(FileTree::plus)
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
index a662ff4dbcaeb..3657808b91979 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.gradle.precommit;
 
 import groovy.lang.Closure;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Task;
@@ -65,7 +65,7 @@ public class TestingConventionsTasks extends DefaultTask {
     public TestingConventionsTasks() {
         setDescription("Tests various testing conventions");
         // Run only after everything is compiled
-        Boilerplate.getJavaSourceSets(getProject()).all(sourceSet -> dependsOn(sourceSet.getOutput().getClassesDirs()));
+        GradleUtils.getJavaSourceSets(getProject()).all(sourceSet -> dependsOn(sourceSet.getOutput().getClassesDirs()));
         naming = getProject().container(TestingConventionRule.class);
     }
 
@@ -81,7 +81,7 @@ public Map<String, Set<File>> getClassFilesPerEnabledTask() {
     @Input
     public Map<String, File> getTestClassNames() {
         if (testClassNames == null) {
-            testClassNames = Boilerplate.getJavaSourceSets(getProject())
+            testClassNames = GradleUtils.getJavaSourceSets(getProject())
                 .getByName("test")
                 .getOutput()
                 .getClassesDirs()
@@ -110,7 +110,7 @@ public void naming(Closure<TestingConventionRule> action) {
 
     @Input
     public Set<String> getMainClassNamedLikeTests() {
-        SourceSetContainer javaSourceSets = Boilerplate.getJavaSourceSets(getProject());
+        SourceSetContainer javaSourceSets = GradleUtils.getJavaSourceSets(getProject());
         if (javaSourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME) == null) {
             // some test projects don't have a main source set
             return Collections.emptySet();
@@ -351,7 +351,7 @@ private FileCollection getTestsClassPath() {
         // running the tests.
         return getProject().files(
             getProject().getConfigurations().getByName("testRuntime").resolve(),
-            Boilerplate.getJavaSourceSets(getProject())
+            GradleUtils.getJavaSourceSets(getProject())
                 .stream()
                 .flatMap(sourceSet -> sourceSet.getOutput().getClassesDirs().getFiles().stream())
                 .collect(Collectors.toList())
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
index b09441148558f..9826c50836e11 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.gradle.docker.DockerSupportPlugin;
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
 import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
 import org.elasticsearch.gradle.vagrant.VagrantExtension;
@@ -90,7 +90,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(DistributionDownloadPlugin.class);
         project.getPluginManager().apply("elasticsearch.build");
 
-        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+        Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index 1cb3034419bbe..a059400844c3d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -20,7 +20,7 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
@@ -203,6 +203,6 @@ private File getTestOutputResourceDir() {
     }
 
     private SourceSet getTestSourceSet() {
-        return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
+        return GradleUtils.getJavaSourceSets(getProject()).findByName("test");
     }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 4c86b86b56f5b..0d067b40edb29 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -20,7 +20,7 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
@@ -146,6 +146,6 @@ void copy() {
     }
 
     private SourceSet getTestSourceSet() {
-        return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
+        return GradleUtils.getJavaSourceSets(getProject()).findByName("test");
     }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java
index 5cd88ea01a32a..62427bc1d0a66 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java
@@ -1,6 +1,6 @@
 package org.elasticsearch.gradle.testclusters;
 
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.services.internal.BuildServiceRegistryInternal;
 import org.gradle.api.tasks.CacheableTask;
@@ -62,7 +62,7 @@ public Collection<ElasticsearchCluster> getClusters() {
     public List<ResourceLock> getSharedResources() {
         List<ResourceLock> locks = new ArrayList<>(super.getSharedResources());
         BuildServiceRegistryInternal serviceRegistry = getServices().get(BuildServiceRegistryInternal.class);
-        Provider<TestClustersThrottle> throttleProvider = Boilerplate.getBuildService(serviceRegistry, THROTTLE_SERVICE_NAME);
+        Provider<TestClustersThrottle> throttleProvider = GradleUtils.getBuildService(serviceRegistry, THROTTLE_SERVICE_NAME);
         SharedResource resource = serviceRegistry.forService(throttleProvider);
 
         int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
index ebdf24d0a318f..48c436c42e03b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.ReaperPlugin;
 import org.elasticsearch.gradle.ReaperService;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -36,7 +36,7 @@
 
 import java.io.File;
 
-import static org.elasticsearch.gradle.tool.Boilerplate.noop;
+import static org.elasticsearch.gradle.util.GradleUtils.noop;
 
 public class TestClustersPlugin implements Plugin<Project> {
 
@@ -102,7 +102,7 @@ public void apply(Project project) {
                 throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
             }
 
-            Provider<TestClustersRegistry> registryProvider = Boilerplate.getBuildService(
+            Provider<TestClustersRegistry> registryProvider = GradleUtils.getBuildService(
                 project.getGradle().getSharedServices(),
                 REGISTRY_SERVICE_NAME
             );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
index 64ea96e9f8827..88e744043b747 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.precommit.TestingConventionsTasks;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Action;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Plugin;
@@ -67,7 +67,7 @@ public void apply(Project project) {
             .getSharedServices()
             .registerIfAbsent(DOCKER_COMPOSE_THROTTLE, DockerComposeThrottle.class, spec -> spec.getMaxParallelUsages().set(1));
 
-        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+        Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
similarity index 98%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
rename to buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
index 85002b9c49821..81de7ea8176c3 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.tool;
+package org.elasticsearch.gradle.util;
 
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
@@ -36,7 +36,7 @@
 
 import java.util.Optional;
 
-public abstract class Boilerplate {
+public abstract class GradleUtils {
 
     public static <T> Action<T> noop() {
         return t -> {};
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java
similarity index 60%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/Util.java
rename to buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java
index 9ca4efc8b9f37..03ebe906625b6 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java
@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -17,10 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle;
+package org.elasticsearch.gradle.util;
 
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
 import org.gradle.api.GradleException;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
 import java.util.Locale;
 
 public class Util {
@@ -39,6 +44,24 @@ public static boolean getBooleanProperty(String property, boolean defaultValue)
         }
     }
 
+    public static String getResourceContents(String resourcePath) {
+        try (
+            BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))
+        ) {
+            StringBuilder b = new StringBuilder();
+            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+                if (b.length() != 0) {
+                    b.append('\n');
+                }
+                b.append(line);
+            }
+
+            return b.toString();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e);
+        }
+    }
+
     public static String capitalize(String s) {
         return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
     }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java
index 945a5d4ed0374..507b977569c0c 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.gradle.LoggedExec;
 import org.elasticsearch.gradle.LoggingOutputStream;
 import org.elasticsearch.gradle.ReaperService;
-import org.elasticsearch.gradle.Util;
+import org.elasticsearch.gradle.util.Util;
 import org.gradle.api.Action;
 import org.gradle.api.Project;
 import org.gradle.internal.logging.progress.ProgressLogger;
diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
index a76413d6b47d4..624a3e6f5af61 100644
--- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
+++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
@@ -3,11 +3,6 @@
 import org.gradle.api.JavaVersion;
 
 import java.io.File;
-import java.lang.annotation.Documented;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
 import java.lang.reflect.Modifier;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
@@ -75,12 +70,10 @@ public static JavaVersion getGradleJavaVersion() {
         return value(gradleJavaVersion);
     }
 
-    @ExecutionTime
     public static JavaVersion getCompilerJavaVersion() {
         return value(compilerJavaVersion);
     }
 
-    @ExecutionTime
     public static JavaVersion getRuntimeJavaVersion() {
         return value(runtimeJavaVersion);
     }
@@ -120,22 +113,13 @@ public static boolean isSnapshotBuild() {
     private static <T> T value(T object) {
         if (object == null) {
             String callingMethod = Thread.currentThread().getStackTrace()[2].getMethodName();
-            boolean executionTime;
-            try {
-                executionTime = BuildParams.class.getMethod(callingMethod).getAnnotation(ExecutionTime.class) != null;
-            } catch (NoSuchMethodException e) {
-                throw new RuntimeException(e);
-            }
 
-            String message = "Build parameter '" + propertyName(callingMethod) + "' has not been initialized. ";
-            if (executionTime) {
-                message += "This property is initialized at execution time, "
-                    + "please ensure you are not attempting to access it during project configuration.";
-            } else {
-                message += "Perhaps the plugin responsible for initializing this property has not been applied.";
-            }
-
-            throw new IllegalStateException(message);
+            throw new IllegalStateException(
+                "Build parameter '"
+                    + propertyName(callingMethod)
+                    + "' has not been initialized.\n"
+                    + "Perhaps the plugin responsible for initializing this property has not been applied."
+            );
         }
 
         return object;
@@ -236,14 +220,4 @@ public void setIsSnapshotBuild(final boolean isSnapshotBuild) {
         }
 
     }
-
-    /**
-     * Indicates that a build parameter is initialized at task execution time and is not available at project configuration time.
-     * Attempts to read an uninitialized parameter wil result in an {@link IllegalStateException}.
-     */
-    @Target({ ElementType.METHOD, ElementType.FIELD })
-    @Retention(RetentionPolicy.RUNTIME)
-    @Documented
-    public @interface ExecutionTime {
-    }
 }
diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java
index 29ca2bafc79dc..1fe376545eaac 100644
--- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java
+++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java
@@ -1,35 +1,27 @@
 package org.elasticsearch.gradle.info;
 
-import org.gradle.api.tasks.Input;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.Optional;
-import org.gradle.api.tasks.PathSensitive;
-import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.provider.Provider;
 
 import java.io.File;
 
 public class JavaHome {
     private Integer version;
-    private File javaHome;
+    private Provider<File> javaHome;
 
-    private JavaHome(int version, File javaHome) {
+    private JavaHome(int version, Provider<File> javaHome) {
         this.version = version;
         this.javaHome = javaHome;
     }
 
-    public static JavaHome of(int version, File javaHome) {
+    public static JavaHome of(int version, Provider<File> javaHome) {
         return new JavaHome(version, javaHome);
     }
 
-    @Input
     public Integer getVersion() {
         return version;
     }
 
-    @InputDirectory
-    @Optional
-    @PathSensitive(PathSensitivity.RELATIVE)
-    public File getJavaHome() {
+    public Provider<File> getJavaHome() {
         return javaHome;
     }
 }
diff --git a/gradle/ide.gradle b/gradle/ide.gradle
index b3d866d8342b8..62c0caa37ec42 100644
--- a/gradle/ide.gradle
+++ b/gradle/ide.gradle
@@ -108,7 +108,7 @@ idea {
                not use this file except in compliance with the License.
                You may obtain a copy of the License at
 
-                       http://www.apache.org/licenses/LICENSE-2.0
+                   http://www.apache.org/licenses/LICENSE-2.0
 
                Unless required by applicable law or agreed to in writing,
                software distributed under the License is distributed on an
diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle
index 58ab5515b14cb..969fdba727ea8 100644
--- a/x-pack/qa/reindex-tests-with-security/build.gradle
+++ b/x-pack/qa/reindex-tests-with-security/build.gradle
@@ -39,10 +39,8 @@ testClusters.integTest {
   setting 'reindex.ssl.truststore.password', 'password'
 
   // Workaround for JDK-8212885
-  rootProject.globalInfo.ready {
-    if (BuildParams.runtimeJavaVersion.isJava12Compatible() == false) {
-      setting 'reindex.ssl.supported_protocols', 'TLSv1.2'
-    }
+  if (BuildParams.runtimeJavaVersion.isJava12Compatible() == false) {
+    setting 'reindex.ssl.supported_protocols', 'TLSv1.2'
   }
 
   extraConfigFile 'roles.yml', file('roles.yml')

From 756a297ea6d5e8070ebb133723056e6ece18ff9c Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Mon, 23 Mar 2020 16:53:01 -0400
Subject: [PATCH 241/285] [ML] adds multi-class feature importance support
 (#53803)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds multi-class feature importance calculation.

Feature importance objects are now mapped as follows
(logistic) Regression:
```
{
   "feature_name": "feature_0",
   "importance": -1.3
}
```
Multi-class [class names are `foo`, `bar`, `baz`]
```
{
   “feature_name”: “feature_0”,
   “importance”: 2.0, // sum(abs()) of class importances
   “foo”: 1.0,
   “bar”: 0.5,
   “baz”: -0.5
},
```

For users to get the full benefit of aggregating and searching for feature importance, they should update their index mapping as follows (before turning this option on in their pipelines)
```
 "ml.inference.feature_importance": {
          "type": "nested",
          "dynamic": true,
          "properties": {
            "feature_name": {
              "type": "keyword"
            },
            "importance": {
              "type": "double"
            }
          }
        }
```
The mapping field name is as follows
`ml.<inference.target_field>.<inference.tag>.feature_importance`
if `inference.tag` is not provided in the processor definition, it is not part of the field path.
`inference.target_field` is defaulted to `ml.inference`.
//cc @lcawl ^ Where should we document this?

If this makes it in for 7.7, there shouldn't be any feature_importance at inference BWC worries as 7.7 is the first version to have it.
---
 .../ClassificationInferenceResults.java       | 11 ++-
 .../inference/results/FeatureImportance.java  | 97 +++++++++++++++++++
 .../results/RawInferenceResults.java          |  6 +-
 .../results/RegressionInferenceResults.java   | 14 ++-
 .../results/SingleValueInferenceResults.java  | 31 +++---
 .../trainedmodel/InferenceHelpers.java        | 40 +++++++-
 .../inference/trainedmodel/TrainedModel.java  |  4 +-
 .../trainedmodel/ensemble/Ensemble.java       | 37 ++++---
 .../langident/LangIdentNeuralNetwork.java     |  4 +-
 .../ml/inference/trainedmodel/tree/Tree.java  | 73 +++++++-------
 .../ClassificationInferenceResultsTests.java  | 44 +++++++++
 .../results/FeatureImportanceTests.java       | 44 +++++++++
 .../results/RawInferenceResultsTests.java     |  7 +-
 .../RegressionInferenceResultsTests.java      | 39 +++++++-
 .../trainedmodel/ensemble/EnsembleTests.java  | 42 ++++----
 .../trainedmodel/tree/TreeTests.java          | 18 ++--
 .../ml/integration/InferenceIngestIT.java     | 12 ++-
 .../ingest/InferenceProcessorTests.java       | 25 +++--
 18 files changed, 410 insertions(+), 138 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
index a354ae74bf23c..d196506adc187 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
@@ -35,13 +35,13 @@ public ClassificationInferenceResults(double value,
                                           String classificationLabel,
                                           List<TopClassEntry> topClasses,
                                           InferenceConfig config) {
-        this(value, classificationLabel, topClasses, Collections.emptyMap(), (ClassificationConfig)config);
+        this(value, classificationLabel, topClasses, Collections.emptyList(), (ClassificationConfig)config);
     }
 
     public ClassificationInferenceResults(double value,
                                           String classificationLabel,
                                           List<TopClassEntry> topClasses,
-                                          Map<String, Double> featureImportance,
+                                          List<FeatureImportance> featureImportance,
                                           InferenceConfig config) {
         this(value, classificationLabel, topClasses, featureImportance, (ClassificationConfig)config);
     }
@@ -49,7 +49,7 @@ public ClassificationInferenceResults(double value,
     private ClassificationInferenceResults(double value,
                                            String classificationLabel,
                                            List<TopClassEntry> topClasses,
-                                           Map<String, Double> featureImportance,
+                                           List<FeatureImportance> featureImportance,
                                            ClassificationConfig classificationConfig) {
         super(value,
             SingleValueInferenceResults.takeTopFeatureImportances(featureImportance,
@@ -118,7 +118,10 @@ public void writeResult(IngestDocument document, String parentResultField) {
                 topClasses.stream().map(TopClassEntry::asValueMap).collect(Collectors.toList()));
         }
         if (getFeatureImportance().size() > 0) {
-            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance());
+            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance()
+                .stream()
+                .map(FeatureImportance::toMap)
+                .collect(Collectors.toList()));
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java
new file mode 100644
index 0000000000000..a6576a25e3141
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.inference.results;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class FeatureImportance implements Writeable {
+
+    private final Map<String, Double> classImportance;
+    private final double importance;
+    private final String featureName;
+    private static final String IMPORTANCE = "importance";
+    private static final String FEATURE_NAME = "feature_name";
+
+    public static FeatureImportance forRegression(String featureName, double importance) {
+        return new FeatureImportance(featureName, importance, null);
+    }
+
+    public static FeatureImportance forClassification(String featureName, Map<String, Double> classImportance) {
+        return new FeatureImportance(featureName, classImportance.values().stream().mapToDouble(Math::abs).sum(), classImportance);
+    }
+
+    private FeatureImportance(String featureName, double importance, Map<String, Double> classImportance) {
+        this.featureName = Objects.requireNonNull(featureName);
+        this.importance = importance;
+        this.classImportance = classImportance == null ? null : Collections.unmodifiableMap(classImportance);
+    }
+
+    public FeatureImportance(StreamInput in) throws IOException {
+        this.featureName = in.readString();
+        this.importance = in.readDouble();
+        if (in.readBoolean()) {
+            this.classImportance = in.readMap(StreamInput::readString, StreamInput::readDouble);
+        } else {
+            this.classImportance = null;
+        }
+    }
+
+    public Map<String, Double> getClassImportance() {
+        return classImportance;
+    }
+
+    public double getImportance() {
+        return importance;
+    }
+
+    public String getFeatureName() {
+        return featureName;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(this.featureName);
+        out.writeDouble(this.importance);
+        out.writeBoolean(this.classImportance != null);
+        if (this.classImportance != null) {
+            out.writeMap(this.classImportance, StreamOutput::writeString, StreamOutput::writeDouble);
+        }
+    }
+
+    public Map<String, Object> toMap() {
+        Map<String, Object> map = new LinkedHashMap<>();
+        map.put(FEATURE_NAME, featureName);
+        map.put(IMPORTANCE, importance);
+        if (classImportance != null) {
+            classImportance.forEach(map::put);
+        }
+        return map;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) { return true; }
+        if (object == null || getClass() != object.getClass()) { return false; }
+        FeatureImportance that = (FeatureImportance) object;
+        return Objects.equals(featureName, that.featureName)
+            && Objects.equals(importance, that.importance)
+            && Objects.equals(classImportance, that.classImportance);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(featureName, importance, classImportance);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java
index 2c166e39f8067..216d708a6b471 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java
@@ -18,9 +18,9 @@ public class RawInferenceResults implements InferenceResults {
     public static final String NAME = "raw";
 
     private final double[] value;
-    private final Map<String, Double> featureImportance;
+    private final Map<String, double[]> featureImportance;
 
-    public RawInferenceResults(double[] value, Map<String, Double> featureImportance) {
+    public RawInferenceResults(double[] value, Map<String, double[]> featureImportance) {
         this.value = value;
         this.featureImportance = featureImportance;
     }
@@ -29,7 +29,7 @@ public double[] getValue() {
         return value;
     }
 
-    public Map<String, Double> getFeatureImportance() {
+    public Map<String, double[]> getFeatureImportance() {
         return featureImportance;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java
index a0647b8dffa2d..985b1233cede8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java
@@ -14,8 +14,9 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Map;
+import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class RegressionInferenceResults extends SingleValueInferenceResults {
 
@@ -24,14 +25,14 @@ public class RegressionInferenceResults extends SingleValueInferenceResults {
     private final String resultsField;
 
     public RegressionInferenceResults(double value, InferenceConfig config) {
-        this(value, (RegressionConfig) config, Collections.emptyMap());
+        this(value, (RegressionConfig) config, Collections.emptyList());
     }
 
-    public RegressionInferenceResults(double value, InferenceConfig config, Map<String, Double> featureImportance) {
+    public RegressionInferenceResults(double value, InferenceConfig config, List<FeatureImportance> featureImportance) {
         this(value, (RegressionConfig)config, featureImportance);
     }
 
-    private RegressionInferenceResults(double value, RegressionConfig regressionConfig, Map<String, Double> featureImportance) {
+    private RegressionInferenceResults(double value, RegressionConfig regressionConfig, List<FeatureImportance> featureImportance) {
         super(value,
             SingleValueInferenceResults.takeTopFeatureImportances(featureImportance,
                 regressionConfig.getNumTopFeatureImportanceValues()));
@@ -70,7 +71,10 @@ public void writeResult(IngestDocument document, String parentResultField) {
         ExceptionsHelper.requireNonNull(parentResultField, "parentResultField");
         document.setFieldValue(parentResultField + "." + this.resultsField, value());
         if (getFeatureImportance().size() > 0) {
-            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance());
+            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance()
+                .stream()
+                .map(FeatureImportance::toMap)
+                .collect(Collectors.toList()));
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java
index cd739b557abea..249a966ff6294 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java
@@ -8,45 +8,46 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public abstract class SingleValueInferenceResults implements InferenceResults {
 
     private final double value;
-    private final Map<String, Double> featureImportance;
+    private final List<FeatureImportance> featureImportance;
 
-    static Map<String, Double> takeTopFeatureImportances(Map<String, Double> unsortedFeatureImportances, int numTopFeatures) {
-        return unsortedFeatureImportances.entrySet()
-            .stream()
-            .sorted((l, r)-> Double.compare(Math.abs(r.getValue()), Math.abs(l.getValue())))
+    static List<FeatureImportance> takeTopFeatureImportances(List<FeatureImportance> unsortedFeatureImportances, int numTopFeatures) {
+        if (unsortedFeatureImportances == null || unsortedFeatureImportances.isEmpty()) {
+            return unsortedFeatureImportances;
+        }
+        return unsortedFeatureImportances.stream()
+            .sorted((l, r)-> Double.compare(Math.abs(r.getImportance()), Math.abs(l.getImportance())))
             .limit(numTopFeatures)
-            .collect(LinkedHashMap::new, (h, e) -> h.put(e.getKey(), e.getValue()) , LinkedHashMap::putAll);
+            .collect(Collectors.toList());
     }
 
     SingleValueInferenceResults(StreamInput in) throws IOException {
         value = in.readDouble();
         if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            this.featureImportance = in.readMap(StreamInput::readString, StreamInput::readDouble);
+            this.featureImportance = in.readList(FeatureImportance::new);
         } else {
-            this.featureImportance = Collections.emptyMap();
+            this.featureImportance = Collections.emptyList();
         }
     }
 
-    SingleValueInferenceResults(double value, Map<String, Double> featureImportance) {
+    SingleValueInferenceResults(double value, List<FeatureImportance> featureImportance) {
         this.value = value;
-        this.featureImportance = ExceptionsHelper.requireNonNull(featureImportance, "featureImportance");
+        this.featureImportance = featureImportance == null ? Collections.emptyList() : featureImportance;
     }
 
     public Double value() {
         return value;
     }
 
-    public Map<String, Double> getFeatureImportance() {
+    public List<FeatureImportance> getFeatureImportance() {
         return featureImportance;
     }
 
@@ -58,7 +59,7 @@ public String valueAsString() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeDouble(value);
         if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-            out.writeMap(this.featureImportance, StreamOutput::writeString, StreamOutput::writeDouble);
+            out.writeList(this.featureImportance);
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java
index 52cabb4f7b9f3..05b0693ac04c5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java
@@ -8,12 +8,14 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.FeatureImportance;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -100,18 +102,46 @@ public static Double toDouble(Object value) {
         return null;
     }
 
-    public static Map<String, Double> decodeFeatureImportances(Map<String, String> processedFeatureToOriginalFeatureMap,
-                                                               Map<String, Double> featureImportances) {
+    public static Map<String, double[]> decodeFeatureImportances(Map<String, String> processedFeatureToOriginalFeatureMap,
+                                                                 Map<String, double[]> featureImportances) {
         if (processedFeatureToOriginalFeatureMap == null || processedFeatureToOriginalFeatureMap.isEmpty()) {
             return featureImportances;
         }
 
-        Map<String, Double> originalFeatureImportance = new HashMap<>();
+        Map<String, double[]> originalFeatureImportance = new HashMap<>();
         featureImportances.forEach((feature, importance) -> {
             String featureName = processedFeatureToOriginalFeatureMap.getOrDefault(feature, feature);
-            originalFeatureImportance.compute(featureName, (f, v1) -> v1 == null ? importance : v1 + importance);
+            originalFeatureImportance.compute(featureName, (f, v1) -> v1 == null ? importance : sumDoubleArrays(importance, v1));
         });
-
         return originalFeatureImportance;
     }
+
+    public static List<FeatureImportance> transformFeatureImportance(Map<String, double[]> featureImportance,
+                                                                     @Nullable List<String> classificationLabels) {
+        List<FeatureImportance> importances = new ArrayList<>(featureImportance.size());
+        featureImportance.forEach((k, v) -> {
+            // This indicates regression, or logistic regression
+            // If the length > 1, we assume multi-class classification.
+            if (v.length == 1) {
+                importances.add(FeatureImportance.forRegression(k, v[0]));
+            } else {
+                Map<String, Double> classImportance = new LinkedHashMap<>(v.length, 1.0f);
+                // If the classificationLabels exist, their length must match leaf_value length
+                assert classificationLabels == null || classificationLabels.size() == v.length;
+                for (int i = 0; i < v.length; i++) {
+                    classImportance.put(classificationLabels == null ? String.valueOf(i) : classificationLabels.get(i), v[i]);
+                }
+                importances.add(FeatureImportance.forClassification(k, classImportance));
+            }
+        });
+        return importances;
+    }
+
+    public static double[] sumDoubleArrays(double[] sumTo, double[] inc) {
+        assert sumTo != null && inc != null && sumTo.length == inc.length;
+        for (int i = 0; i < inc.length; i++) {
+            sumTo[i] += inc[i];
+        }
+        return sumTo;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java
index 8901c62bb00e8..33cff4d0bb47b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java
@@ -60,9 +60,9 @@ public interface TrainedModel extends NamedXContentObject, NamedWriteable, Accou
      * NOTE: Must be thread safe
      * @param fields The fields inferring against
      * @param featureDecoder A Map translating processed feature names to their original feature names
-     * @return A {@code Map<String, Double>} mapping each featureName to its importance
+     * @return A {@code Map<String, double[]>} mapping each featureName to its importance
      */
-    Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder);
+    Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder);
 
     default Version getMinimalCompatibilityVersion() {
         return Version.V_7_6_0;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
index 5955ecda6f276..7ee3ef1668b82 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -45,6 +45,8 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceHelpers.classificationLabel;
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceHelpers.decodeFeatureImportances;
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceHelpers.transformFeatureImportance;
 
 public class Ensemble implements LenientlyParsedTrainedModel, StrictlyParsedTrainedModel {
 
@@ -139,7 +141,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
                 "Cannot infer using configuration for [{}] when model target_type is [{}]", config.getName(), targetType.toString());
         }
         double[][] inferenceResults = new double[this.models.size()][];
-        List<Map<String, Double>> featureInfluence = new ArrayList<>();
+        List<Map<String, double[]>> featureInfluence = new ArrayList<>();
         int i = 0;
         NullInferenceConfig subModelInferenceConfig = new NullInferenceConfig(config.requestingImportance());
         for (TrainedModel model : models) {
@@ -152,7 +154,9 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
             }
         }
         double[] processed = outputAggregator.processValues(inferenceResults);
-        return buildResults(processed, featureInfluence, config, featureDecoderMap);
+        return buildResults(processed,
+            decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)),
+            config);
     }
 
     @Override
@@ -161,19 +165,19 @@ public TargetType targetType() {
     }
 
     private InferenceResults buildResults(double[] processedInferences,
-                                          List<Map<String, Double>> featureInfluence,
-                                          InferenceConfig config,
-                                          Map<String, String> featureDecoderMap) {
+                                          Map<String, double[]> featureInfluence,
+                                          InferenceConfig config) {
         // Indicates that the config is useless and the caller just wants the raw value
         if (config instanceof NullInferenceConfig) {
-            return new RawInferenceResults(new double[] {outputAggregator.aggregate(processedInferences)},
-                InferenceHelpers.decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)));
+            return new RawInferenceResults(
+                new double[] {outputAggregator.aggregate(processedInferences)},
+                featureInfluence);
         }
         switch(targetType) {
             case REGRESSION:
                 return new RegressionInferenceResults(outputAggregator.aggregate(processedInferences),
                     config,
-                    InferenceHelpers.decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)));
+                    transformFeatureImportance(featureInfluence, null));
             case CLASSIFICATION:
                 ClassificationConfig classificationConfig = (ClassificationConfig) config;
                 assert classificationWeights == null || processedInferences.length == classificationWeights.length;
@@ -186,7 +190,7 @@ private InferenceResults buildResults(double[] processedInferences,
                 return new ClassificationInferenceResults((double)topClasses.v1(),
                     classificationLabel(topClasses.v1(), classificationLabels),
                     topClasses.v2(),
-                    InferenceHelpers.decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)),
+                    transformFeatureImportance(featureInfluence, classificationLabels),
                     config);
             default:
                 throw new UnsupportedOperationException("unsupported target_type [" + targetType + "] for inference on ensemble model");
@@ -313,20 +317,23 @@ public boolean supportsFeatureImportance() {
         return models.stream().allMatch(TrainedModel::supportsFeatureImportance);
     }
 
-    Map<String, Double> featureImportance(Map<String, Object> fields) {
+    Map<String, double[]> featureImportance(Map<String, Object> fields) {
         return featureImportance(fields, Collections.emptyMap());
     }
 
     @Override
-    public Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
-        Map<String, Double> collapsed = mergeFeatureImportances(models.stream()
+    public Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
+        Map<String, double[]> collapsed = mergeFeatureImportances(models.stream()
             .map(trainedModel -> trainedModel.featureImportance(fields, Collections.emptyMap()))
             .collect(Collectors.toList()));
-        return InferenceHelpers.decodeFeatureImportances(featureDecoder, collapsed);
+        return decodeFeatureImportances(featureDecoder, collapsed);
     }
 
-    private static Map<String, Double> mergeFeatureImportances(List<Map<String, Double>> featureImportances) {
-        return featureImportances.stream().collect(HashMap::new, (a, b) -> b.forEach((k, v) -> a.merge(k, v, Double::sum)), Map::putAll);
+    private static Map<String, double[]> mergeFeatureImportances(List<Map<String, double[]>> featureImportances) {
+        return featureImportances.stream()
+            .collect(HashMap::new,
+                (a, b) -> b.forEach((k, v) -> a.merge(k, v, InferenceHelpers::sumDoubleArrays)),
+                Map::putAll);
     }
 
     public static Builder builder() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java
index f9a5601f8345a..5d31caba2eff1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java
@@ -142,7 +142,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
         return new ClassificationInferenceResults(topClasses.v1(),
             LANGUAGE_NAMES.get(topClasses.v1()),
             topClasses.v2(),
-            Collections.emptyMap(),
+            Collections.emptyList(),
             classificationConfig);
     }
 
@@ -170,7 +170,7 @@ public boolean supportsFeatureImportance() {
     }
 
     @Override
-    public Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
+    public Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
         throw new UnsupportedOperationException("[lang_ident] does not support feature importance");
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java
index e59a7299645bc..ddb1d6027ef6e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java
@@ -91,8 +91,8 @@ public static Tree fromXContentLenient(XContentParser parser) {
     private final List<String> classificationLabels;
     private final CachedSupplier<Double> highestOrderCategory;
     // populated lazily when feature importance is calculated
-    private double[] nodeEstimates;
     private Integer maxDepth;
+    private Integer leafSize;
 
     Tree(List<String> featureNames, List<TreeNode> nodes, TargetType targetType, List<String> classificationLabels) {
         this.featureNames = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(featureNames, FEATURE_NAMES));
@@ -137,7 +137,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
             .map(f -> InferenceHelpers.toDouble(MapHelper.dig(f, fields)))
             .collect(Collectors.toList());
 
-        Map<String, Double> featureImportance = config.requestingImportance() ?
+        Map<String, double[]> featureImportance = config.requestingImportance() ?
             featureImportance(features, featureDecoderMap) :
             Collections.emptyMap();
 
@@ -149,7 +149,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
         return buildResult(node.getLeafValue(), featureImportance, config);
     }
 
-    private InferenceResults buildResult(double[] value, Map<String, Double> featureImportance, InferenceConfig config) {
+    private InferenceResults buildResult(double[] value, Map<String, double[]> featureImportance, InferenceConfig config) {
         assert value != null && value.length > 0;
         // Indicates that the config is useless and the caller just wants the raw value
         if (config instanceof NullInferenceConfig) {
@@ -166,10 +166,12 @@ private InferenceResults buildResult(double[] value, Map<String, Double> feature
                 return new ClassificationInferenceResults(topClasses.v1(),
                     classificationLabel(topClasses.v1(), classificationLabels),
                     topClasses.v2(),
-                    featureImportance,
+                    InferenceHelpers.transformFeatureImportance(featureImportance, classificationLabels),
                     config);
             case REGRESSION:
-                return new RegressionInferenceResults(value[0], config, featureImportance);
+                return new RegressionInferenceResults(value[0],
+                    config,
+                    InferenceHelpers.transformFeatureImportance(featureImportance, null));
             default:
                 throw new UnsupportedOperationException("unsupported target_type [" + targetType + "] for inference on tree model");
         }
@@ -283,7 +285,7 @@ public void validate() {
     }
 
     @Override
-    public Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
+    public Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
         if (nodes.stream().allMatch(n -> n.getNumberSamples() == 0)) {
             throw ExceptionsHelper.badRequestException("[tree_structure.number_samples] must be greater than zero for feature importance");
         }
@@ -293,9 +295,12 @@ public Map<String, Double> featureImportance(Map<String, Object> fields, Map<Str
         return featureImportance(features, featureDecoder);
     }
 
-    private Map<String, Double> featureImportance(List<Double> fieldValues, Map<String, String> featureDecoder) {
-        calculateNodeEstimatesIfNeeded();
-        double[] featureImportance = new double[fieldValues.size()];
+    private Map<String, double[]> featureImportance(List<Double> fieldValues, Map<String, String> featureDecoder) {
+        calculateDepthAndLeafValueSize();
+        double[][] featureImportance = new double[fieldValues.size()][leafSize];
+        for (int i = 0; i < fieldValues.size(); i++) {
+            featureImportance[i] = new double[leafSize];
+        }
         int arrSize = ((this.maxDepth + 1) * (this.maxDepth + 2))/2;
         ShapPath.PathElement[] elements = new ShapPath.PathElement[arrSize];
         for (int i = 0; i < arrSize; i++) {
@@ -303,24 +308,22 @@ private Map<String, Double> featureImportance(List<Double> fieldValues, Map<Stri
         }
         double[] scale = new double[arrSize];
         ShapPath initialPath = new ShapPath(elements, scale);
-        shapRecursive(fieldValues, this.nodeEstimates, initialPath, 0, 1.0, 1.0, -1, featureImportance, 0);
+        shapRecursive(fieldValues, initialPath, 0, 1.0, 1.0, -1, featureImportance, 0);
         return InferenceHelpers.decodeFeatureImportances(featureDecoder,
             IntStream.range(0, featureImportance.length)
                 .boxed()
                 .collect(Collectors.toMap(featureNames::get, i -> featureImportance[i])));
     }
 
-    private void calculateNodeEstimatesIfNeeded() {
-        if (this.nodeEstimates != null && this.maxDepth != null) {
+    private void calculateDepthAndLeafValueSize() {
+        if (this.maxDepth != null && this.leafSize != null) {
             return;
         }
         synchronized (this) {
-            if (this.nodeEstimates != null && this.maxDepth != null) {
+            if (this.maxDepth != null && this.leafSize != null) {
                 return;
             }
-            double[] estimates = new double[nodes.size()];
-            this.maxDepth = fillNodeEstimates(estimates, 0, 0);
-            this.nodeEstimates = estimates;
+            this.maxDepth = getDepth(0, 0);
         }
     }
 
@@ -331,23 +334,24 @@ private void calculateNodeEstimatesIfNeeded() {
      * side first and then ported to the Java side.
      */
     private void shapRecursive(List<Double> processedFeatures,
-                               double[] nodeValues,
                                ShapPath parentSplitPath,
                                int nodeIndex,
                                double parentFractionZero,
                                double parentFractionOne,
                                int parentFeatureIndex,
-                               double[] featureImportance,
+                               double[][] featureImportance,
                                int nextIndex) {
         ShapPath splitPath = new ShapPath(parentSplitPath, nextIndex);
         TreeNode currNode = nodes.get(nodeIndex);
         nextIndex = splitPath.extend(parentFractionZero, parentFractionOne, parentFeatureIndex, nextIndex);
         if (currNode.isLeaf()) {
-            double leafValue = nodeValues[nodeIndex];
+            double[] leafValue = currNode.getLeafValue();
             for (int i = 1; i < nextIndex; ++i) {
-                double scale = splitPath.sumUnwoundPath(i, nextIndex);
                 int inputColumnIndex = splitPath.featureIndex(i);
-                featureImportance[inputColumnIndex] += scale * (splitPath.fractionOnes(i) - splitPath.fractionZeros(i)) * leafValue;
+                double scaled = splitPath.sumUnwoundPath(i, nextIndex) * (splitPath.fractionOnes(i) - splitPath.fractionZeros(i));
+                for (int j = 0; j < leafValue.length; j++) {
+                    featureImportance[inputColumnIndex][j] += scaled * leafValue[j];
+                }
             }
         } else {
             int hotIndex = currNode.compare(processedFeatures);
@@ -365,41 +369,32 @@ private void shapRecursive(List<Double> processedFeatures,
 
             double hotFractionZero = nodes.get(hotIndex).getNumberSamples() / (double)currNode.getNumberSamples();
             double coldFractionZero = nodes.get(coldIndex).getNumberSamples() / (double)currNode.getNumberSamples();
-            shapRecursive(processedFeatures, nodeValues, splitPath,
+            shapRecursive(processedFeatures, splitPath,
                 hotIndex, incomingFractionZero * hotFractionZero,
                 incomingFractionOne, splitFeature, featureImportance, nextIndex);
-            shapRecursive(processedFeatures, nodeValues, splitPath,
+            shapRecursive(processedFeatures, splitPath,
                 coldIndex, incomingFractionZero * coldFractionZero,
                 0.0, splitFeature, featureImportance, nextIndex);
         }
     }
 
     /**
-     * This recursively populates the provided {@code double[]} with the node estimated values
+     * Get the depth of the tree and sets leafSize if it is null
      *
-     * Used when calculating feature importance.
-     * @param nodeEstimates Array to update in place with the node estimated values
      * @param nodeIndex Current node index
      * @param depth Current depth
      * @return The current max depth
      */
-    private int fillNodeEstimates(double[] nodeEstimates, int nodeIndex, int depth) {
+    private int getDepth(int nodeIndex, int depth) {
         TreeNode node = nodes.get(nodeIndex);
         if (node.isLeaf()) {
-            // TODO multi-value????
-            nodeEstimates[nodeIndex] = node.getLeafValue()[0];
+            if (leafSize == null) {
+                this.leafSize = node.getLeafValue().length;
+            }
             return 0;
         }
-
-        int depthLeft = fillNodeEstimates(nodeEstimates, node.getLeftChild(), depth + 1);
-        int depthRight = fillNodeEstimates(nodeEstimates, node.getRightChild(), depth + 1);
-        long leftWeight = nodes.get(node.getLeftChild()).getNumberSamples();
-        long rightWeight = nodes.get(node.getRightChild()).getNumberSamples();
-        long divisor = leftWeight + rightWeight;
-        double averageValue = divisor == 0 ?
-            0.0 :
-            (leftWeight * nodeEstimates[node.getLeftChild()] + rightWeight * nodeEstimates[node.getRightChild()]) / divisor;
-        nodeEstimates[nodeIndex] = averageValue;
+        int depthLeft = getDepth(node.getLeftChild(), depth + 1);
+        int depthRight = getDepth(node.getRightChild(), depth + 1);
         return Math.max(depthLeft, depthRight) + 1;
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
index 3ccfb54bd6924..e6ea0398dbc83 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
@@ -16,20 +16,30 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 public class ClassificationInferenceResultsTests extends AbstractWireSerializingTestCase<ClassificationInferenceResults> {
 
     public static ClassificationInferenceResults createRandomResults() {
+        Supplier<FeatureImportance> featureImportanceCtor = randomBoolean() ?
+            FeatureImportanceTests::randomClassification :
+            FeatureImportanceTests::randomRegression;
+
         return new ClassificationInferenceResults(randomDouble(),
             randomBoolean() ? null : randomAlphaOfLength(10),
             randomBoolean() ? null :
                 Stream.generate(ClassificationInferenceResultsTests::createRandomClassEntry)
                     .limit(randomIntBetween(0, 10))
                     .collect(Collectors.toList()),
+            randomBoolean() ? null :
+                Stream.generate(featureImportanceCtor)
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toList()),
             ClassificationConfigTests.randomClassificationConfig());
     }
 
@@ -81,6 +91,40 @@ public void testWriteResultsWithTopClasses() {
         assertThat(document.getFieldValue("result_field.my_results", String.class), equalTo("foo"));
     }
 
+    public void testWriteResultsWithImportance() {
+        Supplier<FeatureImportance> featureImportanceCtor = randomBoolean() ?
+            FeatureImportanceTests::randomClassification :
+            FeatureImportanceTests::randomRegression;
+
+        List<FeatureImportance> importanceList = Stream.generate(featureImportanceCtor)
+            .limit(5)
+            .collect(Collectors.toList());
+        ClassificationInferenceResults result = new ClassificationInferenceResults(0.0,
+            "foo",
+            Collections.emptyList(),
+            importanceList,
+            new ClassificationConfig(0, "predicted_value", "top_classes", 3));
+        IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>());
+        result.writeResult(document, "result_field");
+
+        assertThat(document.getFieldValue("result_field.predicted_value", String.class), equalTo("foo"));
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> writtenImportance = (List<Map<String, Object>>)document.getFieldValue(
+            "result_field.feature_importance",
+            List.class);
+        assertThat(writtenImportance, hasSize(3));
+        importanceList.sort((l, r)-> Double.compare(Math.abs(r.getImportance()), Math.abs(l.getImportance())));
+        for (int i = 0; i < 3; i++) {
+            Map<String, Object> objectMap = writtenImportance.get(i);
+            FeatureImportance importance = importanceList.get(i);
+            assertThat(objectMap.get("feature_name"), equalTo(importance.getFeatureName()));
+            assertThat(objectMap.get("importance"), equalTo(importance.getImportance()));
+            if (importance.getClassImportance() != null) {
+                importance.getClassImportance().forEach((k, v) -> assertThat(objectMap.get(k), equalTo(v)));
+            }
+        }
+    }
+
     @Override
     protected ClassificationInferenceResults createTestInstance() {
         return createRandomResults();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java
new file mode 100644
index 0000000000000..d58171d479b58
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.inference.results;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+
+public class FeatureImportanceTests extends AbstractWireSerializingTestCase<FeatureImportance> {
+
+    public static FeatureImportance createRandomInstance() {
+        return randomBoolean() ? randomClassification() : randomRegression();
+    }
+
+    static FeatureImportance randomRegression() {
+        return FeatureImportance.forRegression(randomAlphaOfLength(10), randomDoubleBetween(-10.0, 10.0, false));
+    }
+
+    static FeatureImportance randomClassification() {
+        return FeatureImportance.forClassification(
+            randomAlphaOfLength(10),
+            Stream.generate(() -> randomAlphaOfLength(10))
+                .limit(randomLongBetween(2, 10))
+                .collect(Collectors.toMap(Function.identity(), (k) -> randomDoubleBetween(-10, 10, false))));
+
+    }
+
+    @Override
+    protected FeatureImportance createTestInstance() {
+        return createRandomInstance();
+    }
+
+    @Override
+    protected Writeable.Reader<FeatureImportance> instanceReader() {
+        return FeatureImportance::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java
index 325768054d0b0..07a89a5bd63f7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java
@@ -22,7 +22,8 @@ public static RawInferenceResults createRandomResults() {
         for (int i = 0; i < n; i++) {
             results[i] = randomDouble();
         }
-        return new RawInferenceResults(results, randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("foo", 1.08));
+        return new RawInferenceResults(results,
+            randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("foo", new double[]{1.08}));
     }
 
     public void testEqualityAndHashcode() {
@@ -31,7 +32,9 @@ public void testEqualityAndHashcode() {
         for (int i = 0; i < n; i++) {
             results[i] = randomDouble();
         }
-        Map<String, Double> importance = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("foo", 1.08);
+        Map<String, double[]> importance = randomBoolean() ?
+            Collections.emptyMap() :
+            Collections.singletonMap("foo", new double[]{1.08, 42.0});
         RawInferenceResults lft = new RawInferenceResults(results, new HashMap<>(importance));
         RawInferenceResults rgt = new RawInferenceResults(Arrays.copyOf(results, n), new HashMap<>(importance));
         assertThat(lft, equalTo(rgt));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
index 5414f97e08783..dbd6a309e8c74 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
@@ -8,19 +8,28 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigTests;
 
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 
 public class RegressionInferenceResultsTests extends AbstractWireSerializingTestCase<RegressionInferenceResults> {
 
     public static RegressionInferenceResults createRandomResults() {
-        return new RegressionInferenceResults(randomDouble(), RegressionConfigTests.randomRegressionConfig());
+        return new RegressionInferenceResults(randomDouble(),
+            RegressionConfigTests.randomRegressionConfig(),
+            randomBoolean() ? null :
+                Stream.generate(FeatureImportanceTests::randomRegression)
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toList()));
     }
 
     public void testWriteResults() {
@@ -31,6 +40,32 @@ public void testWriteResults() {
         assertThat(document.getFieldValue("result_field.predicted_value", Double.class), equalTo(0.3));
     }
 
+    public void testWriteResultsWithImportance() {
+        List<FeatureImportance> importanceList = Stream.generate(FeatureImportanceTests::randomRegression)
+            .limit(5)
+            .collect(Collectors.toList());
+        RegressionInferenceResults result = new RegressionInferenceResults(0.3,
+            new RegressionConfig("predicted_value", 3),
+            importanceList);
+        IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>());
+        result.writeResult(document, "result_field");
+
+        assertThat(document.getFieldValue("result_field.predicted_value", Double.class), equalTo(0.3));
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> writtenImportance = (List<Map<String, Object>>)document.getFieldValue(
+            "result_field.feature_importance",
+            List.class);
+        assertThat(writtenImportance, hasSize(3));
+        importanceList.sort((l, r)-> Double.compare(Math.abs(r.getImportance()), Math.abs(l.getImportance())));
+        for (int i = 0; i < 3; i++) {
+            Map<String, Object> objectMap = writtenImportance.get(i);
+            FeatureImportance importance = importanceList.get(i);
+            assertThat(objectMap.get("feature_name"), equalTo(importance.getFeatureName()));
+            assertThat(objectMap.get("importance"), equalTo(importance.getImportance()));
+            assertThat(objectMap.size(), equalTo(2));
+        }
+    }
+
     @Override
     protected RegressionInferenceResults createTestInstance() {
         return createRandomResults();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
index f9345fd6e78dd..da84341c01902 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
@@ -684,45 +684,45 @@ public void testFeatureImportance() {
             .build();
 
 
-        Map<String, Double> featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.0, 0.9)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.653200025, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        Map<String, double[]> featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.0, 0.9)));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.653200025, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.1, 0.8)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.653200025, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.653200025, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.2, 0.7)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.653200025, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.653200025, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.3, 0.6)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.16997162, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.16997162, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.4, 0.5)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.16997162, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.16997162, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.5, 0.4)));
-        assertThat(featureImportance.get("foo"), closeTo(0.0798679, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(0.0798679, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.6, 0.3)));
-        assertThat(featureImportance.get("foo"), closeTo(1.80491886, eps));
-        assertThat(featureImportance.get("bar"), closeTo(-0.4355742, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(1.80491886, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(-0.4355742, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.7, 0.2)));
-        assertThat(featureImportance.get("foo"), closeTo(2.0538184, eps));
-        assertThat(featureImportance.get("bar"), closeTo(0.1451914, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(2.0538184, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(0.1451914, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.8, 0.1)));
-        assertThat(featureImportance.get("foo"), closeTo(2.0538184, eps));
-        assertThat(featureImportance.get("bar"), closeTo(0.1451914, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(2.0538184, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(0.1451914, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.9, 0.0)));
-        assertThat(featureImportance.get("foo"), closeTo(2.0538184, eps));
-        assertThat(featureImportance.get("bar"), closeTo(0.1451914, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(2.0538184, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(0.1451914, eps));
     }
 
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
index c86f1c38a08cf..75ac3bb917e0a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
@@ -390,22 +390,22 @@ public void testFeatureImportance() {
                 TreeNode.builder(5).setLeafValue(13.0).setNumberSamples(1L),
                 TreeNode.builder(6).setLeafValue(18.0).setNumberSamples(1L)).build();
 
-        Map<String, Double> featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.25, 0.25)),
+        Map<String, double[]> featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.25, 0.25)),
             Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(-5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(-2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(-2.5, eps));
 
         featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.25, 0.75)), Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(-5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(2.5, eps));
 
         featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.75, 0.25)), Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(-2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(-2.5, eps));
 
         featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.75, 0.75)), Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(2.5, eps));
     }
 
     public void testMaxFeatureIndex() {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
index d6c88fbfdb994..4f729d1dec122 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
@@ -156,8 +156,10 @@ public void testSimulate() throws IOException {
         String responseString = EntityUtils.toString(response.getEntity());
         assertThat(responseString, containsString("\"predicted_value\":\"second\""));
         assertThat(responseString, containsString("\"predicted_value\":1.0"));
-        assertThat(responseString, containsString("\"col2\":0.944"));
-        assertThat(responseString, containsString("\"col1\":0.19999"));
+        assertThat(responseString, containsString("\"feature_name\":\"col1\""));
+        assertThat(responseString, containsString("\"feature_name\":\"col2\""));
+        assertThat(responseString, containsString("\"importance\":0.944"));
+        assertThat(responseString, containsString("\"importance\":0.19999"));
 
         String sourceWithMissingModel = "{\n" +
             "  \"pipeline\": {\n" +
@@ -221,8 +223,10 @@ public void testSimulateWithDefaultMappedField() throws IOException {
         Response response = client().performRequest(simulateRequest(source));
         String responseString = EntityUtils.toString(response.getEntity());
         assertThat(responseString, containsString("\"predicted_value\":\"second\""));
-        assertThat(responseString, containsString("\"col2\":0.944"));
-        assertThat(responseString, containsString("\"col1\":0.19999"));
+        assertThat(responseString, containsString("\"feature_name\":\"col1\""));
+        assertThat(responseString, containsString("\"feature_name\":\"col2\""));
+        assertThat(responseString, containsString("\"importance\":0.944"));
+        assertThat(responseString, containsString("\"importance\":0.19999"));
     }
 
     public void testSimulateLangIdent() throws IOException {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java
index 478621b8de880..d4969468ef910 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.action.InternalInferModelAction;
 import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.FeatureImportance;
 import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig;
@@ -120,9 +121,9 @@ public void testMutateDocumentClassificationFeatureInfluence() {
         classes.add(new ClassificationInferenceResults.TopClassEntry("foo", 0.6));
         classes.add(new ClassificationInferenceResults.TopClassEntry("bar", 0.4));
 
-        Map<String, Double> featureInfluence = new HashMap<>();
-        featureInfluence.put("feature_1", 1.13);
-        featureInfluence.put("feature_2", -42.0);
+        List<FeatureImportance> featureInfluence = new ArrayList<>();
+        featureInfluence.add(FeatureImportance.forRegression("feature_1", 1.13));
+        featureInfluence.add(FeatureImportance.forRegression("feature_2", -42.0));
 
         InternalInferModelAction.Response response = new InternalInferModelAction.Response(
             Collections.singletonList(new ClassificationInferenceResults(1.0,
@@ -135,8 +136,10 @@ public void testMutateDocumentClassificationFeatureInfluence() {
 
         assertThat(document.getFieldValue("ml.my_processor.model_id", String.class), equalTo("classification_model"));
         assertThat(document.getFieldValue("ml.my_processor.predicted_value", String.class), equalTo("foo"));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_1", Double.class), equalTo(1.13));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_2", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.importance", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.feature_name", String.class), equalTo("feature_2"));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.importance", Double.class), equalTo(1.13));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.feature_name", String.class), equalTo("feature_1"));
     }
 
     @SuppressWarnings("unchecked")
@@ -205,9 +208,9 @@ public void testMutateDocumentRegressionWithTopFetures() {
         Map<String, Object> ingestMetadata = new HashMap<>();
         IngestDocument document = new IngestDocument(source, ingestMetadata);
 
-        Map<String, Double> featureInfluence = new HashMap<>();
-        featureInfluence.put("feature_1", 1.13);
-        featureInfluence.put("feature_2", -42.0);
+        List<FeatureImportance> featureInfluence = new ArrayList<>();
+        featureInfluence.add(FeatureImportance.forRegression("feature_1", 1.13));
+        featureInfluence.add(FeatureImportance.forRegression("feature_2", -42.0));
 
         InternalInferModelAction.Response response = new InternalInferModelAction.Response(
             Collections.singletonList(new RegressionInferenceResults(0.7, regressionConfig, featureInfluence)), true);
@@ -215,8 +218,10 @@ public void testMutateDocumentRegressionWithTopFetures() {
 
         assertThat(document.getFieldValue("ml.my_processor.foo", Double.class), equalTo(0.7));
         assertThat(document.getFieldValue("ml.my_processor.model_id", String.class), equalTo("regression_model"));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_1", Double.class), equalTo(1.13));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_2", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.importance", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.feature_name", String.class), equalTo("feature_2"));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.importance", Double.class), equalTo(1.13));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.feature_name", String.class), equalTo("feature_1"));
     }
 
     public void testGenerateRequestWithEmptyMapping() {

From 9b47ac7d81ca32ec9fdae7d210f321fc5fb53fb2 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 17:06:12 -0400
Subject: [PATCH 242/285] Use deprecation logger holder in byte size value
 (#53928)

If a setting is touched during bootstrap before logging is configured,
and that setting uses a byte size value, the deprecation logger for
ByteSizeValue will be initialized. However, this means a logger will be
configured before log4j is initialized, which we reject at startup. This
commit puts this deprecation logger in a holder pattern so that it is
not initialized until first use, which will happen after logging is
configured.
---
 .../elasticsearch/common/unit/ByteSizeValue.java    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
index 0b32091fb4fa1..0f7207a9d37b5 100644
--- a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
+++ b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
@@ -26,6 +26,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -35,7 +37,14 @@
 
 public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue>, ToXContentFragment {
 
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ByteSizeValue.class));
+    /**
+     * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
+     * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
+     * {@link Setting} object constructed in, for example, settings in {@link org.elasticsearch.common.network.NetworkService}.
+     */
+    static class DeprecationLoggerHolder {
+        static DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ByteSizeValue.class));
+    }
 
     public static final ByteSizeValue ZERO = new ByteSizeValue(0, ByteSizeUnit.BYTES);
 
@@ -227,7 +236,7 @@ private static ByteSizeValue parse(final String initialInput, final String norma
             } catch (final NumberFormatException e) {
                 try {
                     final double doubleValue = Double.parseDouble(s);
-                    deprecationLogger.deprecated(
+                    DeprecationLoggerHolder.deprecationLogger.deprecated(
                             "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]",
                             initialInput, settingName);
                     return new ByteSizeValue((long) (doubleValue * unit.toBytes(1)));

From 747a164fae6240a03ce3352c07618c301e0e6102 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Mon, 23 Mar 2020 17:09:01 -0400
Subject: [PATCH 243/285] [DOCS] Fix "letter case" typo

Changes "lettercase" to "letter case" in the `uppercase` token filter
docs.
---
 .../analysis/tokenfilters/uppercase-tokenfilter.asciidoc       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc
index 780e09fa951cc..84c9ebd186595 100644
--- a/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc
@@ -16,7 +16,8 @@ Depending on the language, an uppercase character can map to multiple
 lowercase characters. Using the `uppercase` filter could result in the loss of
 lowercase character information.
 
-To avoid this loss but still have a consistent lettercase, use the <<analysis-lowercase-tokenfilter,`lowercase`>> filter instead.
+To avoid this loss but still have a consistent letter case, use the
+<<analysis-lowercase-tokenfilter,`lowercase`>> filter instead.
 ====
 
 [[analysis-uppercase-tokenfilter-analyze-ex]]

From 0a35f3900d47c16e43e8fe768eb5c914a815e9ec Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Mon, 23 Mar 2020 17:15:37 -0400
Subject: [PATCH 244/285] [DOCS] Remove double space in WDG docs

---
 .../tokenfilters/word-delimiter-graph-tokenfilter.asciidoc      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
index 1f2f61a5071dc..2fa9c41ad79b6 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
@@ -429,7 +429,7 @@ PUT /my_index
 [[analysis-word-delimiter-graph-differences]]
 ==== Differences between `word_delimiter_graph` and `word_delimiter` 
 
-Both the  `word_delimiter_graph` and
+Both the `word_delimiter_graph` and
 <<analysis-word-delimiter-tokenfilter,`word_delimiter`>> filters produce tokens
 that span multiple positions when any of the following parameters are `true`:
 

From 39785ebdbb276b365027cb4401e68acf029f6463 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Mon, 23 Mar 2020 23:29:01 +0200
Subject: [PATCH 245/285] [ML] Data frame analytics data counts (#53998)

This commit instruments data frame analytics
with stats for the data that are being analyzed.
In particular, we count training docs, test docs,
and skipped docs.

In order to account docs with missing values as skipped
docs for analyses that do not support missing values,
this commit changes the extractor so that it only ignores
docs with missing values when it collects the data summary,
which is used to estimate memory usage.
---
 .../ml/dataframe/DataFrameAnalyticsStats.java |  27 +++-
 .../ml/dataframe/stats/common/DataCounts.java | 119 +++++++++++++++++
 .../DataFrameAnalyticsStatsTests.java         |   5 +
 .../stats/common/DataCountsTests.java         |  51 ++++++++
 .../GetDataFrameAnalyticsStatsAction.java     |  29 ++++-
 .../ml/dataframe/stats/common/DataCounts.java | 120 ++++++++++++++++++
 .../xpack/core/ml/stats_index_mappings.json   |   9 ++
 ...rameAnalyticsStatsActionResponseTests.java |   5 +-
 .../stats/common/DataCountsTests.java         |  66 ++++++++++
 .../ml/integration/ClassificationIT.java      |  16 +++
 .../OutlierDetectionWithMissingFieldsIT.java  |   7 +
 .../xpack/ml/integration/RegressionIT.java    |  17 +++
 .../integration/RunDataFrameAnalyticsIT.java  |   5 +
 ...sportGetDataFrameAnalyticsStatsAction.java |   9 ++
 .../extractor/DataFrameDataExtractor.java     |  23 +++-
 .../DataFrameDataExtractorContext.java        |   6 +-
 .../DataFrameDataExtractorFactory.java        |  30 +----
 .../process/AnalyticsProcessManager.java      |  30 ++++-
 .../process/AnalyticsResultProcessor.java     |  44 ++-----
 .../CrossValidationSplitter.java              |   2 +-
 .../CrossValidationSplitterFactory.java       |   2 +-
 .../RandomCrossValidationSplitter.java        |  25 ++--
 .../ml/dataframe/stats/DataCountsTracker.java |  37 ++++++
 .../xpack/ml/dataframe/stats/StatsHolder.java |   6 +
 .../ml/dataframe/stats/StatsPersister.java    |  66 ++++++++++
 .../DataFrameDataExtractorTests.java          |  57 +++++++--
 .../AnalyticsResultProcessorTests.java        |   8 +-
 .../RandomCrossValidationSplitterTests.java   |  46 ++++---
 28 files changed, 744 insertions(+), 123 deletions(-)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
index 70169ce09b278..acdb9cccca1eb 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.client.ml.NodeAttributes;
 import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.client.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsage;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -47,6 +48,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
     static final ParseField STATE = new ParseField("state");
     static final ParseField FAILURE_REASON = new ParseField("failure_reason");
     static final ParseField PROGRESS = new ParseField("progress");
+    static final ParseField DATA_COUNTS = new ParseField("data_counts");
     static final ParseField MEMORY_USAGE = new ParseField("memory_usage");
     static final ParseField ANALYSIS_STATS = new ParseField("analysis_stats");
     static final ParseField NODE = new ParseField("node");
@@ -60,10 +62,11 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
                 (DataFrameAnalyticsState) args[1],
                 (String) args[2],
                 (List<PhaseProgress>) args[3],
-                (MemoryUsage) args[4],
-                (AnalysisStats) args[5],
-                (NodeAttributes) args[6],
-                (String) args[7]));
+                (DataCounts) args[4],
+                (MemoryUsage) args[5],
+                (AnalysisStats) args[6],
+                (NodeAttributes) args[7],
+                (String) args[8]));
 
     static {
         PARSER.declareString(constructorArg(), ID);
@@ -75,6 +78,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
         }, STATE, ObjectParser.ValueType.STRING);
         PARSER.declareString(optionalConstructorArg(), FAILURE_REASON);
         PARSER.declareObjectArray(optionalConstructorArg(), PhaseProgress.PARSER, PROGRESS);
+        PARSER.declareObject(optionalConstructorArg(), DataCounts.PARSER, DATA_COUNTS);
         PARSER.declareObject(optionalConstructorArg(), MemoryUsage.PARSER, MEMORY_USAGE);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAnalysisStats(p), ANALYSIS_STATS);
         PARSER.declareObject(optionalConstructorArg(), NodeAttributes.PARSER, NODE);
@@ -93,19 +97,21 @@ private static AnalysisStats parseAnalysisStats(XContentParser parser) throws IO
     private final DataFrameAnalyticsState state;
     private final String failureReason;
     private final List<PhaseProgress> progress;
+    private final DataCounts dataCounts;
     private final MemoryUsage memoryUsage;
     private final AnalysisStats analysisStats;
     private final NodeAttributes node;
     private final String assignmentExplanation;
 
     public DataFrameAnalyticsStats(String id, DataFrameAnalyticsState state, @Nullable String failureReason,
-                                   @Nullable List<PhaseProgress> progress, @Nullable MemoryUsage memoryUsage,
-                                   @Nullable AnalysisStats analysisStats, @Nullable NodeAttributes node,
+                                   @Nullable List<PhaseProgress> progress, @Nullable DataCounts dataCounts,
+                                   @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats, @Nullable NodeAttributes node,
                                    @Nullable String assignmentExplanation) {
         this.id = id;
         this.state = state;
         this.failureReason = failureReason;
         this.progress = progress;
+        this.dataCounts = dataCounts;
         this.memoryUsage = memoryUsage;
         this.analysisStats = analysisStats;
         this.node = node;
@@ -128,6 +134,11 @@ public List<PhaseProgress> getProgress() {
         return progress;
     }
 
+    @Nullable
+    public DataCounts getDataCounts() {
+        return dataCounts;
+    }
+
     @Nullable
     public MemoryUsage getMemoryUsage() {
         return memoryUsage;
@@ -156,6 +167,7 @@ public boolean equals(Object o) {
             && Objects.equals(state, other.state)
             && Objects.equals(failureReason, other.failureReason)
             && Objects.equals(progress, other.progress)
+            && Objects.equals(dataCounts, other.dataCounts)
             && Objects.equals(memoryUsage, other.memoryUsage)
             && Objects.equals(analysisStats, other.analysisStats)
             && Objects.equals(node, other.node)
@@ -164,7 +176,7 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
+        return Objects.hash(id, state, failureReason, progress, dataCounts, memoryUsage, analysisStats, node, assignmentExplanation);
     }
 
     @Override
@@ -174,6 +186,7 @@ public String toString() {
             .add("state", state)
             .add("failureReason", failureReason)
             .add("progress", progress)
+            .add("dataCounts", dataCounts)
             .add("memoryUsage", memoryUsage)
             .add("analysisStats", analysisStats)
             .add("node", node)
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java
new file mode 100644
index 0000000000000..b7a90b1f0b5c6
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.internal.ToStringBuilder;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class DataCounts implements ToXContentObject {
+
+    public static final String TYPE_VALUE = "analytics_data_counts";
+
+    public static final ParseField TRAINING_DOCS_COUNT = new ParseField("training_docs_count");
+    public static final ParseField TEST_DOCS_COUNT = new ParseField("test_docs_count");
+    public static final ParseField SKIPPED_DOCS_COUNT = new ParseField("skipped_docs_count");
+
+    public static final ConstructingObjectParser<DataCounts, Void> PARSER = new ConstructingObjectParser<>(TYPE_VALUE, true,
+        a -> {
+            Long trainingDocsCount = (Long) a[0];
+            Long testDocsCount = (Long) a[1];
+            Long skippedDocsCount = (Long) a[2];
+            return new DataCounts(
+                getOrDefault(trainingDocsCount, 0L),
+                getOrDefault(testDocsCount, 0L),
+                getOrDefault(skippedDocsCount, 0L)
+            );
+        });
+
+    static {
+        PARSER.declareLong(optionalConstructorArg(), TRAINING_DOCS_COUNT);
+        PARSER.declareLong(optionalConstructorArg(), TEST_DOCS_COUNT);
+        PARSER.declareLong(optionalConstructorArg(), SKIPPED_DOCS_COUNT);
+    }
+
+    private final long trainingDocsCount;
+    private final long testDocsCount;
+    private final long skippedDocsCount;
+
+    public DataCounts(long trainingDocsCount, long testDocsCount, long skippedDocsCount) {
+        this.trainingDocsCount = trainingDocsCount;
+        this.testDocsCount = testDocsCount;
+        this.skippedDocsCount = skippedDocsCount;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TRAINING_DOCS_COUNT.getPreferredName(), trainingDocsCount);
+        builder.field(TEST_DOCS_COUNT.getPreferredName(), testDocsCount);
+        builder.field(SKIPPED_DOCS_COUNT.getPreferredName(), skippedDocsCount);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataCounts that = (DataCounts) o;
+        return trainingDocsCount == that.trainingDocsCount
+            && testDocsCount == that.testDocsCount
+            && skippedDocsCount == that.skippedDocsCount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(trainingDocsCount, testDocsCount, skippedDocsCount);
+    }
+
+    @Override
+    public String toString() {
+        return new ToStringBuilder(getClass())
+            .add(TRAINING_DOCS_COUNT.getPreferredName(), trainingDocsCount)
+            .add(TEST_DOCS_COUNT.getPreferredName(), testDocsCount)
+            .add(SKIPPED_DOCS_COUNT.getPreferredName(), skippedDocsCount)
+            .toString();
+    }
+
+    public long getTrainingDocsCount() {
+        return trainingDocsCount;
+    }
+
+    public long getTestDocsCount() {
+        return testDocsCount;
+    }
+
+    public long getSkippedDocsCount() {
+        return skippedDocsCount;
+    }
+
+    private static <T> T getOrDefault(@Nullable T value, T defaultValue) {
+        return value != null ? value : defaultValue;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
index 25345181982af..d251f568dfa79 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
 import org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider;
 import org.elasticsearch.client.ml.dataframe.stats.classification.ClassificationStatsTests;
+import org.elasticsearch.client.ml.dataframe.stats.common.DataCountsTests;
 import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsageTests;
 import org.elasticsearch.client.ml.dataframe.stats.outlierdetection.OutlierDetectionStatsTests;
 import org.elasticsearch.client.ml.dataframe.stats.regression.RegressionStatsTests;
@@ -68,6 +69,7 @@ public static DataFrameAnalyticsStats randomDataFrameAnalyticsStats() {
             randomFrom(DataFrameAnalyticsState.values()),
             randomBoolean() ? null : randomAlphaOfLength(10),
             randomBoolean() ? null : createRandomProgress(),
+            randomBoolean() ? null : DataCountsTests.createRandom(),
             randomBoolean() ? null : MemoryUsageTests.createRandom(),
             analysisStats,
             randomBoolean() ? null : NodeAttributesTests.createRandom(),
@@ -93,6 +95,9 @@ public static void toXContent(DataFrameAnalyticsStats stats, XContentBuilder bui
         if (stats.getProgress() != null) {
             builder.field(DataFrameAnalyticsStats.PROGRESS.getPreferredName(), stats.getProgress());
         }
+        if (stats.getDataCounts() != null) {
+            builder.field(DataFrameAnalyticsStats.DATA_COUNTS.getPreferredName(), stats.getDataCounts());
+        }
         if (stats.getMemoryUsage() != null) {
             builder.field(DataFrameAnalyticsStats.MEMORY_USAGE.getPreferredName(), stats.getMemoryUsage());
         }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java
new file mode 100644
index 0000000000000..5e877e2d40f7b
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class DataCountsTests extends AbstractXContentTestCase<DataCounts> {
+
+    @Override
+    protected DataCounts createTestInstance() {
+        return createRandom();
+    }
+
+    public static DataCounts createRandom() {
+        return new DataCounts(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
+    }
+
+    @Override
+    protected DataCounts doParseInstance(XContentParser parser) throws IOException {
+        return DataCounts.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
index 209058e00468b..3f65a917a7427 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
@@ -165,6 +166,9 @@ public static class Stats implements ToXContentObject, Writeable {
              */
             private final List<PhaseProgress> progress;
 
+            @Nullable
+            private final DataCounts dataCounts;
+
             @Nullable
             private final MemoryUsage memoryUsage;
 
@@ -177,12 +181,13 @@ public static class Stats implements ToXContentObject, Writeable {
             private final String assignmentExplanation;
 
             public Stats(String id, DataFrameAnalyticsState state, @Nullable String failureReason, List<PhaseProgress> progress,
-                         @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats, @Nullable DiscoveryNode node,
-                         @Nullable String assignmentExplanation) {
+                         @Nullable DataCounts dataCounts, @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats,
+                         @Nullable DiscoveryNode node, @Nullable String assignmentExplanation) {
                 this.id = Objects.requireNonNull(id);
                 this.state = Objects.requireNonNull(state);
                 this.failureReason = failureReason;
                 this.progress = Objects.requireNonNull(progress);
+                this.dataCounts = dataCounts;
                 this.memoryUsage = memoryUsage;
                 this.analysisStats = analysisStats;
                 this.node = node;
@@ -198,6 +203,11 @@ public Stats(StreamInput in) throws IOException {
                 } else {
                     progress = in.readList(PhaseProgress::new);
                 }
+                if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                    dataCounts = in.readOptionalWriteable(DataCounts::new);
+                } else {
+                    dataCounts = null;
+                }
                 if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
                     memoryUsage = in.readOptionalWriteable(MemoryUsage::new);
                 } else {
@@ -261,6 +271,11 @@ public List<PhaseProgress> getProgress() {
                 return progress;
             }
 
+            @Nullable
+            public DataCounts getDataCounts() {
+                return dataCounts;
+            }
+
             @Nullable
             public MemoryUsage getMemoryUsage() {
                 return memoryUsage;
@@ -293,6 +308,9 @@ public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOExc
                 if (progress != null) {
                     builder.field("progress", progress);
                 }
+                if (dataCounts != null) {
+                    builder.field("data_counts", dataCounts);
+                }
                 if (memoryUsage != null) {
                     builder.field("memory_usage", memoryUsage);
                 }
@@ -331,6 +349,9 @@ public void writeTo(StreamOutput out) throws IOException {
                 } else {
                     out.writeList(progress);
                 }
+                if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                    out.writeOptionalWriteable(dataCounts);
+                }
                 if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                     out.writeOptionalWriteable(memoryUsage);
                 }
@@ -369,7 +390,8 @@ private void writeProgressToLegacy(StreamOutput out) throws IOException {
 
             @Override
             public int hashCode() {
-                return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
+                return Objects.hash(id, state, failureReason, progress, dataCounts, memoryUsage, analysisStats, node,
+                    assignmentExplanation);
             }
 
             @Override
@@ -385,6 +407,7 @@ public boolean equals(Object obj) {
                         && Objects.equals(this.state, other.state)
                         && Objects.equals(this.failureReason, other.failureReason)
                         && Objects.equals(this.progress, other.progress)
+                        && Objects.equals(this.dataCounts, other.dataCounts)
                         && Objects.equals(this.memoryUsage, other.memoryUsage)
                         && Objects.equals(this.analysisStats, other.analysisStats)
                         && Objects.equals(this.node, other.node)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java
new file mode 100644
index 0000000000000..f77cc781c746a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DataCounts implements ToXContentObject, Writeable {
+
+    public static final String TYPE_VALUE = "analytics_data_counts";
+
+    public static final ParseField TRAINING_DOCS_COUNT = new ParseField("training_docs_count");
+    public static final ParseField TEST_DOCS_COUNT = new ParseField("test_docs_count");
+    public static final ParseField SKIPPED_DOCS_COUNT = new ParseField("skipped_docs_count");
+
+    public static final ConstructingObjectParser<DataCounts, Void> STRICT_PARSER = createParser(false);
+    public static final ConstructingObjectParser<DataCounts, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<DataCounts, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<DataCounts, Void> parser = new ConstructingObjectParser<>(TYPE_VALUE, ignoreUnknownFields,
+            a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3]));
+
+        parser.declareString((bucket, s) -> {}, Fields.TYPE);
+        parser.declareString(ConstructingObjectParser.constructorArg(), Fields.JOB_ID);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), TRAINING_DOCS_COUNT);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), TEST_DOCS_COUNT);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), SKIPPED_DOCS_COUNT);
+        return parser;
+    }
+
+    private final String jobId;
+    private final long trainingDocsCount;
+    private final long testDocsCount;
+    private final long skippedDocsCount;
+
+    public DataCounts(String jobId, long trainingDocsCount, long testDocsCount, long skippedDocsCount) {
+        this.jobId = Objects.requireNonNull(jobId);
+        this.trainingDocsCount = trainingDocsCount;
+        this.testDocsCount = testDocsCount;
+        this.skippedDocsCount = skippedDocsCount;
+    }
+
+    public DataCounts(StreamInput in) throws IOException {
+        this.jobId = in.readString();
+        this.trainingDocsCount = in.readVLong();
+        this.testDocsCount = in.readVLong();
+        this.skippedDocsCount = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(jobId);
+        out.writeVLong(trainingDocsCount);
+        out.writeVLong(testDocsCount);
+        out.writeVLong(skippedDocsCount);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+            builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
+            builder.field(Fields.JOB_ID.getPreferredName(), jobId);
+        }
+        builder.field(TRAINING_DOCS_COUNT.getPreferredName(), trainingDocsCount);
+        builder.field(TEST_DOCS_COUNT.getPreferredName(), testDocsCount);
+        builder.field(SKIPPED_DOCS_COUNT.getPreferredName(), skippedDocsCount);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataCounts that = (DataCounts) o;
+        return Objects.equals(jobId, that.jobId)
+            && trainingDocsCount == that.trainingDocsCount
+            && testDocsCount == that.testDocsCount
+            && skippedDocsCount == that.skippedDocsCount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, trainingDocsCount, testDocsCount, skippedDocsCount);
+    }
+
+    public static String documentId(String jobId) {
+        return TYPE_VALUE + "_" + jobId;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public long getTrainingDocsCount() {
+        return trainingDocsCount;
+    }
+
+    public long getTestDocsCount() {
+        return testDocsCount;
+    }
+
+    public long getSkippedDocsCount() {
+        return skippedDocsCount;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
index 5b742c9f6d91e..7a846f5743b91 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
@@ -85,6 +85,9 @@
       "peak_usage_bytes" : {
         "type" : "long"
       },
+      "skipped_docs_count": {
+        "type": "long"
+      },
       "timestamp" : {
         "type" : "date"
       },
@@ -98,6 +101,12 @@
           }
         }
       },
+      "test_docs_count": {
+        "type": "long"
+      },
+      "training_docs_count": {
+        "type": "long"
+      },
       "type" : {
         "type" : "keyword"
       },
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
index 5cb2b3fef5450..3f957f95a902c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStatsNamedWriteablesProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCountsTests;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsageTests;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStatsTests;
@@ -42,6 +44,7 @@ public static Response randomResponse(int listSize) {
             List<PhaseProgress> progress = new ArrayList<>(progressSize);
             IntStream.of(progressSize).forEach(progressIndex -> progress.add(
                 new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100))));
+            DataCounts dataCounts = randomBoolean() ? null : DataCountsTests.createRandom();
             MemoryUsage memoryUsage = randomBoolean() ? null : MemoryUsageTests.createRandom();
             AnalysisStats analysisStats = randomBoolean() ? null :
                 randomFrom(
@@ -50,7 +53,7 @@ public static Response randomResponse(int listSize) {
                     RegressionStatsTests.createRandom()
                 );
             Response.Stats stats = new Response.Stats(DataFrameAnalyticsConfigTests.randomValidId(),
-                randomFrom(DataFrameAnalyticsState.values()), failureReason, progress, memoryUsage, analysisStats, null,
+                randomFrom(DataFrameAnalyticsState.values()), failureReason, progress, dataCounts, memoryUsage, analysisStats, null,
                 randomAlphaOfLength(20));
             analytics.add(stats);
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
new file mode 100644
index 0000000000000..84033d49de655
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.common;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class DataCountsTests extends AbstractBWCSerializationTestCase<DataCounts> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected DataCounts mutateInstanceForVersion(DataCounts instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected DataCounts doParseInstance(XContentParser parser) throws IOException {
+        return lenient ? DataCounts.LENIENT_PARSER.apply(parser, null) : DataCounts.STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ToXContent.Params getToXContentParams() {
+        return new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
+    }
+
+    @Override
+    protected Writeable.Reader<DataCounts> instanceReader() {
+        return DataCounts::new;
+    }
+
+    @Override
+    protected DataCounts createTestInstance() {
+        return createRandom();
+    }
+
+    public static DataCounts createRandom() {
+        return new DataCounts(
+            randomAlphaOfLength(10),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
+    }
+}
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index 9c4511527eeba..a96f6e8b19b9e 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
@@ -48,6 +49,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -157,6 +159,12 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsHundred() throws Excepti
             assertTopClasses(resultsObject, 2, KEYWORD_FIELD, KEYWORD_FIELD_VALUES);
         }
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(300L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
         assertModelStatePersisted(stateDocId());
@@ -224,6 +232,14 @@ public <T> void testWithOnlyTrainingRowsAndTrainingPercentIsFifty(String jobId,
         assertThat(trainingRowsCount, greaterThan(0));
         assertThat(nonTrainingRowsCount, greaterThan(0));
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), lessThan(300L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), lessThan(300L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
         assertModelStatePersisted(stateDocId());
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java
index e32d923852038..ced8c987fd685 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
 import org.junit.After;
@@ -79,6 +80,12 @@ public void testMissingFields() throws Exception {
         startAnalytics(id);
         waitUntilAnalyticsIsStopped(id);
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(id);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(id));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(5L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(2L));
+
         SearchResponse sourceData = client().prepareSearch(sourceIndex).get();
         for (SearchHit hit : sourceData.getHits()) {
             GetResponse destDocGetResponse = client().prepareGet().setIndex(config.getDest().getIndex()).setId(hit.getId()).get();
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index fc20bbb5afd7b..07752468a743f 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
@@ -31,6 +32,7 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 
 public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
 
@@ -141,6 +143,13 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsHundred() throws Excepti
 
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
+
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(350L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertModelStatePersisted(stateDocId());
         assertInferenceModelPersisted(jobId);
         assertMlResultsFieldMappings(destIndex, predictedClassField, "double");
@@ -197,6 +206,14 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty() throws Exception
         assertThat(trainingRowsCount, greaterThan(0));
         assertThat(nonTrainingRowsCount, greaterThan(0));
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), lessThan(350L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), lessThan(350L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
         assertModelStatePersisted(stateDocId());
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
index 4ffe948c9f5b7..3f7a5a7156637 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
@@ -85,6 +85,11 @@ public void testOutlierDetectionWithFewDocuments() throws Exception {
 
         startAnalytics(id);
         waitUntilAnalyticsIsStopped(id);
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(id);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(id));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(5L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
 
         SearchResponse sourceData = client().prepareSearch(sourceIndex).get();
         double scoreOfOutlier = 0.0;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
index 9ee28ba29cf3a..cb8d7bcb2be4a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
@@ -108,6 +109,7 @@ protected void taskOperation(GetDataFrameAnalyticsStatsAction.Request request, D
                 Stats stats = buildStats(
                     task.getParams().getId(),
                     statsHolder.getProgressTracker().report(),
+                    statsHolder.getDataCountsTracker().report(task.getParams().getId()),
                     statsHolder.getMemoryUsage(),
                     statsHolder.getAnalysisStats()
                 );
@@ -200,6 +202,7 @@ private void searchStats(String configId, ActionListener<Stats> listener) {
 
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
         multiSearchRequest.add(buildStoredProgressSearch(configId));
+        multiSearchRequest.add(buildStatsDocSearch(configId, DataCounts.TYPE_VALUE));
         multiSearchRequest.add(buildStatsDocSearch(configId, MemoryUsage.TYPE_VALUE));
         multiSearchRequest.add(buildStatsDocSearch(configId, OutlierDetectionStats.TYPE_VALUE));
         multiSearchRequest.add(buildStatsDocSearch(configId, ClassificationStats.TYPE_VALUE));
@@ -224,6 +227,7 @@ private void searchStats(String configId, ActionListener<Stats> listener) {
                 }
                 listener.onResponse(buildStats(configId,
                     retrievedStatsHolder.progress.get(),
+                    retrievedStatsHolder.dataCounts,
                     retrievedStatsHolder.memoryUsage,
                     retrievedStatsHolder.analysisStats
                 ));
@@ -258,6 +262,8 @@ private static void parseHit(SearchHit hit, String configId, RetrievedStatsHolde
         String hitId = hit.getId();
         if (StoredProgress.documentId(configId).equals(hitId)) {
             retrievedStatsHolder.progress = MlParserUtils.parse(hit, StoredProgress.PARSER);
+        } else if (DataCounts.documentId(configId).equals(hitId)) {
+            retrievedStatsHolder.dataCounts = MlParserUtils.parse(hit, DataCounts.LENIENT_PARSER);
         } else if (hitId.startsWith(MemoryUsage.documentIdPrefix(configId))) {
             retrievedStatsHolder.memoryUsage = MlParserUtils.parse(hit, MemoryUsage.LENIENT_PARSER);
         } else if (hitId.startsWith(OutlierDetectionStats.documentIdPrefix(configId))) {
@@ -273,6 +279,7 @@ private static void parseHit(SearchHit hit, String configId, RetrievedStatsHolde
 
     private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concreteAnalyticsId,
                                                                        List<PhaseProgress> progress,
+                                                                       DataCounts dataCounts,
                                                                        MemoryUsage memoryUsage,
                                                                        AnalysisStats analysisStats) {
         ClusterState clusterState = clusterService.state();
@@ -295,6 +302,7 @@ private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concre
             analyticsState,
             failureReason,
             progress,
+            dataCounts,
             memoryUsage,
             analysisStats,
             node,
@@ -305,6 +313,7 @@ private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concre
     private static class RetrievedStatsHolder {
 
         private volatile StoredProgress progress = new StoredProgress(new ProgressTracker().report());
+        private volatile DataCounts dataCounts;
         private volatile MemoryUsage memoryUsage;
         private volatile AnalysisStats analysisStats;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
index ba1bdf9a6a5d0..aad06b71c075a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
@@ -19,6 +19,9 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.sort.SortOrder;
@@ -187,7 +190,7 @@ private Row createRow(SearchHit hit) {
             if (values.length == 1 && (values[0] instanceof Number || values[0] instanceof String)) {
                 extractedValues[i] = Objects.toString(values[0]);
             } else {
-                if (values.length == 0 && context.includeRowsWithMissingValues) {
+                if (values.length == 0 && context.supportsRowsWithMissingValues) {
                     // if values is empty then it means it's a missing value
                     extractedValues[i] = NULL_VALUE;
                 } else {
@@ -263,13 +266,29 @@ public void collectDataSummaryAsync(ActionListener<DataSummary> dataSummaryActio
     }
 
     private SearchRequestBuilder buildDataSummarySearchRequestBuilder() {
+
+        QueryBuilder summaryQuery = context.query;
+        if (context.supportsRowsWithMissingValues == false) {
+            summaryQuery = QueryBuilders.boolQuery()
+                .filter(summaryQuery)
+                .filter(allExtractedFieldsExistQuery());
+        }
+
         return new SearchRequestBuilder(client, SearchAction.INSTANCE)
             .setIndices(context.indices)
             .setSize(0)
-            .setQuery(context.query)
+            .setQuery(summaryQuery)
             .setTrackTotalHits(true);
     }
 
+    private QueryBuilder allExtractedFieldsExistQuery() {
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        for (ExtractedField field : context.extractedFields.getAllFields()) {
+            query.filter(QueryBuilders.existsQuery(field.getName()));
+        }
+        return query;
+    }
+
     public Set<String> getCategoricalFields(DataFrameAnalysis analysis) {
         return ExtractedFieldsDetector.getCategoricalFields(context.extractedFields, analysis);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
index 0cf391bc33b2e..64ad4bed452e7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
@@ -21,10 +21,10 @@ public class DataFrameDataExtractorContext {
     final int scrollSize;
     final Map<String, String> headers;
     final boolean includeSource;
-    final boolean includeRowsWithMissingValues;
+    final boolean supportsRowsWithMissingValues;
 
     DataFrameDataExtractorContext(String jobId, ExtractedFields extractedFields, List<String> indices, QueryBuilder query, int scrollSize,
-                                  Map<String, String> headers, boolean includeSource, boolean includeRowsWithMissingValues) {
+                                  Map<String, String> headers, boolean includeSource, boolean supportsRowsWithMissingValues) {
         this.jobId = Objects.requireNonNull(jobId);
         this.extractedFields = Objects.requireNonNull(extractedFields);
         this.indices = indices.toArray(new String[indices.size()]);
@@ -32,6 +32,6 @@ public class DataFrameDataExtractorContext {
         this.scrollSize = scrollSize;
         this.headers = headers;
         this.includeSource = includeSource;
-        this.includeRowsWithMissingValues = includeRowsWithMissingValues;
+        this.supportsRowsWithMissingValues = supportsRowsWithMissingValues;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
index 3243d92bf77b6..a699e16a7d602 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
@@ -7,11 +7,9 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
-import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
 
 import java.util.Arrays;
@@ -28,18 +26,18 @@ public class DataFrameDataExtractorFactory {
     private final QueryBuilder sourceQuery;
     private final ExtractedFields extractedFields;
     private final Map<String, String> headers;
-    private final boolean includeRowsWithMissingValues;
+    private final boolean supportsRowsWithMissingValues;
 
     private DataFrameDataExtractorFactory(Client client, String analyticsId, List<String> indices, QueryBuilder sourceQuery,
                                           ExtractedFields extractedFields, Map<String, String> headers,
-                                          boolean includeRowsWithMissingValues) {
+                                          boolean supportsRowsWithMissingValues) {
         this.client = Objects.requireNonNull(client);
         this.analyticsId = Objects.requireNonNull(analyticsId);
         this.indices = Objects.requireNonNull(indices);
         this.sourceQuery = Objects.requireNonNull(sourceQuery);
         this.extractedFields = Objects.requireNonNull(extractedFields);
         this.headers = headers;
-        this.includeRowsWithMissingValues = includeRowsWithMissingValues;
+        this.supportsRowsWithMissingValues = supportsRowsWithMissingValues;
     }
 
     public DataFrameDataExtractor newExtractor(boolean includeSource) {
@@ -47,11 +45,11 @@ public DataFrameDataExtractor newExtractor(boolean includeSource) {
                 analyticsId,
                 extractedFields,
                 indices,
-                createQuery(),
+                QueryBuilders.boolQuery().filter(sourceQuery),
                 1000,
                 headers,
                 includeSource,
-                includeRowsWithMissingValues
+                supportsRowsWithMissingValues
             );
         return new DataFrameDataExtractor(client, context);
     }
@@ -60,23 +58,6 @@ public ExtractedFields getExtractedFields() {
         return extractedFields;
     }
 
-    private QueryBuilder createQuery() {
-        BoolQueryBuilder query = QueryBuilders.boolQuery();
-        query.filter(sourceQuery);
-        if (includeRowsWithMissingValues == false) {
-            query.filter(allExtractedFieldsExistQuery());
-        }
-        return query;
-    }
-
-    private QueryBuilder allExtractedFieldsExistQuery() {
-        BoolQueryBuilder query = QueryBuilders.boolQuery();
-        for (ExtractedField field : extractedFields.getAllFields()) {
-            query.filter(QueryBuilders.existsQuery(field.getName()));
-        }
-        return query;
-    }
-
     /**
      * Create a new extractor factory
      *
@@ -109,6 +90,7 @@ public static void createForDestinationIndex(Client client,
         extractedFieldsDetectorFactory.createFromDest(config, ActionListener.wrap(
             extractedFieldsDetector -> {
                 ExtractedFields extractedFields = extractedFieldsDetector.detect().v1();
+
                 DataFrameDataExtractorFactory extractorFactory = new DataFrameDataExtractorFactory(client, config.getId(),
                     Collections.singletonList(config.getDest().getIndex()), config.getSource().getParsedQuery(), extractedFields,
                     config.getHeaders(), config.getAnalysis().supportsMissingValues());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
index c7baad202d2f6..c1f09ff3fb34f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
@@ -22,8 +23,10 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -34,7 +37,9 @@
 import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitter;
 import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory;
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
+import org.elasticsearch.xpack.ml.dataframe.stats.DataCountsTracker;
 import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker;
+import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
@@ -156,7 +161,10 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
         AnalyticsResultProcessor resultProcessor = processContext.resultProcessor.get();
         try {
             writeHeaderRecord(dataExtractor, process);
-            writeDataRows(dataExtractor, process, config.getAnalysis(), task.getStatsHolder().getProgressTracker());
+            writeDataRows(dataExtractor, process, config.getAnalysis(), task.getStatsHolder().getProgressTracker(),
+                task.getStatsHolder().getDataCountsTracker());
+            processContext.statsPersister.persistWithRetry(task.getStatsHolder().getDataCountsTracker().report(config.getId()),
+                DataCounts::documentId);
             process.writeEndOfDataMessage();
             process.flushStream();
 
@@ -205,8 +213,8 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
         }
     }
 
-    private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProcess<AnalyticsResult> process,
-                               DataFrameAnalysis analysis, ProgressTracker progressTracker) throws IOException {
+    private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProcess<AnalyticsResult> process, DataFrameAnalysis analysis,
+                               ProgressTracker progressTracker, DataCountsTracker dataCountsTracker) throws IOException {
 
         CrossValidationSplitter crossValidationSplitter = new CrossValidationSplitterFactory(dataExtractor.getFieldNames())
             .create(analysis);
@@ -223,11 +231,14 @@ private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProces
             Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
             if (rows.isPresent()) {
                 for (DataFrameDataExtractor.Row row : rows.get()) {
-                    if (row.shouldSkip() == false) {
+                    if (row.shouldSkip()) {
+                        dataCountsTracker.incrementSkippedDocsCount();
+                    } else {
                         String[] rowValues = row.getValues();
                         System.arraycopy(rowValues, 0, record, 0, rowValues.length);
                         record[record.length - 2] = String.valueOf(row.getChecksum());
-                        crossValidationSplitter.process(record);
+                        crossValidationSplitter.process(record, dataCountsTracker::incrementTrainingDocsCount,
+                            dataCountsTracker::incrementTestDocsCount);
                         process.writeRecord(record);
                     }
                 }
@@ -253,6 +264,10 @@ private void writeHeaderRecord(DataFrameDataExtractor dataExtractor, AnalyticsPr
         process.writeRecord(headerRecord);
     }
 
+    private void indexDataCounts(DataCounts dataCounts) {
+        IndexRequest indexRequest = new IndexRequest(MlStatsIndex.writeAlias());
+    }
+
     private void restoreState(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config, @Nullable BytesReference state,
                               AnalyticsProcess<AnalyticsResult> process) {
         if (config.getAnalysis().persistsState() == false) {
@@ -353,9 +368,11 @@ class ProcessContext {
         private final SetOnce<DataFrameDataExtractor> dataExtractor = new SetOnce<>();
         private final SetOnce<AnalyticsResultProcessor> resultProcessor = new SetOnce<>();
         private final SetOnce<String> failureReason = new SetOnce<>();
+        private final StatsPersister statsPersister;
 
         ProcessContext(DataFrameAnalyticsConfig config) {
             this.config = Objects.requireNonNull(config);
+            this.statsPersister = new StatsPersister(config.getId(), resultsPersisterService, auditor);
         }
 
         String getFailureReason() {
@@ -378,6 +395,7 @@ synchronized void stop() {
             if (resultProcessor.get() != null) {
                 resultProcessor.get().cancel();
             }
+            statsPersister.cancel();
             if (process.get() != null) {
                 try {
                     process.get().kill();
@@ -434,7 +452,7 @@ private AnalyticsResultProcessor createResultProcessor(DataFrameAnalyticsTask ta
             DataFrameRowsJoiner dataFrameRowsJoiner =
                 new DataFrameRowsJoiner(config.getId(), dataExtractorFactory.newExtractor(true), resultsPersisterService);
             return new AnalyticsResultProcessor(
-                config, dataFrameRowsJoiner, task.getStatsHolder(), trainedModelProvider, auditor, resultsPersisterService,
+                config, dataFrameRowsJoiner, task.getStatsHolder(), trainedModelProvider, auditor, statsPersister,
                 dataExtractor.get().getAllExtractedFields());
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
index eaa202df5fc4a..d27fa31b41193 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -11,14 +11,10 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.license.License;
-import org.elasticsearch.xpack.core.ml.MlStatsIndex;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
@@ -31,18 +27,16 @@
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
+import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister;
 import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
-import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 
-import java.io.IOException;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.Iterator;
@@ -51,7 +45,6 @@
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
@@ -77,7 +70,7 @@ public class AnalyticsResultProcessor {
     private final StatsHolder statsHolder;
     private final TrainedModelProvider trainedModelProvider;
     private final DataFrameAnalyticsAuditor auditor;
-    private final ResultsPersisterService resultsPersisterService;
+    private final StatsPersister statsPersister;
     private final List<ExtractedField> fieldNames;
     private final CountDownLatch completionLatch = new CountDownLatch(1);
     private volatile String failure;
@@ -85,14 +78,13 @@ public class AnalyticsResultProcessor {
 
     public AnalyticsResultProcessor(DataFrameAnalyticsConfig analytics, DataFrameRowsJoiner dataFrameRowsJoiner,
                                     StatsHolder statsHolder, TrainedModelProvider trainedModelProvider,
-                                    DataFrameAnalyticsAuditor auditor, ResultsPersisterService resultsPersisterService,
-                                    List<ExtractedField> fieldNames) {
+                                    DataFrameAnalyticsAuditor auditor, StatsPersister statsPersister, List<ExtractedField> fieldNames) {
         this.analytics = Objects.requireNonNull(analytics);
         this.dataFrameRowsJoiner = Objects.requireNonNull(dataFrameRowsJoiner);
         this.statsHolder = Objects.requireNonNull(statsHolder);
         this.trainedModelProvider = Objects.requireNonNull(trainedModelProvider);
         this.auditor = Objects.requireNonNull(auditor);
-        this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService);
+        this.statsPersister = Objects.requireNonNull(statsPersister);
         this.fieldNames = Collections.unmodifiableList(Objects.requireNonNull(fieldNames));
     }
 
@@ -112,6 +104,7 @@ public void awaitForCompletion() {
 
     public void cancel() {
         dataFrameRowsJoiner.cancel();
+        statsPersister.cancel();
         isCancelled = true;
     }
 
@@ -177,22 +170,22 @@ private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJo
         MemoryUsage memoryUsage = result.getMemoryUsage();
         if (memoryUsage != null) {
             statsHolder.setMemoryUsage(memoryUsage);
-            indexStatsResult(memoryUsage, memoryUsage::documentId);
+            statsPersister.persistWithRetry(memoryUsage, memoryUsage::documentId);
         }
         OutlierDetectionStats outlierDetectionStats = result.getOutlierDetectionStats();
         if (outlierDetectionStats != null) {
             statsHolder.setAnalysisStats(outlierDetectionStats);
-            indexStatsResult(outlierDetectionStats, outlierDetectionStats::documentId);
+            statsPersister.persistWithRetry(outlierDetectionStats, outlierDetectionStats::documentId);
         }
         ClassificationStats classificationStats = result.getClassificationStats();
         if (classificationStats != null) {
             statsHolder.setAnalysisStats(classificationStats);
-            indexStatsResult(classificationStats, classificationStats::documentId);
+            statsPersister.persistWithRetry(classificationStats, classificationStats::documentId);
         }
         RegressionStats regressionStats = result.getRegressionStats();
         if (regressionStats != null) {
             statsHolder.setAnalysisStats(regressionStats);
-            indexStatsResult(regressionStats, regressionStats::documentId);
+            statsPersister.persistWithRetry(regressionStats, regressionStats::documentId);
         }
     }
 
@@ -275,23 +268,4 @@ private void setAndReportFailure(Exception e) {
         failure = "error processing results; " + e.getMessage();
         auditor.error(analytics.getId(), "Error processing results; " + e.getMessage());
     }
-
-    private void indexStatsResult(ToXContentObject result, Function<String, String> docIdSupplier) {
-        try {
-            resultsPersisterService.indexWithRetry(analytics.getId(),
-                MlStatsIndex.writeAlias(),
-                result,
-                new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
-                WriteRequest.RefreshPolicy.IMMEDIATE,
-                docIdSupplier.apply(analytics.getId()),
-                () -> isCancelled == false,
-                errorMsg -> auditor.error(analytics.getId(),
-                    "failed to persist result with id [" + docIdSupplier.apply(analytics.getId()) + "]; " + errorMsg)
-            );
-        } catch (IOException ioe) {
-            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed serializing stats result", analytics.getId()), ioe);
-        } catch (Exception e) {
-            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed indexing stats result", analytics.getId()), e);
-        }
-    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
index 5d12a2a81a607..fce602b28e2c5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
@@ -10,5 +10,5 @@
  */
 public interface CrossValidationSplitter {
 
-    void process(String[] row);
+    void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs);
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
index 47c052dd0bf84..986633aaa3705 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
@@ -31,6 +31,6 @@ public CrossValidationSplitter create(DataFrameAnalysis analysis) {
             return new RandomCrossValidationSplitter(
                 fieldNames, classification.getDependentVariable(), classification.getTrainingPercent(), classification.getRandomizeSeed());
         }
-        return row -> {};
+        return (row, incrementTrainingDocs, incrementTestDocs) -> incrementTrainingDocs.run();
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
index 0afc59628e7de..e4e343083ee25 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
@@ -40,22 +40,25 @@ private static int findDependentVariableIndex(List<String> fieldNames, String de
     }
 
     @Override
-    public void process(String[] row) {
-        if (canBeUsedForTraining(row)) {
-            if (isFirstRow) {
-                // Let's make sure we have at least one training row
-                isFirstRow = false;
-            } else if (isRandomlyExcludedFromTraining()) {
-                row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE;
-            }
+    public void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs) {
+        if (canBeUsedForTraining(row) && isPickedForTraining()) {
+            incrementTrainingDocs.run();
+        } else {
+            row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE;
+            incrementTestDocs.run();
         }
     }
 
     private boolean canBeUsedForTraining(String[] row) {
-        return row[dependentVariableIndex].length() > 0;
+        return row[dependentVariableIndex] != DataFrameDataExtractor.NULL_VALUE;
     }
 
-    private boolean isRandomlyExcludedFromTraining() {
-        return random.nextDouble() * 100 > trainingPercent;
+    private boolean isPickedForTraining() {
+        if (isFirstRow) {
+            // Let's make sure we have at least one training row
+            isFirstRow = false;
+            return true;
+        }
+        return random.nextDouble() * 100 <= trainingPercent;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java
new file mode 100644
index 0000000000000..bed9f52b448cf
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.dataframe.stats;
+
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
+
+public class DataCountsTracker {
+
+    private volatile long trainingDocsCount;
+    private volatile long testDocsCount;
+    private volatile long skippedDocsCount;
+
+    public void incrementTrainingDocsCount() {
+        trainingDocsCount++;
+    }
+
+    public void incrementTestDocsCount() {
+        testDocsCount++;
+    }
+
+    public void incrementSkippedDocsCount() {
+        skippedDocsCount++;
+    }
+
+    public DataCounts report(String jobId) {
+        return new DataCounts(
+            jobId,
+            trainingDocsCount,
+            testDocsCount,
+            skippedDocsCount
+        );
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
index ff6b9ec7bcfef..d01eee6a3a3d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
@@ -19,11 +19,13 @@ public class StatsHolder {
     private final ProgressTracker progressTracker;
     private final AtomicReference<MemoryUsage> memoryUsageHolder;
     private final AtomicReference<AnalysisStats> analysisStatsHolder;
+    private final DataCountsTracker dataCountsTracker;
 
     public StatsHolder() {
         progressTracker = new ProgressTracker();
         memoryUsageHolder = new AtomicReference<>();
         analysisStatsHolder = new AtomicReference<>();
+        dataCountsTracker = new DataCountsTracker();
     }
 
     public ProgressTracker getProgressTracker() {
@@ -45,4 +47,8 @@ public void setAnalysisStats(AnalysisStats analysisStats) {
     public AnalysisStats getAnalysisStats() {
         return analysisStatsHolder.get();
     }
+
+    public DataCountsTracker getDataCountsTracker() {
+        return dataCountsTracker;
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
new file mode 100644
index 0000000000000..eeb8924928ce4
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.dataframe.stats;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
+import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.function.Function;
+
+public class StatsPersister {
+
+    private static final Logger LOGGER = LogManager.getLogger(StatsPersister.class);
+
+    private final String jobId;
+    private final ResultsPersisterService resultsPersisterService;
+    private final DataFrameAnalyticsAuditor auditor;
+    private volatile boolean isCancelled;
+
+    public StatsPersister(String jobId, ResultsPersisterService resultsPersisterService, DataFrameAnalyticsAuditor auditor) {
+        this.jobId = Objects.requireNonNull(jobId);
+        this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService);
+        this.auditor = Objects.requireNonNull(auditor);
+    }
+
+    public void persistWithRetry(ToXContentObject result, Function<String, String> docIdSupplier) {
+        if (isCancelled) {
+            return;
+        }
+
+        try {
+            resultsPersisterService.indexWithRetry(jobId,
+                MlStatsIndex.writeAlias(),
+                result,
+                new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
+                WriteRequest.RefreshPolicy.IMMEDIATE,
+                docIdSupplier.apply(jobId),
+                () -> isCancelled == false,
+                errorMsg -> auditor.error(jobId,
+                    "failed to persist result with id [" + docIdSupplier.apply(jobId) + "]; " + errorMsg)
+            );
+        } catch (IOException ioe) {
+            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed serializing stats result", jobId), ioe);
+        } catch (Exception e) {
+            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed indexing stats result", jobId), e);
+        }
+    }
+
+    public void cancel() {
+        isCancelled = true;
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
index b75392e03c2bb..01661f6ec82a6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
@@ -324,9 +324,46 @@ public void testIncludeSourceIsFalseAndAtLeastOneSourceField() throws IOExceptio
         assertThat(searchRequest, containsString("\"_source\":{\"includes\":[\"field_2\"],\"excludes\":[]}"));
     }
 
-    public void testMissingValues_GivenShouldNotInclude() throws IOException {
+    public void testCollectDataSummary_GivenAnalysisSupportsMissingFields() {
+        TestExtractor dataExtractor = createExtractor(true, true);
+
+        // First and only batch
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1, 1_2), Arrays.asList(2_1, 2_2));
+        dataExtractor.setNextResponse(response);
+
+        DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary();
+
+        assertThat(dataSummary.rows, equalTo(2L));
+        assertThat(dataSummary.cols, equalTo(2));
+
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString("\"query\":{\"match_all\":{\"boost\":1.0}}"));
+    }
+
+    public void testCollectDataSummary_GivenAnalysisDoesNotSupportMissingFields() {
         TestExtractor dataExtractor = createExtractor(true, false);
 
+        // First and only batch
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1, 1_2), Arrays.asList(2_1, 2_2));
+        dataExtractor.setNextResponse(response);
+
+        DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary();
+
+        assertThat(dataSummary.rows, equalTo(2L));
+        assertThat(dataSummary.cols, equalTo(2));
+
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString(
+            "\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}},{\"bool\":{\"filter\":" +
+                "[{\"exists\":{\"field\":\"field_1\",\"boost\":1.0}},{\"exists\":{\"field\":\"field_2\",\"boost\":1.0}}]," +
+                "\"boost\":1.0}}],\"boost\":1.0}"));
+    }
+
+    public void testMissingValues_GivenSupported() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true, true);
+
         // First and only batch
         SearchResponse response1 = createSearchResponse(Arrays.asList(1_1, null, 1_3), Arrays.asList(2_1, 2_2, 2_3));
         dataExtractor.setNextResponse(response1);
@@ -343,11 +380,12 @@ public void testMissingValues_GivenShouldNotInclude() throws IOException {
         assertThat(rows.get().size(), equalTo(3));
 
         assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
-        assertThat(rows.get().get(1).getValues(), is(nullValue()));
+        assertThat(rows.get().get(1).getValues()[0], equalTo(DataFrameDataExtractor.NULL_VALUE));
+        assertThat(rows.get().get(1).getValues()[1], equalTo("22"));
         assertThat(rows.get().get(2).getValues(), equalTo(new String[] {"13", "23"}));
 
         assertThat(rows.get().get(0).shouldSkip(), is(false));
-        assertThat(rows.get().get(1).shouldSkip(), is(true));
+        assertThat(rows.get().get(1).shouldSkip(), is(false));
         assertThat(rows.get().get(2).shouldSkip(), is(false));
 
         assertThat(dataExtractor.hasNext(), is(true));
@@ -358,8 +396,8 @@ public void testMissingValues_GivenShouldNotInclude() throws IOException {
         assertThat(dataExtractor.hasNext(), is(false));
     }
 
-    public void testMissingValues_GivenShouldInclude() throws IOException {
-        TestExtractor dataExtractor = createExtractor(true, true);
+    public void testMissingValues_GivenNotSupported() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true, false);
 
         // First and only batch
         SearchResponse response1 = createSearchResponse(Arrays.asList(1_1, null, 1_3), Arrays.asList(2_1, 2_2, 2_3));
@@ -377,12 +415,11 @@ public void testMissingValues_GivenShouldInclude() throws IOException {
         assertThat(rows.get().size(), equalTo(3));
 
         assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
-        assertThat(rows.get().get(1).getValues()[0], equalTo(DataFrameDataExtractor.NULL_VALUE));
-        assertThat(rows.get().get(1).getValues()[1], equalTo("22"));
+        assertThat(rows.get().get(1).getValues(), is(nullValue()));
         assertThat(rows.get().get(2).getValues(), equalTo(new String[] {"13", "23"}));
 
         assertThat(rows.get().get(0).shouldSkip(), is(false));
-        assertThat(rows.get().get(1).shouldSkip(), is(false));
+        assertThat(rows.get().get(1).shouldSkip(), is(true));
         assertThat(rows.get().get(2).shouldSkip(), is(false));
 
         assertThat(dataExtractor.hasNext(), is(true));
@@ -424,9 +461,9 @@ public void testGetCategoricalFields() {
             containsInAnyOrder("field_keyword", "field_text", "field_boolean"));
     }
 
-    private TestExtractor createExtractor(boolean includeSource, boolean includeRowsWithMissingValues) {
+    private TestExtractor createExtractor(boolean includeSource, boolean supportsRowsWithMissingValues) {
         DataFrameDataExtractorContext context = new DataFrameDataExtractorContext(
-            JOB_ID, extractedFields, indices, query, scrollSize, headers, includeSource, includeRowsWithMissingValues);
+            JOB_ID, extractedFields, indices, query, scrollSize, headers, includeSource, supportsRowsWithMissingValues);
         return new TestExtractor(client, context);
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
index da7bbff71d5a7..ee2b399ef2d48 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
@@ -24,13 +24,13 @@
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
+import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister;
 import org.elasticsearch.xpack.ml.extractor.DocValueField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
 import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
-import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 import org.mockito.InOrder;
@@ -66,7 +66,7 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
     private StatsHolder statsHolder = new StatsHolder();
     private TrainedModelProvider trainedModelProvider;
     private DataFrameAnalyticsAuditor auditor;
-    private ResultsPersisterService resultsPersisterService;
+    private StatsPersister statsPersister;
     private DataFrameAnalyticsConfig analyticsConfig;
 
     @Before
@@ -76,7 +76,7 @@ public void setUpMocks() {
         dataFrameRowsJoiner = mock(DataFrameRowsJoiner.class);
         trainedModelProvider = mock(TrainedModelProvider.class);
         auditor = mock(DataFrameAnalyticsAuditor.class);
-        resultsPersisterService = mock(ResultsPersisterService.class);
+        statsPersister = mock(StatsPersister.class);
         analyticsConfig = new DataFrameAnalyticsConfig.Builder()
             .setId(JOB_ID)
             .setDescription(JOB_DESCRIPTION)
@@ -251,7 +251,7 @@ private AnalyticsResultProcessor createResultProcessor(List<ExtractedField> fiel
             statsHolder,
             trainedModelProvider,
             auditor,
-            resultsPersisterService,
+            statsPersister,
             fieldNames);
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
index eea102e673893..0bbc9d75d8be5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
@@ -26,6 +26,8 @@ public class RandomCrossValidationSplitterTests extends ESTestCase {
     private int dependentVariableIndex;
     private String dependentVariable;
     private long randomizeSeed;
+    private long trainingDocsCount;
+    private long testDocsCount;
 
     @Before
     public void setUpTests() {
@@ -40,47 +42,48 @@ public void setUpTests() {
     }
 
     public void testProcess_GivenRowsWithoutDependentVariableValue() {
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(fields, dependentVariable, 50.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(50.0);
 
         for (int i = 0; i < 100; i++) {
             String[] row = new String[fields.size()];
             for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
-                String value = fieldIndex == dependentVariableIndex ? "" : randomAlphaOfLength(10);
+                String value = fieldIndex == dependentVariableIndex ? DataFrameDataExtractor.NULL_VALUE : randomAlphaOfLength(10);
                 row[fieldIndex] = value;
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            crossValidationSplitter.process(processedRow);
+            crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
             // As all these rows have no dependent variable value, they're not for training and should be unaffected
             assertThat(Arrays.equals(processedRow, row), is(true));
         }
+        assertThat(trainingDocsCount, equalTo(0L));
+        assertThat(testDocsCount, equalTo(100L));
     }
 
     public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() {
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
-            fields, dependentVariable, 100.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(100.0);
 
         for (int i = 0; i < 100; i++) {
             String[] row = new String[fields.size()];
             for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
-                String value = fieldIndex == dependentVariableIndex ? "" : randomAlphaOfLength(10);
-                row[fieldIndex] = value;
+                row[fieldIndex] = randomAlphaOfLength(10);
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            crossValidationSplitter.process(processedRow);
+            crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
             // We should pick them all as training percent is 100
             assertThat(Arrays.equals(processedRow, row), is(true));
         }
+        assertThat(trainingDocsCount, equalTo(100L));
+        assertThat(testDocsCount, equalTo(0L));
     }
 
     public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() {
         double trainingPercent = randomDoubleBetween(1.0, 100.0, true);
         double trainingFraction = trainingPercent / 100;
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
-            fields, dependentVariable, trainingPercent, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(trainingPercent);
 
         int runCount = 20;
         int rowsCount = 1000;
@@ -94,7 +97,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
                 }
 
                 String[] processedRow = Arrays.copyOf(row, row.length);
-                crossValidationSplitter.process(processedRow);
+                crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
                 for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
                     if (fieldIndex != dependentVariableIndex) {
@@ -126,8 +129,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
     }
 
     public void testProcess_ShouldHaveAtLeastOneTrainingRow() {
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
-            fields, dependentVariable, 1.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(1.0);
 
         // We have some non-training rows and then a training row to check
         // we maintain the first training row and not just the first row
@@ -135,16 +137,30 @@ public void testProcess_ShouldHaveAtLeastOneTrainingRow() {
             String[] row = new String[fields.size()];
             for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
                 if (i < 9 && fieldIndex == dependentVariableIndex) {
-                    row[fieldIndex] = "";
+                    row[fieldIndex] = DataFrameDataExtractor.NULL_VALUE;
                 } else {
                     row[fieldIndex] = randomAlphaOfLength(10);
                 }
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            crossValidationSplitter.process(processedRow);
+            crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
             assertThat(Arrays.equals(processedRow, row), is(true));
         }
+        assertThat(trainingDocsCount, equalTo(1L));
+        assertThat(testDocsCount, equalTo(9L));
+    }
+
+    private RandomCrossValidationSplitter createSplitter(double trainingPercent) {
+        return new RandomCrossValidationSplitter(fields, dependentVariable, trainingPercent, randomizeSeed);
+    }
+
+    private void incrementTrainingDocsCount() {
+        trainingDocsCount++;
+    }
+
+    private void incrementTestDocsCount() {
+        testDocsCount++;
     }
 }

From ba431a0f5f958505a0bc508bd8500a838b05fe55 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 18:17:53 -0400
Subject: [PATCH 246/285] Fix classifier on OSS Linux aarch64 archive

This commit fixes the classifier on the OSS Linux aarch64 archive.
---
 distribution/archives/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index ee274b93c270e..969e92fda28ce 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -188,7 +188,7 @@ task buildLinuxTar(type: SymbolicLinkPreservingTar) {
 
 task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
-  archiveClassifier = 'linux-x86_64'
+  archiveClassifier = 'linux-aarch64'
   with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'aarch64', true, true)
 }
 

From c5d073185e882db822ac9d79fbae46d660d082b7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 18:31:24 -0400
Subject: [PATCH 247/285] Give helpful message on remote connections disabled
 (#53690)

Today when cluster.remote.connect is set to false, and some aspect of
the codebase tries to get a remote client, today we return a no such
remote cluster exception. This can be quite perplexing to users,
especially if the remote cluster is actually defined in their cluster
state, it is only that the local node is not a remote cluter
client. This commit addresses this by providing a dedicated error
message when a remote cluster is not available because the local node is
not a remote cluster client.
---
 .../transport/RemoteClusterService.java       | 16 ++++++++++++
 .../transport/RemoteClusterClientTests.java   | 14 +++++++++++
 .../transport/RemoteClusterServiceTests.java  | 25 +++++++++++++++++++
 3 files changed, 55 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index 1d6b6a12af0c0..74bc93421b108 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -116,11 +116,18 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl
         (ns, key) -> boolSetting(key, TransportSettings.TRANSPORT_COMPRESS,
             new RemoteConnectionEnabled<>(ns, key), Setting.Property.Dynamic, Setting.Property.NodeScope));
 
+    private final boolean enabled;
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
     private final TransportService transportService;
     private final Map<String, RemoteClusterConnection> remoteClusters = ConcurrentCollections.newConcurrentMap();
 
     RemoteClusterService(Settings settings, TransportService transportService) {
         super(settings);
+        this.enabled = ENABLE_REMOTE_CLUSTERS.get(settings);
         this.transportService = transportService;
     }
 
@@ -200,6 +207,9 @@ public Transport.Connection getConnection(String cluster) {
     }
 
     RemoteClusterConnection getRemoteClusterConnection(String cluster) {
+        if (enabled == false) {
+            throw new IllegalArgumentException("remote cluster service is not enabled");
+        }
         RemoteClusterConnection connection = remoteClusters.get(cluster);
         if (connection == null) {
             throw new NoSuchRemoteClusterException(cluster);
@@ -336,6 +346,9 @@ public Stream<RemoteConnectionInfo> getRemoteConnectionInfos() {
      * function on success.
      */
     public void collectNodes(Set<String> clusters, ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
+        if (enabled == false) {
+            throw new IllegalArgumentException("remote cluster service is not enabled");
+        }
         Map<String, RemoteClusterConnection> remoteClusters = this.remoteClusters;
         for (String cluster : clusters) {
             if (remoteClusters.containsKey(cluster) == false) {
@@ -379,6 +392,9 @@ public void onFailure(Exception e) {
      * @throws IllegalArgumentException if the given clusterAlias doesn't exist
      */
     public Client getRemoteClusterClient(ThreadPool threadPool, String clusterAlias) {
+        if (transportService.getRemoteClusterService().isEnabled() == false) {
+            throw new IllegalArgumentException("remote cluster service is not enabled");
+        }
         if (transportService.getRemoteClusterService().getRemoteClusterNames().contains(clusterAlias) == false) {
             throw new NoSuchRemoteClusterException(clusterAlias);
         }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
index ed71b7f85c863..7c1d3edf4550c 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
@@ -34,6 +34,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.transport.RemoteClusterConnectionTests.startTransport;
+import static org.hamcrest.Matchers.equalTo;
 
 public class RemoteClusterClientTests extends ESTestCase {
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
@@ -119,4 +120,17 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
             }
         }
     }
+
+    public void testRemoteClusterServiceNotEnabled() {
+        final Settings settings = Settings.builder().put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), false).build();
+        try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
+            service.start();
+            service.acceptIncomingRequests();
+            final RemoteClusterService remoteClusterService = service.getRemoteClusterService();
+            final IllegalArgumentException e =
+                expectThrows(IllegalArgumentException.class, () -> remoteClusterService.getRemoteClusterClient(threadPool, "test"));
+            assertThat(e.getMessage(), equalTo("remote cluster service is not enabled"));
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
index 9b35f8367914c..d16ebe74f075e 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
@@ -42,6 +42,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -843,10 +844,34 @@ public void testSkipUnavailable() {
         }
     }
 
+    public void testRemoteClusterServiceNotEnabledGetRemoteClusterConnection() {
+        final Settings settings = Settings.builder().put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), false).build();
+        try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
+            service.start();
+            service.acceptIncomingRequests();
+            final IllegalArgumentException e =
+                expectThrows(IllegalArgumentException.class, () -> service.getRemoteClusterService().getRemoteClusterConnection("test"));
+            assertThat(e.getMessage(), equalTo("remote cluster service is not enabled"));
+        }
+    }
+
+    public void testRemoteClusterServiceNotEnabledGetCollectNodes() {
+        final Settings settings = Settings.builder().put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), false).build();
+        try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
+            service.start();
+            service.acceptIncomingRequests();
+            final IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> service.getRemoteClusterService().collectNodes(Set.of(), ActionListener.wrap(r -> {}, r -> {})));
+            assertThat(e.getMessage(), equalTo("remote cluster service is not enabled"));
+        }
+    }
+
     private static Settings createSettings(String clusterAlias, List<String> seeds) {
         Settings.Builder builder = Settings.builder();
         builder.put(SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS.getConcreteSettingForNamespace(clusterAlias).getKey(),
             Strings.collectionToCommaDelimitedString(seeds));
         return builder.build();
     }
+
 }

From 83e82d39d3c9a8dbc1628acc3b89c2eaedc4a47e Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 23 Mar 2020 16:18:52 -0700
Subject: [PATCH 248/285] Merge analyze and write phases in Painless "user"
 tree (#53685)

This is the next step in producing immutable state in the Painless "user"
tree. This combines the analyze phase and the write phase so that both
semantic checking and the ir tree are generated in the same phase. This
allows the removal of the Input/Output objects created in the previous PRs
from being mutable state on the nodes. Instead they are now local state
for the analyze phase.

Relates: #53561
Relates: #49869
---
 .../org/elasticsearch/painless/Compiler.java  |  6 +-
 .../painless/node/AExpression.java            | 45 ++++-----
 .../elasticsearch/painless/node/ANode.java    |  8 --
 .../painless/node/AStatement.java             | 19 ++--
 .../painless/node/AStoreable.java             | 10 +-
 .../painless/node/EAssignment.java            | 97 ++++++++++---------
 .../elasticsearch/painless/node/EBinary.java  | 76 +++++++--------
 .../elasticsearch/painless/node/EBool.java    | 34 +++----
 .../elasticsearch/painless/node/EBoolean.java | 17 ++--
 .../painless/node/ECallLocal.java             | 75 ++++++++------
 .../painless/node/ECapturingFunctionRef.java  | 27 +++---
 .../elasticsearch/painless/node/EComp.java    | 49 +++++-----
 .../painless/node/EConditional.java           | 45 ++++-----
 .../painless/node/EConstant.java              | 16 ++-
 .../elasticsearch/painless/node/EDecimal.java | 23 ++---
 .../elasticsearch/painless/node/EElvis.java   | 38 ++++----
 .../painless/node/EExplicit.java              | 23 ++---
 .../painless/node/EFunctionRef.java           | 25 +++--
 .../painless/node/EInstanceof.java            | 36 ++++---
 .../elasticsearch/painless/node/ELambda.java  | 69 ++++++-------
 .../painless/node/EListInit.java              | 41 ++++----
 .../elasticsearch/painless/node/EMapInit.java | 62 ++++++------
 .../painless/node/ENewArray.java              | 39 ++++----
 .../painless/node/ENewArrayFunctionRef.java   | 31 +++---
 .../elasticsearch/painless/node/ENewObj.java  | 45 +++++----
 .../elasticsearch/painless/node/ENull.java    | 16 ++-
 .../elasticsearch/painless/node/ENumeric.java | 25 ++---
 .../elasticsearch/painless/node/ERegex.java   | 24 ++---
 .../elasticsearch/painless/node/EStatic.java  | 18 ++--
 .../elasticsearch/painless/node/EString.java  | 17 ++--
 .../elasticsearch/painless/node/EUnary.java   | 44 ++++-----
 .../painless/node/EVariable.java              | 27 ++----
 .../elasticsearch/painless/node/PBrace.java   | 50 +++++-----
 .../painless/node/PCallInvoke.java            | 42 ++++----
 .../elasticsearch/painless/node/PField.java   | 54 +++++------
 .../painless/node/PSubArrayLength.java        | 27 ++----
 .../painless/node/PSubBrace.java              | 30 +++---
 .../painless/node/PSubCallInvoke.java         | 37 +++----
 .../painless/node/PSubDefArray.java           | 33 +++----
 .../painless/node/PSubDefCall.java            | 45 ++++-----
 .../painless/node/PSubDefField.java           | 23 ++---
 .../painless/node/PSubField.java              | 23 ++---
 .../painless/node/PSubListShortcut.java       | 40 +++-----
 .../painless/node/PSubMapShortcut.java        | 40 +++-----
 .../painless/node/PSubNullSafeCallInvoke.java | 23 ++---
 .../painless/node/PSubNullSafeField.java      | 38 +++-----
 .../painless/node/PSubShortcut.java           | 29 ++----
 .../elasticsearch/painless/node/SBlock.java   | 30 +++---
 .../elasticsearch/painless/node/SBreak.java   | 17 ++--
 .../elasticsearch/painless/node/SCatch.java   | 36 ++++---
 .../elasticsearch/painless/node/SClass.java   | 52 ++--------
 .../painless/node/SContinue.java              | 17 ++--
 .../painless/node/SDeclBlock.java             | 28 +++---
 .../painless/node/SDeclaration.java           | 34 +++----
 .../org/elasticsearch/painless/node/SDo.java  | 38 +++-----
 .../elasticsearch/painless/node/SEach.java    | 61 ++++++------
 .../painless/node/SExpression.java            | 36 +++----
 .../elasticsearch/painless/node/SField.java   | 70 -------------
 .../org/elasticsearch/painless/node/SFor.java | 73 +++++++-------
 .../painless/node/SFunction.java              | 46 ++++-----
 .../org/elasticsearch/painless/node/SIf.java  | 33 +++----
 .../elasticsearch/painless/node/SIfElse.java  | 39 ++++----
 .../elasticsearch/painless/node/SReturn.java  | 28 +++---
 .../painless/node/SSubEachArray.java          | 50 ++++------
 .../painless/node/SSubEachIterable.java       | 54 +++++------
 .../elasticsearch/painless/node/SThrow.java   | 26 ++---
 .../org/elasticsearch/painless/node/STry.java | 38 ++++----
 .../elasticsearch/painless/node/SWhile.java   | 40 ++++----
 .../painless/DefOptimizationTests.java        |  1 +
 .../painless/node/NodeToStringTests.java      | 25 -----
 70 files changed, 1074 insertions(+), 1459 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index ab09010d248d4..e9dcdb1ef409d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -210,8 +210,7 @@ ScriptRoot compile(Loader loader, String name, String source, CompilerSettings s
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SClass root = Walker.buildPainlessTree(scriptClassInfo, name, source, settings, painlessLookup, null);
         ScriptRoot scriptRoot = new ScriptRoot(painlessLookup, settings, scriptClassInfo, root);
-        root.analyze(scriptRoot);
-        ClassNode classNode = root.writeClass();
+        ClassNode classNode = root.writeClass(scriptRoot);
         DefBootstrapInjectionPhase.phase(classNode);
         ScriptInjectionPhase.phase(scriptRoot, classNode);
         byte[] bytes = classNode.write();
@@ -240,8 +239,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SClass root = Walker.buildPainlessTree(scriptClassInfo, name, source, settings, painlessLookup, debugStream);
         ScriptRoot scriptRoot = new ScriptRoot(painlessLookup, settings, scriptClassInfo, root);
-        root.analyze(scriptRoot);
-        ClassNode classNode = root.writeClass();
+        ClassNode classNode = root.writeClass(scriptRoot);
         DefBootstrapInjectionPhase.phase(classNode);
         ScriptInjectionPhase.phase(scriptRoot, classNode);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 189cb94dcd70b..e7fb9e951c59e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -79,6 +79,18 @@ public static class Output {
          * called on this node to get the type of the node after the cast.</b>
          */
         Class<?> actual = null;
+
+        /**
+         * The {@link PainlessCast} to convert this expression's actual type
+         * to the parent expression's expected type. {@code null} if no cast
+         * is required.
+         */
+        PainlessCast painlessCast = null;
+
+        /**
+         * The {@link ExpressionNode}(s) generated from this expression.
+         */
+        ExpressionNode expressionNode = null;
     }
 
     /**
@@ -90,16 +102,6 @@ public static class Output {
      */
     AExpression prefix;
 
-    // TODO: remove placeholders once analysis and write are combined into build
-    // TODO: https://github.com/elastic/elasticsearch/issues/53561
-    // This are used to support the transition from a mutable to immutable state.
-    // Currently, the IR tree is built during the user tree "write" phase, so
-    // these are stored on the node to set during the "semantic" phase and then
-    // use during the "write" phase.
-    Input input = null;
-    Output output = null;
-    PainlessCast cast = null;
-
     /**
      * Standard constructor with location used for error tracking.
      */
@@ -121,29 +123,24 @@ public static class Output {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         throw new UnsupportedOperationException();
     }
 
-    /**
-     * Writes ASM based on the data collected during the analysis phase.
-     */
-    abstract ExpressionNode write(ClassNode classNode);
-
-    void cast() {
-        cast = AnalyzerCaster.getLegalCast(location, output.actual, input.expected, input.explicit, input.internal);
+    void cast(Input input, Output output) {
+        output.painlessCast = AnalyzerCaster.getLegalCast(location, output.actual, input.expected, input.explicit, input.internal);
     }
 
-    ExpressionNode cast(ExpressionNode expressionNode) {
-        if (cast == null) {
-            return expressionNode;
+    ExpressionNode cast(Output output) {
+        if (output.painlessCast == null) {
+            return output.expressionNode;
         }
 
         CastNode castNode = new CastNode();
         castNode.setLocation(location);
-        castNode.setExpressionType(cast.targetType);
-        castNode.setCast(cast);
-        castNode.setChildNode(expressionNode);
+        castNode.setExpressionType(output.painlessCast.targetType);
+        castNode.setCast(output.painlessCast);
+        castNode.setChildNode(output.expressionNode);
 
         return castNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index 0566399095565..4790e222e99d7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.IRNode;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -51,12 +49,6 @@ public abstract class ANode {
         this.location = Objects.requireNonNull(location);
     }
 
-    /**
-     * Writes ASM based on the data collected during the analysis phase.
-     * @param classNode the root {@link ClassNode}
-     */
-    abstract IRNode write(ClassNode classNode);
-
     /**
      * Create an error with location information pointing to this node.
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index d8d12d8a14698..06c1affc4a4a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -94,13 +94,13 @@ public static class Output {
          * Set to the approximate number of statements in a loop block to prevent
          * infinite loops during runtime.
          */
-        int statementCount = 0;
-    }
+        int statementCount = 1;
 
-    // TODO: remove placeholders once analysis and write are combined into build
-    // TODO: https://github.com/elastic/elasticsearch/issues/53561
-    Input input;
-    Output output;
+        /**
+         * The {@link StatementNode}(s) generated from this expression.
+         */
+        StatementNode statementNode = null;
+    }
 
     /**
      * Standard constructor with location used for error tracking.
@@ -112,12 +112,7 @@ public static class Output {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         throw new UnsupportedOperationException();
     }
-
-    /**
-     * Writes ASM based on the data collected during the analysis phase.
-     */
-    abstract StatementNode write(ClassNode classNode);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
index cc7262f670a94..4bca9bb37cbc5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -57,7 +58,7 @@ public static class Input extends AExpression.Input {
         this.prefix = Objects.requireNonNull(prefix);
     }
 
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         throw new UnsupportedOperationException();
     }
 
@@ -66,11 +67,4 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
      * rhs actual type to avoid an unnecessary cast.
      */
     abstract boolean isDefOptimized();
-
-    /**
-     * If this node or a sub-node of this node uses dynamic calls then
-     * actual will be set to this value. This is used for an optimization
-     * during assignment to def type targets.
-     */
-    abstract void updateActual(Class<?> actual);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index e995d637520c7..970d31935831b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -25,7 +25,13 @@
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.AssignmentNode;
+import org.elasticsearch.painless.ir.BinaryMathNode;
+import org.elasticsearch.painless.ir.BraceNode;
+import org.elasticsearch.painless.ir.BraceSubDefNode;
 import org.elasticsearch.painless.ir.ClassNode;
+import org.elasticsearch.painless.ir.DotNode;
+import org.elasticsearch.painless.ir.DotSubDefNode;
+import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -37,19 +43,13 @@
 /**
  * Represents an assignment with the lhs and rhs as child nodes.
  */
-public final class EAssignment extends AExpression {
+public class EAssignment extends AExpression {
 
-    private AExpression lhs;
-    private AExpression rhs;
-    private final boolean pre;
-    private final boolean post;
-    private Operation operation;
-
-    private boolean cat = false;
-    private Class<?> promote = null;
-    private Class<?> shiftDistance; // for shifts, the RHS is promoted independently
-    private PainlessCast there = null;
-    private PainlessCast back = null;
+    protected final AExpression lhs;
+    protected final AExpression rhs;
+    protected final boolean pre;
+    protected final boolean post;
+    protected final Operation operation;
 
     public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean pre, boolean post, Operation operation) {
         super(location);
@@ -62,11 +62,20 @@ public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        AExpression rhs = this.rhs;
+        Operation operation = this.operation;
+        boolean cat = false;
+        Class<?> promote = null;
+        Class<?> shiftDistance = null;
+        PainlessCast there = null;
+        PainlessCast back = null;
 
         Output leftOutput;
+
+        Input rightInput = new Input();
         Output rightOutput;
 
         if (lhs instanceof AStoreable) {
@@ -75,7 +84,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
             leftInput.read = input.read;
             leftInput.write = true;
-            leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
+            leftOutput = lhs.analyze(classNode, scriptRoot, scope, leftInput);
         } else {
             throw new IllegalArgumentException("Left-hand side cannot be assigned a value.");
         }
@@ -117,7 +126,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (operation != null) {
-            rightOutput = rhs.analyze(scriptRoot, scope, new Input());
+            rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
             boolean shift = false;
 
             if (operation == Operation.MUL) {
@@ -161,25 +170,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
             if (cat) {
                 if (rhs instanceof EBinary && ((EBinary)rhs).operation == Operation.ADD && rightOutput.actual == String.class) {
-                    ((EBinary)rhs).cat = true;
+                    ((BinaryMathNode)rightOutput.expressionNode).setCat(true);
                 }
             }
 
             if (shift) {
                 if (promote == def.class) {
                     // shifts are promoted independently, but for the def type, we need object.
-                    rhs.input.expected = promote;
+                    rightInput.expected = promote;
                 } else if (shiftDistance == long.class) {
-                    rhs.input.expected = int.class;
-                    rhs.input.explicit = true;
+                    rightInput.expected = int.class;
+                    rightInput.explicit = true;
                 } else {
-                    rhs.input.expected = shiftDistance;
+                    rightInput.expected = shiftDistance;
                 }
             } else {
-                rhs.input.expected = promote;
+                rightInput.expected = promote;
             }
 
-            rhs.cast();
+            rhs.cast(rightInput, rightOutput);
 
             there = AnalyzerCaster.getLegalCast(location, leftOutput.actual, promote, false, false);
             back = AnalyzerCaster.getLegalCast(location, promote, leftOutput.actual, true, false);
@@ -188,24 +197,33 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         } else if (rhs != null) {
             AStoreable lhs = (AStoreable)this.lhs;
 
+            // TODO: move this optimization to a later phase
             // If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
             if (lhs.isDefOptimized()) {
-                rightOutput = rhs.analyze(scriptRoot, scope, new Input());
+                rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
 
                 if (rightOutput.actual == void.class) {
                     throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
                 }
 
-                rhs.input.expected = rightOutput.actual;
-                lhs.updateActual(rightOutput.actual);
+                rightInput.expected = rightOutput.actual;
+                leftOutput.actual = rightOutput.actual;
+                leftOutput.expressionNode.setExpressionType(rightOutput.actual);
+
+                ExpressionNode expressionNode = leftOutput.expressionNode;
+
+                if (expressionNode instanceof DotNode && ((DotNode)expressionNode).getRightNode() instanceof DotSubDefNode) {
+                    ((DotNode)expressionNode).getRightNode().setExpressionType(leftOutput.actual);
+                } else if (expressionNode instanceof BraceNode && ((BraceNode)expressionNode).getRightNode() instanceof BraceSubDefNode) {
+                    ((BraceNode)expressionNode).getRightNode().setExpressionType(leftOutput.actual);
+                }
             // Otherwise, we must adapt the rhs type to the lhs type with a cast.
             } else {
-                Input rightInput = new Input();
                 rightInput.expected = leftOutput.actual;
-                rhs.analyze(scriptRoot, scope, rightInput);
+                rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
             }
 
-            rhs.cast();
+            rhs.cast(rightInput, rightOutput);
         } else {
             throw new IllegalStateException("Illegal tree structure.");
         }
@@ -213,21 +231,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.statement = true;
         output.actual = input.read ? leftOutput.actual : void.class;
 
-        return output;
-    }
-
-    /**
-     * Handles writing byte code for variable/method chains for all given possibilities
-     * including String concatenation, compound assignment, regular assignment, and simple
-     * reads.  Includes proper duplication for chained assignments and assignments that are
-     * also read from.
-     */
-    @Override
-    AssignmentNode write(ClassNode classNode) {
         AssignmentNode assignmentNode = new AssignmentNode();
 
-        assignmentNode.setLeftNode(lhs.write(classNode));
-        assignmentNode.setRightNode(rhs.cast(rhs.write(classNode)));
+        assignmentNode.setLeftNode(leftOutput.expressionNode);
+        assignmentNode.setRightNode(rhs.cast(rightOutput));
 
         assignmentNode.setLocation(location);
         assignmentNode.setExpressionType(output.actual);
@@ -240,7 +247,9 @@ AssignmentNode write(ClassNode classNode) {
         assignmentNode.setThere(there);
         assignmentNode.setBack(back);
 
-        return assignmentNode;
+        output.expressionNode = assignmentNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 5a65d74d4158c..9f6fc56f97ff2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -35,16 +35,11 @@
 /**
  * Represents a binary math expression.
  */
-public final class EBinary extends AExpression {
+public class EBinary extends AExpression {
 
-    final Operation operation;
-    private AExpression left;
-    private AExpression right;
-
-    private Class<?> promote = null;            // promoted type
-    private Class<?> shiftDistance = null;      // for shifts, the rhs is promoted independently
-    boolean cat = false;
-    private boolean originallyExplicit = false; // record whether there was originally an explicit cast
+    protected final Operation operation;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EBinary(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -55,18 +50,22 @@ public EBinary(Location location, Operation operation, AExpression left, AExpres
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Class<?> promote = null;            // promoted type
+        Class<?> shiftDistance = null;      // for shifts, the rhs is promoted independently
+        boolean originallyExplicit = input.explicit; // record whether there was originally an explicit cast
+
+        Output output = new Output();
 
-        originallyExplicit = input.explicit;
+        Input leftInput = new Input();
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
 
-        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
-        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
+        Input rightInput = new Input();
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
 
         if (operation == Operation.FIND || operation == Operation.MATCH) {
-            left.input.expected = String.class;
-            right.input.expected = Pattern.class;
+            leftInput.expected = String.class;
+            rightInput.expected = Pattern.class;
             promote = boolean.class;
             output.actual = boolean.class;
         } else {
@@ -99,61 +98,58 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = promote;
 
             if (operation == Operation.ADD && promote == String.class) {
-                left.input.expected = leftOutput.actual;
-                right.input.expected = rightOutput.actual;
+                leftInput.expected = leftOutput.actual;
+                rightInput.expected = rightOutput.actual;
 
                 if (left instanceof EBinary && ((EBinary) left).operation == Operation.ADD && leftOutput.actual == String.class) {
-                    ((EBinary) left).cat = true;
+                    ((BinaryMathNode)leftOutput.expressionNode).setCat(true);
                 }
 
                 if (right instanceof EBinary && ((EBinary) right).operation == Operation.ADD && rightOutput.actual == String.class) {
-                    ((EBinary) right).cat = true;
+                    ((BinaryMathNode)rightOutput.expressionNode).setCat(true);
                 }
-            } else if (promote == def.class || shiftDistance != null && shiftDistance == def.class) {
-                left.input.expected = leftOutput.actual;
-                right.input.expected = rightOutput.actual;
+            } else if (promote == def.class || shiftDistance == def.class) {
+                leftInput.expected = leftOutput.actual;
+                rightInput.expected = rightOutput.actual;
 
                 if (input.expected != null) {
                     output.actual = input.expected;
                 }
             } else {
-                left.input.expected = promote;
+                leftInput.expected = promote;
 
                 if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
                     if (shiftDistance == long.class) {
-                        right.input.expected = int.class;
-                        right.input.explicit = true;
+                        rightInput.expected = int.class;
+                        rightInput.explicit = true;
                     } else {
-                        right.input.expected = shiftDistance;
+                        rightInput.expected = shiftDistance;
                     }
                 } else {
-                    right.input.expected = promote;
+                    rightInput.expected = promote;
                 }
             }
         }
 
-        left.cast();
-        right.cast();
-
-        return output;
-    }
+        left.cast(leftInput, leftOutput);
+        right.cast(rightInput, rightOutput);
 
-    @Override
-    BinaryMathNode write(ClassNode classNode) {
         BinaryMathNode binaryMathNode = new BinaryMathNode();
 
-        binaryMathNode.setLeftNode(left.cast(left.write(classNode)));
-        binaryMathNode.setRightNode(right.cast(right.write(classNode)));
+        binaryMathNode.setLeftNode(left.cast(leftOutput));
+        binaryMathNode.setRightNode(right.cast(rightOutput));
 
         binaryMathNode.setLocation(location);
         binaryMathNode.setExpressionType(output.actual);
         binaryMathNode.setBinaryType(promote);
         binaryMathNode.setShiftType(shiftDistance);
         binaryMathNode.setOperation(operation);
-        binaryMathNode.setCat(cat);
+        binaryMathNode.setCat(false);
         binaryMathNode.setOriginallExplicit(originallyExplicit);
 
-        return binaryMathNode;
+        output.expressionNode = binaryMathNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index f2acae9299f20..e9dac2123c6cd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -31,11 +31,11 @@
 /**
  * Represents a boolean expression.
  */
-public final class EBool extends AExpression {
+public class EBool extends AExpression {
 
-    private final Operation operation;
-    private AExpression left;
-    private AExpression right;
+    protected final Operation operation;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EBool(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -46,37 +46,33 @@ public EBool(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         Input leftInput = new Input();
         leftInput.expected = boolean.class;
-        left.analyze(scriptRoot, scope, leftInput);
-        left.cast();
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
+        left.cast(leftInput, leftOutput);
 
         Input rightInput = new Input();
         rightInput.expected = boolean.class;
-        right.analyze(scriptRoot, scope, rightInput);
-        right.cast();
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
+        right.cast(rightInput, rightOutput);
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    BooleanNode write(ClassNode classNode) {
         BooleanNode booleanNode = new BooleanNode();
 
-        booleanNode.setLeftNode(left.cast(left.write(classNode)));
-        booleanNode.setRightNode(right.cast(right.write(classNode)));
+        booleanNode.setLeftNode(left.cast(leftOutput));
+        booleanNode.setRightNode(right.cast(rightOutput));
 
         booleanNode.setLocation(location);
         booleanNode.setExpressionType(output.actual);
         booleanNode.setOperation(operation);
 
-        return booleanNode;
+        output.expressionNode = booleanNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 6e78fb972008c..b1362e99cb2a5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -23,13 +23,12 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 /**
  * Represents a boolean constant.
  */
-public final class EBoolean extends AExpression {
+public class EBoolean extends AExpression {
 
     protected boolean constant;
 
@@ -40,9 +39,8 @@ public EBoolean(Location location, boolean constant) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
@@ -50,17 +48,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index ad4dce45d4d7a..c86966df03f60 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.MemberCallNode;
+import org.elasticsearch.painless.ir.FieldNode;
 import org.elasticsearch.painless.lookup.PainlessClassBinding;
 import org.elasticsearch.painless.lookup.PainlessInstanceBinding;
 import org.elasticsearch.painless.lookup.PainlessMethod;
@@ -32,35 +33,35 @@
 
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents a user-defined call.
  */
-public final class ECallLocal extends AExpression {
+public class ECallLocal extends AExpression {
 
-    private final String name;
-    private final List<AExpression> arguments;
-
-    private FunctionTable.LocalFunction localFunction = null;
-    private PainlessMethod importedMethod = null;
-    private PainlessClassBinding classBinding = null;
-    private int classBindingOffset = 0;
-    private PainlessInstanceBinding instanceBinding = null;
-    private String bindingName = null;
+    protected final String name;
+    protected final List<AExpression> arguments;
 
     public ECallLocal(Location location, String name, List<AExpression> arguments) {
         super(location);
 
         this.name = Objects.requireNonNull(name);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        FunctionTable.LocalFunction localFunction = null;
+        PainlessMethod importedMethod = null;
+        PainlessClassBinding classBinding = null;
+        int classBindingOffset = 0;
+        PainlessInstanceBinding instanceBinding = null;
+        String bindingName = null;
+
+        Output output = new Output();
 
         localFunction = scriptRoot.getFunctionTable().getFunction(name, arguments.size());
 
@@ -125,43 +126,53 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             typeParameters = new ArrayList<>(classBinding.typeParameters);
             output.actual = classBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("class_binding");
-            scriptRoot.getClassNode().addField(new SField(location,
-                    Modifier.PRIVATE, bindingName, classBinding.javaConstructor.getDeclaringClass()));
+
+            FieldNode fieldNode = new FieldNode();
+            fieldNode.setLocation(location);
+            fieldNode.setModifiers(Modifier.PRIVATE);
+            fieldNode.setFieldType(classBinding.javaConstructor.getDeclaringClass());
+            fieldNode.setName(bindingName);
+
+            classNode.addFieldNode(fieldNode);
         } else if (instanceBinding != null) {
             typeParameters = new ArrayList<>(instanceBinding.typeParameters);
             output.actual = instanceBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("instance_binding");
-            scriptRoot.getClassNode().addField(new SField(location, Modifier.STATIC | Modifier.PUBLIC,
-                    bindingName, instanceBinding.targetInstance.getClass()));
+
+            FieldNode fieldNode = new FieldNode();
+            fieldNode.setLocation(location);
+            fieldNode.setModifiers(Modifier.PUBLIC | Modifier.STATIC);
+            fieldNode.setFieldType(instanceBinding.targetInstance.getClass());
+            fieldNode.setName(bindingName);
+
+            classNode.addFieldNode(fieldNode);
+
             scriptRoot.addStaticConstant(bindingName, instanceBinding.targetInstance);
         } else {
             throw new IllegalStateException("Illegal tree structure.");
         }
 
+        List<Output> argumentOutputs = new ArrayList<>(arguments.size());
         // if the class binding is using an implicit this reference then the arguments counted must
         // be incremented by 1 as the this reference will not be part of the arguments passed into
         // the class binding call
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = typeParameters.get(argument + classBindingOffset);
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Input argumentInput = new Input();
+            argumentInput.expected = typeParameters.get(argument + classBindingOffset);
+            argumentInput.internal = true;
+            Output argumentOutput = expression.analyze(classNode, scriptRoot, scope, argumentInput);
+            expression.cast(argumentInput, argumentOutput);
+            argumentOutputs.add(argumentOutput);
         }
 
         output.statement = true;
 
-        return output;
-    }
-
-    @Override
-    MemberCallNode write(ClassNode classNode) {
         MemberCallNode memberCallNode = new MemberCallNode();
 
-        for (AExpression argument : arguments) {
-            memberCallNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int argument = 0; argument < arguments.size(); ++argument) {
+            memberCallNode.addArgumentNode(arguments.get(argument).cast(argumentOutputs.get(argument)));
         }
 
         memberCallNode.setLocation(location);
@@ -173,7 +184,9 @@ MemberCallNode write(ClassNode classNode) {
         memberCallNode.setBindingName(bindingName);
         memberCallNode.setInstanceBinding(instanceBinding);
 
-        return memberCallNode;
+        output.expressionNode = memberCallNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index fa8b25cd861f9..e5f3b11910d18 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -35,11 +35,12 @@
 /**
  * Represents a capturing function reference.  For member functions that require a this reference, ie not static.
  */
-public final class ECapturingFunctionRef extends AExpression implements ILambda {
-    private final String variable;
-    private final String call;
+public class ECapturingFunctionRef extends AExpression implements ILambda {
 
-    private FunctionRef ref;
+    protected final String variable;
+    protected final String call;
+
+    // TODO: #54015
     private Variable captured;
     private String defPointer;
 
@@ -51,9 +52,10 @@ public ECapturingFunctionRef(Location location, String variable, String call) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        FunctionRef ref = null;
+
+        Output output = new Output();
 
         captured = scope.getVariable(location, variable);
         if (input.expected == null) {
@@ -75,11 +77,6 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    CapturingFuncRefNode write(ClassNode classNode) {
         CapturingFuncRefNode capturingFuncRefNode = new CapturingFuncRefNode();
 
         capturingFuncRefNode.setLocation(location);
@@ -87,9 +84,11 @@ CapturingFuncRefNode write(ClassNode classNode) {
         capturingFuncRefNode.setCapturedName(captured.getName());
         capturingFuncRefNode.setName(call);
         capturingFuncRefNode.setPointer(defPointer);
-        capturingFuncRefNode.setFuncRef(ref);;
+        capturingFuncRefNode.setFuncRef(ref);
 
-        return capturingFuncRefNode;
+        output.expressionNode = capturingFuncRefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 32555c18625f7..359546e805a40 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -34,13 +34,11 @@
 /**
  * Represents a comparison expression.
  */
-public final class EComp extends AExpression {
+public class EComp extends AExpression {
 
-    private final Operation operation;
-    private AExpression left;
-    private AExpression right;
-
-    private Class<?> promotedType;
+    protected final Operation operation;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EComp(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -51,12 +49,16 @@ public EComp(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Class<?> promotedType;
+
+        Output output = new Output();
+
+        Input leftInput = new Input();
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
 
-        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
-        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
+        Input rightInput = new Input();
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
 
         if (operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER) {
             promotedType = AnalyzerCaster.promoteEquality(leftOutput.actual, rightOutput.actual);
@@ -74,11 +76,11 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (operation != Operation.EQR && operation != Operation.NER && promotedType == def.class) {
-            left.input.expected = leftOutput.actual;
-            right.input.expected = rightOutput.actual;
+            leftInput.expected = leftOutput.actual;
+            rightInput.expected = rightOutput.actual;
         } else {
-            left.input.expected = promotedType;
-            right.input.expected = promotedType;
+            leftInput.expected = promotedType;
+            rightInput.expected = promotedType;
         }
 
         if ((operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER)
@@ -86,27 +88,24 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("extraneous comparison of [null] constants"));
         }
 
-        left.cast();
-        right.cast();
+        left.cast(leftInput, leftOutput);
+        right.cast(rightInput, rightOutput);
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    ComparisonNode write(ClassNode classNode) {
         ComparisonNode comparisonNode = new ComparisonNode();
 
-        comparisonNode.setLeftNode(left.cast(left.write(classNode)));
-        comparisonNode.setRightNode(right.cast(right.write(classNode)));
+        comparisonNode.setLeftNode(left.cast(leftOutput));
+        comparisonNode.setRightNode(right.cast(rightOutput));
 
         comparisonNode.setLocation(location);
         comparisonNode.setExpressionType(output.actual);
         comparisonNode.setComparisonType(promotedType);
         comparisonNode.setOperation(operation);
 
-        return comparisonNode;
+        output.expressionNode = comparisonNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index d523838ecf66d..2b4440787347b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -32,11 +32,11 @@
 /**
  * Represents a conditional expression.
  */
-public final class EConditional extends AExpression {
+public class EConditional extends AExpression {
 
-    private AExpression condition;
-    private AExpression left;
-    private AExpression right;
+    protected final AExpression condition;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EConditional(Location location, AExpression condition, AExpression left, AExpression right) {
         super(location);
@@ -47,30 +47,28 @@ public EConditional(Location location, AExpression condition, AExpression left,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         Input conditionInput = new Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
 
         Input leftInput = new Input();
         leftInput.expected = input.expected;
         leftInput.explicit = input.explicit;
         leftInput.internal = input.internal;
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
 
         Input rightInput = new Input();
         rightInput.expected = input.expected;
         rightInput.explicit = input.explicit;
         rightInput.internal = input.internal;
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
 
         output.actual = input.expected;
 
-        Output leftOutput = left.analyze(scriptRoot, scope, leftInput);
-        Output rightOutput = right.analyze(scriptRoot, scope, rightInput);
-
         if (input.expected == null) {
             Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
 
@@ -80,29 +78,26 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
             }
 
-            left.input.expected = promote;
-            right.input.expected = promote;
+            leftInput.expected = promote;
+            rightInput.expected = promote;
             output.actual = promote;
         }
 
-        left.cast();
-        right.cast();
-
-        return output;
-    }
+        left.cast(leftInput, leftOutput);
+        right.cast(rightInput, rightOutput);
 
-    @Override
-    ConditionalNode write(ClassNode classNode) {
         ConditionalNode conditionalNode = new ConditionalNode();
 
-        conditionalNode.setLeftNode(left.cast(left.write(classNode)));
-        conditionalNode.setRightNode(right.cast(right.write(classNode)));
-        conditionalNode.setConditionNode(condition.cast(condition.write(classNode)));
+        conditionalNode.setLeftNode(left.cast(leftOutput));
+        conditionalNode.setRightNode(right.cast(rightOutput));
+        conditionalNode.setConditionNode(condition.cast(conditionOutput));
 
         conditionalNode.setLocation(location);
         conditionalNode.setExpressionType(output.actual);
 
-        return conditionalNode;
+        output.expressionNode = conditionalNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 5caa6bca92895..317a9dae3bd3d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -30,7 +30,7 @@
  * Represents a constant inserted into the tree replacing
  * other constants during constant folding.  (Internal only.)
  */
-final class EConstant extends AExpression {
+public class EConstant extends AExpression {
 
     protected Object constant;
 
@@ -41,9 +41,8 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (constant instanceof String) {
             output.actual = String.class;
@@ -69,17 +68,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     "for constant node"));
         }
 
-        return output;
-    }
-
-    @Override
-    ConstantNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 405f90b356f1a..35388db216cb5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,11 +30,9 @@
 /**
  * Represents a decimal constant.
  */
-public final class EDecimal extends AExpression {
+public class EDecimal extends AExpression {
 
-    private final String value;
-
-    protected Object constant;
+    protected final String value;
 
     public EDecimal(Location location, String value) {
         super(location);
@@ -44,9 +41,10 @@ public EDecimal(Location location, String value) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Object constant;
+
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
@@ -72,17 +70,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
         }
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
index 6c771af47caf4..6ccb1a3f397b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
@@ -33,8 +33,9 @@
  * non null. If the first expression is null then it evaluates the second expression and returns it.
  */
 public class EElvis extends AExpression {
-    private AExpression lhs;
-    private AExpression rhs;
+
+    protected AExpression lhs;
+    protected AExpression rhs;
 
     public EElvis(Location location, AExpression lhs, AExpression rhs) {
         super(location);
@@ -44,24 +45,26 @@ public EElvis(Location location, AExpression lhs, AExpression rhs) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.expected != null && input.expected.isPrimitive()) {
             throw createError(new IllegalArgumentException("Elvis operator cannot return primitives"));
         }
+
         Input leftInput = new Input();
         leftInput.expected = input.expected;
         leftInput.explicit = input.explicit;
         leftInput.internal = input.internal;
+        Output leftOutput = lhs.analyze(classNode, scriptRoot, scope, leftInput);
+
         Input rightInput = new Input();
         rightInput.expected = input.expected;
         rightInput.explicit = input.explicit;
         rightInput.internal = input.internal;
+        Output rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
+
         output.actual = input.expected;
-        Output leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
-        Output rightOutput = rhs.analyze(scriptRoot, scope, rightInput);
 
         if (lhs instanceof ENull) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is null."));
@@ -83,28 +86,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         if (input.expected == null) {
             Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
 
-            lhs.input.expected = promote;
-            rhs.input.expected = promote;
+            leftInput.expected = promote;
+            rightInput.expected = promote;
             output.actual = promote;
         }
 
-        lhs.cast();
-        rhs.cast();
-
-        return output;
-    }
+        lhs.cast(leftInput, leftOutput);
+        rhs.cast(rightInput, rightOutput);
 
-    @Override
-    ElvisNode write(ClassNode classNode) {
         ElvisNode elvisNode = new ElvisNode();
 
-        elvisNode.setLeftNode(lhs.cast(lhs.write(classNode)));
-        elvisNode.setRightNode(rhs.cast(rhs.write(classNode)));
+        elvisNode.setLeftNode(lhs.cast(leftOutput));
+        elvisNode.setRightNode(rhs.cast(rightOutput));
 
         elvisNode.setLocation(location);
         elvisNode.setExpressionType(output.actual);
 
-        return elvisNode;
+        output.expressionNode = elvisNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index b76dcd946b3b3..532e9607fb7a6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -30,10 +29,10 @@
 /**
  * Represents an explicit cast.
  */
-public final class EExplicit extends AExpression {
+public class EExplicit extends AExpression {
 
-    private final String type;
-    private AExpression child;
+    protected final String type;
+    protected final AExpression child;
 
     public EExplicit(Location location, String type, AExpression child) {
         super(location);
@@ -43,9 +42,8 @@ public EExplicit(Location location, String type, AExpression child) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
 
@@ -56,15 +54,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input childInput = new Input();
         childInput.expected = output.actual;
         childInput.explicit = true;
-        child.analyze(scriptRoot, scope, childInput);
-        child.cast();
+        Output childOutput = child.analyze(classNode, scriptRoot, scope, childInput);
+        child.cast(childInput, childOutput);
 
-        return output;
-    }
+        output.expressionNode = child.cast(childOutput);
 
-    @Override
-    ExpressionNode write(ClassNode classNode) {
-        return child.cast(child.write(classNode));
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 62bb34c215590..a9cb7876fe11a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -33,11 +33,12 @@
 /**
  * Represents a function reference.
  */
-public final class EFunctionRef extends AExpression implements ILambda {
-    private final String type;
-    private final String call;
+public class EFunctionRef extends AExpression implements ILambda {
 
-    private FunctionRef ref;
+    protected final String type;
+    protected final String call;
+
+    // TODO: #54015
     private String defPointer;
 
     public EFunctionRef(Location location, String type, String call) {
@@ -48,9 +49,10 @@ public EFunctionRef(Location location, String type, String call) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        FunctionRef ref;
+
+        Output output = new Output();
 
         if (input.expected == null) {
             ref = null;
@@ -63,18 +65,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    FuncRefNode write(ClassNode classNode) {
         FuncRefNode funcRefNode = new FuncRefNode();
 
         funcRefNode.setLocation(location);
         funcRefNode.setExpressionType(output.actual);
         funcRefNode.setFuncRef(ref);
 
-        return funcRefNode;
+        output.expressionNode = funcRefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index e6ee1a03602b8..9bf0a00e610ca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -33,13 +33,10 @@
  * <p>
  * Unlike java's, this works for primitive types too.
  */
-public final class EInstanceof extends AExpression {
-    private AExpression expression;
-    private final String type;
+public class EInstanceof extends AExpression {
 
-    private Class<?> resolvedType;
-    private Class<?> expressionType;
-    private boolean primitiveExpression;
+    protected final AExpression expression;
+    protected final String type;
 
     public EInstanceof(Location location, AExpression expression, String type) {
         super(location);
@@ -48,9 +45,12 @@ public EInstanceof(Location location, AExpression expression, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Class<?> resolvedType;
+        Class<?> expressionType;
+        boolean primitiveExpression;
+
+        Output output = new Output();
 
         // ensure the specified type is part of the definition
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
@@ -64,9 +64,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 PainlessLookupUtility.typeToJavaType(clazz);
 
         // analyze and cast the expression
-        Output expressionOutput = expression.analyze(scriptRoot, scope, new Input());
-        expression.input.expected = expressionOutput.actual;
-        expression.cast();
+        Input expressionInput = new Input();
+        Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+        expressionInput.expected = expressionOutput.actual;
+        expression.cast(expressionInput, expressionOutput);
 
         // record if the expression returns a primitive
         primitiveExpression = expressionOutput.actual.isPrimitive();
@@ -76,14 +77,9 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    InstanceofNode write(ClassNode classNode) {
         InstanceofNode instanceofNode = new InstanceofNode();
 
-        instanceofNode.setChildNode(expression.cast(expression.write(classNode)));
+        instanceofNode.setChildNode(expression.cast(expressionOutput));
 
         instanceofNode.setLocation(location);
         instanceofNode.setExpressionType(output.actual);
@@ -91,7 +87,9 @@ InstanceofNode write(ClassNode classNode) {
         instanceofNode.setResolvedType(resolvedType);
         instanceofNode.setPrimitiveResult(primitiveExpression);
 
-        return instanceofNode;
+        output.expressionNode = instanceofNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index a4cb19d048c62..41ffd62df044f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.Scope.LambdaScope;
 import org.elasticsearch.painless.Scope.Variable;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.FunctionNode;
 import org.elasticsearch.painless.ir.LambdaNode;
@@ -59,27 +60,16 @@
  * <br>
  * {@code sort(list, lambda$0(capture))}
  */
-public final class ELambda extends AExpression implements ILambda {
+public class ELambda extends AExpression implements ILambda {
 
-    private final List<String> paramTypeStrs;
-    private final List<String> paramNameStrs;
-    private final List<AStatement> statements;
+    protected final List<String> paramTypeStrs;
+    protected final List<String> paramNameStrs;
+    protected final List<AStatement> statements;
 
-    // captured variables
+    // TODO: #54015
     private List<Variable> captures;
-    // static parent, static lambda
-    private FunctionRef ref;
-    // dynamic parent, deferred until link time
     private String defPointer;
 
-    private String name;
-    private Class<?> returnType;
-    private List<Class<?>> typeParameters;
-    private List<String> parameterNames;
-    private SBlock block;
-    private boolean methodEscape;
-    private int maxLoopCounter;
-
     public ELambda(Location location,
                    List<String> paramTypes, List<String> paramNames,
                    List<AStatement> statements) {
@@ -91,11 +81,18 @@ public ELambda(Location location,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        List<Class<?>> typeParameters = new ArrayList<>();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        String name;
+        Class<?> returnType;
+        List<Class<?>> typeParametersWithCaptures;
+        List<String> parameterNames;
+        SBlock block;
+        int maxLoopCounter;
+        FunctionRef ref;
+
+        Output output = new Output();
+
+        List<Class<?>> typeParameters;
         PainlessMethod interfaceMethod;
         // inspect the target first, set interface method if we know it.
         if (input.expected == null) {
@@ -157,8 +154,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         for (int index = 0; index < typeParameters.size(); ++index) {
             Class<?> type = typeParameters.get(index);
-            String name = paramNameStrs.get(index);
-            lambdaScope.defineVariable(location, type, name, true);
+            String paramName = paramNameStrs.get(index);
+            lambdaScope.defineVariable(location, type, paramName, true);
         }
 
         block = new SBlock(location, statements);
@@ -167,7 +164,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
         AStatement.Input blockInput = new AStatement.Input();
         blockInput.lastSource = true;
-        AStatement.Output blockOutput = block.analyze(scriptRoot, lambdaScope, blockInput);
+        AStatement.Output blockOutput = block.analyze(classNode, scriptRoot, lambdaScope, blockInput);
 
         if (blockOutput.methodEscape == false) {
             throw createError(new IllegalArgumentException("not all paths return a value for lambda"));
@@ -177,18 +174,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         // prepend capture list to lambda's arguments
         captures = new ArrayList<>(lambdaScope.getCaptures());
-        this.typeParameters = new ArrayList<>(captures.size() + typeParameters.size());
+        typeParametersWithCaptures = new ArrayList<>(captures.size() + typeParameters.size());
         parameterNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
-            this.typeParameters.add(var.getType());
+            typeParametersWithCaptures.add(var.getType());
             parameterNames.add(var.getName());
         }
-        this.typeParameters.addAll(typeParameters);
+        typeParametersWithCaptures.addAll(typeParameters);
         parameterNames.addAll(paramNameStrs);
 
         // desugar lambda body into a synthetic method
         name = scriptRoot.getNextSyntheticName("lambda");
-        scriptRoot.getFunctionTable().addFunction(name, returnType, this.typeParameters, true, true);
+        scriptRoot.getFunctionTable().addFunction(name, returnType, typeParametersWithCaptures, true, true);
 
         // setup method reference to synthetic method
         if (input.expected == null) {
@@ -202,19 +199,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    LambdaNode write(ClassNode classNode) {
         FunctionNode functionNode = new FunctionNode();
-
-        functionNode.setBlockNode(block.write(classNode));
-
+        functionNode.setBlockNode((BlockNode)blockOutput.statementNode);
         functionNode.setLocation(location);
         functionNode.setName(name);
         functionNode.setReturnType(returnType);
-        functionNode.getTypeParameters().addAll(typeParameters);
+        functionNode.getTypeParameters().addAll(typeParametersWithCaptures);
         functionNode.getParameterNames().addAll(parameterNames);
         functionNode.setStatic(true);
         functionNode.setVarArgs(false);
@@ -224,7 +214,6 @@ LambdaNode write(ClassNode classNode) {
         classNode.addFunctionNode(functionNode);
 
         LambdaNode lambdaNode = new LambdaNode();
-
         lambdaNode.setLocation(location);
         lambdaNode.setExpressionType(output.actual);
         lambdaNode.setFuncRef(ref);
@@ -233,7 +222,9 @@ LambdaNode write(ClassNode classNode) {
             lambdaNode.addCapture(capture.getName());
         }
 
-        return lambdaNode;
+        output.expressionNode = lambdaNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index cb49bce8a11c3..c60a9d1bea78b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -29,29 +29,28 @@
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 
 /**
  * Represents a list initialization shortcut.
  */
-public final class EListInit extends AExpression {
-    private final List<AExpression> values;
+public class EListInit extends AExpression {
 
-    private PainlessConstructor constructor = null;
-    private PainlessMethod method = null;
+    protected final List<AExpression> values;
 
     public EListInit(Location location, List<AExpression> values) {
         super(location);
 
-        this.values = values;
+        this.values = Collections.unmodifiableList(Objects.requireNonNull(values));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from list initializer."));
@@ -59,38 +58,34 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = ArrayList.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
+        PainlessConstructor constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
                     "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "add", 1);
+        PainlessMethod method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "add", 1);
 
         if (method == null) {
             throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", add/1] not found"));
         }
 
-        for (int index = 0; index < values.size(); ++index) {
-            AExpression expression = values.get(index);
+        List<Output> valueOutputs = new ArrayList<>(values.size());
 
+        for (AExpression expression : values) {
             Input expressionInput = new Input();
             expressionInput.expected = def.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            valueOutputs.add(expressionOutput);
         }
 
-        return output;
-    }
-
-    @Override
-    ListInitializationNode write(ClassNode classNode) {
         ListInitializationNode listInitializationNode = new ListInitializationNode();
 
-        for (AExpression value : values) {
-            listInitializationNode.addArgumentNode(value.cast(value.write(classNode)));
+        for (int i = 0; i < values.size(); ++i) {
+            listInitializationNode.addArgumentNode(values.get(i).cast(valueOutputs.get(i)));
         }
 
         listInitializationNode.setLocation(location);
@@ -98,7 +93,9 @@ ListInitializationNode write(ClassNode classNode) {
         listInitializationNode.setConstructor(constructor);
         listInitializationNode.setMethod(method);
 
-        return listInitializationNode;
+        output.expressionNode = listInitializationNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index f846a18eb89f9..fbe500c027dab 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -28,32 +28,32 @@
 import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 
 /**
  * Represents a map initialization shortcut.
  */
-public final class EMapInit extends AExpression {
-    private final List<AExpression> keys;
-    private final List<AExpression> values;
+public class EMapInit extends AExpression {
 
-    private PainlessConstructor constructor = null;
-    private PainlessMethod method = null;
+    protected final List<AExpression> keys;
+    protected final List<AExpression> values;
 
     public EMapInit(Location location, List<AExpression> keys, List<AExpression> values) {
         super(location);
 
-        this.keys = keys;
-        this.values = values;
+        this.keys = Collections.unmodifiableList(Objects.requireNonNull(keys));
+        this.values = Collections.unmodifiableList(Objects.requireNonNull(values));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from map initializer."));
@@ -61,14 +61,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = HashMap.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
+        PainlessConstructor constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
                     "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "put", 2);
+        PainlessMethod method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "put", 2);
 
         if (method == null) {
             throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", put/2] not found"));
@@ -78,37 +78,33 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        for (int index = 0; index < keys.size(); ++index) {
-            AExpression expression = keys.get(index);
+        List<Output> keyOutputs = new ArrayList<>(keys.size());
+        List<Output> valueOutputs = new ArrayList<>(values.size());
 
+        for (int i = 0; i < keys.size(); ++i) {
+            AExpression expression = keys.get(i);
             Input expressionInput = new Input();
             expressionInput.expected = def.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
-        }
-
-        for (int index = 0; index < values.size(); ++index) {
-            AExpression expression = values.get(index);
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            keyOutputs.add(expressionOutput);
 
-            Input expressionInput = new Input();
+            expression = values.get(i);
+            expressionInput = new Input();
             expressionInput.expected = def.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            valueOutputs.add(expressionOutput);
         }
 
-        return output;
-    }
-
-    @Override
-    MapInitializationNode write(ClassNode classNode) {
         MapInitializationNode mapInitializationNode = new MapInitializationNode();
 
-        for (int index = 0; index < keys.size(); ++index) {
+        for (int i = 0; i < keys.size(); ++i) {
             mapInitializationNode.addArgumentNode(
-                    keys.get(index).cast(keys.get(index).write(classNode)),
-                    values.get(index).cast(values.get(index).write(classNode)));
+                    keys.get(i).cast(keyOutputs.get(i)),
+                    values.get(i).cast(valueOutputs.get(i)));
         }
 
         mapInitializationNode.setLocation(location);
@@ -116,7 +112,9 @@ MapInitializationNode write(ClassNode classNode) {
         mapInitializationNode.setConstructor(constructor);
         mapInitializationNode.setMethod(method);
 
-        return mapInitializationNode;
+        output.expressionNode = mapInitializationNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
index 73603d79e6c89..a4ce05d57cf98 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
@@ -25,30 +25,31 @@
 import org.elasticsearch.painless.ir.NewArrayNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents an array instantiation.
  */
-public final class ENewArray extends AExpression {
+public class ENewArray extends AExpression {
 
-    private final String type;
-    private final List<AExpression> arguments;
-    private final boolean initialize;
+    protected final String type;
+    protected final List<AExpression> arguments;
+    protected final boolean initialize;
 
     public ENewArray(Location location, String type, List<AExpression> arguments, boolean initialize) {
         super(location);
 
         this.type = Objects.requireNonNull(type);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
         this.initialize = initialize;
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
              throw createError(new IllegalArgumentException("A newly created array must be read from."));
@@ -60,34 +61,32 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        for (int argument = 0; argument < arguments.size(); ++argument) {
-            AExpression expression = arguments.get(argument);
+        List<Output> argumentOutputs = new ArrayList<>();
 
+        for (AExpression expression : arguments) {
             Input expressionInput = new Input();
             expressionInput.expected = initialize ? clazz.getComponentType() : int.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            argumentOutputs.add(expressionOutput);
         }
 
         output.actual = clazz;
 
-        return output;
-    }
-
-    @Override
-    NewArrayNode write(ClassNode classNode) {
         NewArrayNode newArrayNode = new NewArrayNode();
 
-        for (AExpression argument : arguments) {
-            newArrayNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int i = 0; i < arguments.size(); ++ i) {
+            newArrayNode.addArgumentNode(arguments.get(i).cast(argumentOutputs.get(i)));
         }
 
         newArrayNode.setLocation(location);
         newArrayNode.setExpressionType(output.actual);
         newArrayNode.setInitialize(initialize);
 
-        return newArrayNode;
+        output.expressionNode = newArrayNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
index 0a132c12e5adb..16dc2ee729827 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
+import org.elasticsearch.painless.ir.FunctionNode;
 import org.elasticsearch.painless.ir.NewArrayFuncRefNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
@@ -34,11 +35,11 @@
 /**
  * Represents a function reference.
  */
-public final class ENewArrayFunctionRef extends AExpression implements ILambda {
-    private final String type;
+public class ENewArrayFunctionRef extends AExpression implements ILambda {
 
-    private SFunction function;
-    private FunctionRef ref;
+    protected final String type;
+
+    // TODO: #54015
     private String defPointer;
 
     public ENewArrayFunctionRef(Location location, String type) {
@@ -48,19 +49,20 @@ public ENewArrayFunctionRef(Location location, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         SReturn code = new SReturn(location, new ENewArray(location, type, Arrays.asList(new EVariable(location, "size")), false));
-        function = new SFunction(
+        SFunction function = new SFunction(
                 location, type, scriptRoot.getNextSyntheticName("newarray"),
                 Collections.singletonList("int"), Collections.singletonList("size"),
                 new SBlock(location, Collections.singletonList(code)), true, true, true, false);
         function.generateSignature(scriptRoot.getPainlessLookup());
-        function.analyze(scriptRoot);
+        FunctionNode functionNode = function.writeFunction(classNode, scriptRoot);
         scriptRoot.getFunctionTable().addFunction(function.name, function.returnType, function.typeParameters, true, true);
 
+        FunctionRef ref;
+
         if (input.expected == null) {
             ref = null;
             output.actual = String.class;
@@ -72,12 +74,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    NewArrayFuncRefNode write(ClassNode classNode) {
-        classNode.addFunctionNode(function.write(classNode));
+        classNode.addFunctionNode(functionNode);
 
         NewArrayFuncRefNode newArrayFuncRefNode = new NewArrayFuncRefNode();
 
@@ -85,7 +82,9 @@ NewArrayFuncRefNode write(ClassNode classNode) {
         newArrayFuncRefNode.setExpressionType(output.actual);
         newArrayFuncRefNode.setFuncRef(ref);
 
-        return newArrayFuncRefNode;
+        output.expressionNode = newArrayFuncRefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index dda96482a472d..1659910986173 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -28,6 +28,8 @@
 import org.elasticsearch.painless.spi.annotation.NonDeterministicAnnotation;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -36,24 +38,21 @@
 /**
  * Represents and object instantiation.
  */
-public final class ENewObj extends AExpression {
+public class ENewObj extends AExpression {
 
-    private final String type;
-    private final List<AExpression> arguments;
-
-    private PainlessConstructor constructor;
+    protected final String type;
+    protected final List<AExpression> arguments;
 
     public ENewObj(Location location, String type, List<AExpression> arguments) {
         super(location);
 
         this.type = Objects.requireNonNull(type);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
@@ -61,7 +60,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, arguments.size());
+        PainlessConstructor constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, arguments.size());
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
@@ -79,27 +78,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "]."));
         }
 
-        for (int argument = 0; argument < arguments.size(); ++argument) {
-            AExpression expression = arguments.get(argument);
+        List<Output> argumentOutputs = new ArrayList<>();
+
+        for (int i = 0; i < arguments.size(); ++i) {
+            AExpression expression = arguments.get(i);
 
             Input expressionInput = new Input();
-            expressionInput.expected = types[argument];
+            expressionInput.expected = types[i];
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            argumentOutputs.add(expressionOutput);
         }
 
         output.statement = true;
 
-        return output;
-    }
-
-    @Override
-    NewObjectNode write(ClassNode classNode) {
         NewObjectNode newObjectNode = new NewObjectNode();
 
-        for (AExpression argument : arguments) {
-            newObjectNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int i = 0; i < arguments.size(); ++ i) {
+            newObjectNode.addArgumentNode(arguments.get(i).cast(argumentOutputs.get(i)));
         }
 
         newObjectNode.setLocation(location);
@@ -107,7 +104,9 @@ NewObjectNode write(ClassNode classNode) {
         newObjectNode.setRead(input.read);
         newObjectNode.setConstructor(constructor);
 
-        return newObjectNode;
+        output.expressionNode = newObjectNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 841bd09586bed..1b61a05a68501 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -29,16 +29,15 @@
 /**
  * Represents a null constant.
  */
-public final class ENull extends AExpression {
+public class ENull extends AExpression {
 
     public ENull(Location location) {
         super(location);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from null constant."));
@@ -55,17 +54,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = Object.class;
         }
 
-        return output;
-    }
-
-    @Override
-    NullNode write(ClassNode classNode) {
         NullNode nullNode = new NullNode();
 
         nullNode.setLocation(location);
         nullNode.setExpressionType(output.actual);
 
-        return nullNode;
+        output.expressionNode = nullNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index f7cbe681b639d..1e548252f1a74 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,12 +30,10 @@
 /**
  * Represents a non-decimal numeric constant.
  */
-public final class ENumeric extends AExpression {
+public class ENumeric extends AExpression {
 
-    private final String value;
-    private int radix;
-
-    protected Object constant;
+    protected final String value;
+    protected final int radix;
 
     public ENumeric(Location location, String value, int radix) {
         super(location);
@@ -46,9 +43,10 @@ public ENumeric(Location location, String value, int radix) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Object constant;
+
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
@@ -114,17 +112,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
         }
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
index 3847397407dd8..23e92f018c371 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
@@ -42,11 +42,10 @@
 /**
  * Represents a regex constant. All regexes are constants.
  */
-public final class ERegex extends AExpression {
+public class ERegex extends AExpression {
 
-    private final String pattern;
-    private final int flags;
-    private String name;
+    protected final String pattern;
+    protected final int flags;
 
     public ERegex(Location location, String pattern, String flagsString) {
         super(location);
@@ -63,10 +62,8 @@ public ERegex(Location location, String pattern, String flagsString) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (scriptRoot.getCompilerSettings().areRegexesEnabled() == false) {
             throw createError(new IllegalStateException("Regexes are disabled. Set [script.painless.regex.enabled] to [true] "
@@ -85,14 +82,9 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     new IllegalArgumentException("Error compiling regex: " + e.getDescription()));
         }
 
-        name = scriptRoot.getNextSyntheticName("regex");
+        String name = scriptRoot.getNextSyntheticName("regex");
         output.actual = Pattern.class;
 
-        return output;
-    }
-
-    @Override
-    MemberFieldLoadNode write(ClassNode classNode) {
         FieldNode fieldNode = new FieldNode();
         fieldNode.setLocation(location);
         fieldNode.setModifiers(Modifier.FINAL | Modifier.STATIC | Modifier.PRIVATE);
@@ -169,7 +161,9 @@ MemberFieldLoadNode write(ClassNode classNode) {
         memberFieldLoadNode.setName(name);
         memberFieldLoadNode.setStatic(true);
 
-        return memberFieldLoadNode;
+        output.expressionNode = memberFieldLoadNode;
+
+        return output;
     }
 
     private int flagForChar(char c) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
index 0706e321e5312..b45ba741ae1a3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
@@ -30,9 +30,9 @@
 /**
  * Represents a static type target.
  */
-public final class EStatic extends AExpression {
+public class EStatic extends AExpression {
 
-    private final String type;
+    protected final String type;
 
     public EStatic(Location location, String type) {
         super(location);
@@ -41,9 +41,8 @@ public EStatic(Location location, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
 
@@ -51,17 +50,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    StaticNode write(ClassNode classNode) {
         StaticNode staticNode = new StaticNode();
 
         staticNode.setLocation(location);
         staticNode.setExpressionType(output.actual);
 
-        return staticNode;
+        output.expressionNode = staticNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
index 5cc5b67fe511b..77b9fb857bc08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,7 +30,7 @@
 /**
  * Represents a string constant.
  */
-public final class EString extends AExpression {
+public class EString extends AExpression {
 
     protected String constant;
 
@@ -42,9 +41,8 @@ public EString(Location location, String string) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
@@ -52,17 +50,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = String.class;
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index e498ceb3578c8..cafcda3b7dd2a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.UnaryMathNode;
-import org.elasticsearch.painless.ir.UnaryNode;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -35,13 +34,10 @@
 /**
  * Represents a unary math expression.
  */
-public final class EUnary extends AExpression {
+public class EUnary extends AExpression {
 
-    private final Operation operation;
-    private AExpression child;
-
-    private Class<?> promote;
-    private boolean originallyExplicit = false; // record whether there was originally an explicit cast
+    protected final Operation operation;
+    protected final AExpression child;
 
     public EUnary(Location location, Operation operation, AExpression child) {
         super(location);
@@ -51,21 +47,24 @@ public EUnary(Location location, Operation operation, AExpression child) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        Class<?> promote = null;
+        boolean originallyExplicit = input.explicit;
 
-        originallyExplicit = input.explicit;
+        Input childInput = new Input();
+        Output childOutput;
 
         if (operation == Operation.NOT) {
-            Input childInput = new Input();
+
             childInput.expected = boolean.class;
-            child.analyze(scriptRoot, scope, childInput);
-            child.cast();
+            childOutput = child.analyze(classNode, scriptRoot, scope, childInput);
+            child.cast(childInput, childOutput);
 
             output.actual = boolean.class;
         } else if (operation == Operation.BWNOT || operation == Operation.ADD || operation == Operation.SUB) {
-            Output childOutput = child.analyze(scriptRoot, scope, new Input());
+            childOutput = child.analyze(classNode, scriptRoot, scope, new Input());
 
             promote = AnalyzerCaster.promoteNumeric(childOutput.actual, operation != Operation.BWNOT);
 
@@ -75,8 +74,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(childOutput.actual) + "]"));
             }
 
-            child.input.expected = promote;
-            child.cast();
+            childInput.expected = promote;
+            child.cast(childInput, childOutput);
 
             if (promote == def.class && input.expected != null) {
                 output.actual = input.expected;
@@ -87,14 +86,9 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalStateException("unexpected unary operation [" + operation.name + "]"));
         }
 
-        return output;
-    }
-
-    @Override
-    UnaryNode write(ClassNode classNode) {
         UnaryMathNode unaryMathNode = new UnaryMathNode();
 
-        unaryMathNode.setChildNode(child.cast(child.write(classNode)));
+        unaryMathNode.setChildNode(child.cast(childOutput));
 
         unaryMathNode.setLocation(location);
         unaryMathNode.setExpressionType(output.actual);
@@ -102,7 +96,9 @@ UnaryNode write(ClassNode classNode) {
         unaryMathNode.setOperation(operation);
         unaryMathNode.setOriginallExplicit(originallyExplicit);
 
-        return unaryMathNode;
+        output.expressionNode = unaryMathNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
index 2f1a4d6f09a72..0c345dfa46735 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
@@ -31,9 +31,9 @@
 /**
  * Represents a variable load/store.
  */
-public final class EVariable extends AStoreable {
+public class EVariable extends AStoreable {
 
-    private final String name;
+    protected final String name;
 
     public EVariable(Location location, String name) {
         super(location);
@@ -42,20 +42,19 @@ public EVariable(Location location, String name) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
         AStoreable.Input storeableInput = new AStoreable.Input();
         storeableInput.read = input.read;
         storeableInput.expected = input.expected;
         storeableInput.explicit = input.explicit;
         storeableInput.internal = input.internal;
 
-        return analyze(scriptRoot, scope, storeableInput);
+        return analyze(classNode, scriptRoot, scope, storeableInput);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         Variable variable = scope.getVariable(location, name);
 
@@ -65,18 +64,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
 
         output.actual = variable.getType();
 
-        return output;
-    }
-
-    @Override
-    VariableNode write(ClassNode classNode) {
         VariableNode variableNode = new VariableNode();
 
         variableNode.setLocation(location);
         variableNode.setExpressionType(output.actual);
         variableNode.setName(name);
 
-        return variableNode;
+        output.expressionNode = variableNode;
+
+        return output;
     }
 
     @Override
@@ -84,11 +80,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(name);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 684b015aa835e..664a9e1c0048a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -34,11 +34,12 @@
 /**
  * Represents an array load/store and defers to a child subnode.
  */
-public final class PBrace extends AStoreable {
+public class PBrace extends AStoreable {
 
-    private AExpression index;
+    protected final AExpression index;
 
-    private AStoreable sub = null;
+    // TODO: #54015
+    private boolean isDefOptimized = false;
 
     public PBrace(Location location, AExpression prefix, AExpression index) {
         super(location, prefix);
@@ -47,24 +48,26 @@ public PBrace(Location location, AExpression prefix, AExpression index) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
         AStoreable.Input storeableInput = new AStoreable.Input();
         storeableInput.read = input.read;
         storeableInput.expected = input.expected;
         storeableInput.explicit = input.explicit;
         storeableInput.internal = input.internal;
 
-        return analyze(scriptRoot, scope, storeableInput);
+        return analyze(classNode, scriptRoot, scope, storeableInput);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
-        prefix.cast();
+        Input prefixInput = new Input();
+        Output prefixOutput = prefix.analyze(classNode, scriptRoot, scope, prefixInput);
+        prefixInput.expected = prefixOutput.actual;
+        prefix.cast(prefixInput, prefixOutput);
+
+        AStoreable sub;
 
         if (prefixOutput.actual.isArray()) {
             sub = new PSubBrace(location, prefixOutput.actual, index);
@@ -79,39 +82,32 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual) + "]."));
         }
 
+        isDefOptimized = sub.isDefOptimized();
+
         Input subInput = new Input();
         subInput.write = input.write;
         subInput.read = input.read;
         subInput.expected = input.expected;
         subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, subInput);
         output.actual = subOutput.actual;
 
-        return output;
-    }
-
-    @Override
-    BraceNode write(ClassNode classNode) {
         BraceNode braceNode = new BraceNode();
 
-        braceNode.setLeftNode(prefix.cast(prefix.write(classNode)));
-        braceNode.setRightNode(sub.write(classNode));
+        braceNode.setLeftNode(prefix.cast(prefixOutput));
+        braceNode.setRightNode(subOutput.expressionNode);
 
         braceNode.setLocation(location);
         braceNode.setExpressionType(output.actual);
 
-        return braceNode;
-    }
+        output.expressionNode = braceNode;
 
-    @Override
-    boolean isDefOptimized() {
-        return sub.isDefOptimized();
+        return output;
     }
 
     @Override
-    void updateActual(Class<?> actual) {
-        sub.updateActual(actual);
-        this.output.actual = actual;
+    boolean isDefOptimized() {
+        return isDefOptimized;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 4a0254ef96253..036c04f3878b5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.painless.spi.annotation.NonDeterministicAnnotation;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -36,30 +37,30 @@
 /**
  * Represents a method call and defers to a child subnode.
  */
-public final class PCallInvoke extends AExpression {
+public class PCallInvoke extends AExpression {
 
-    private final String name;
-    private final boolean nullSafe;
-    private final List<AExpression> arguments;
-
-    private AExpression sub = null;
+    protected final String name;
+    protected final boolean nullSafe;
+    protected final List<AExpression> arguments;
 
     public PCallInvoke(Location location, AExpression prefix, String name, boolean nullSafe, List<AExpression> arguments) {
         super(location, prefix);
 
         this.name = Objects.requireNonNull(name);
         this.nullSafe = nullSafe;
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        Input prefixInput = new Input();
+        Output prefixOutput = prefix.analyze(classNode, scriptRoot, scope, prefixInput);
+        prefixInput.expected = prefixOutput.actual;
+        prefix.cast(prefixInput, prefixOutput);
 
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
-        prefix.cast();
+        AExpression sub;
 
         if (prefixOutput.actual == def.class) {
             sub = new PSubDefCall(location, name, arguments);
@@ -84,25 +85,22 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input subInput = new Input();
         subInput.expected = input.expected;
         subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, subInput);
         output.actual = subOutput.actual;
 
         output.statement = true;
 
-        return output;
-    }
-
-    @Override
-    CallNode write(ClassNode classNode) {
         CallNode callNode = new CallNode();
 
-        callNode.setLeftNode(prefix.cast(prefix.write(classNode)));
-        callNode.setRightNode(sub.write(classNode));
+        callNode.setLeftNode(prefix.cast(prefixOutput));
+        callNode.setRightNode(subOutput.expressionNode);
 
         callNode.setLocation(location);
         callNode.setExpressionType(output.actual);
 
-        return callNode;
+        output.expressionNode = callNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 2380aa536edde..9751786e1515b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -38,12 +38,13 @@
 /**
  * Represents a field load/store and defers to a child subnode.
  */
-public final class PField extends AStoreable {
+public class PField extends AStoreable {
 
-    private final boolean nullSafe;
-    private final String value;
+    protected final boolean nullSafe;
+    protected final String value;
 
-    private AStoreable sub = null;
+    // TODO: #54015
+    private boolean isDefOptimized;
 
     public PField(Location location, AExpression prefix, boolean nullSafe, String value) {
         super(location, prefix);
@@ -53,24 +54,26 @@ public PField(Location location, AExpression prefix, boolean nullSafe, String va
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
         AStoreable.Input storeableInput = new AStoreable.Input();
         storeableInput.read = input.read;
         storeableInput.expected = input.expected;
         storeableInput.explicit = input.explicit;
         storeableInput.internal = input.internal;
 
-        return analyze(scriptRoot, scope, storeableInput);
+        return analyze(classNode, scriptRoot, scope, storeableInput);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
-        prefix.cast();
+        Input prefixInput = new Input();
+        Output prefixOutput = prefix.analyze(classNode, scriptRoot, scope, prefixInput);
+        prefixInput.expected = prefixOutput.actual;
+        prefix.cast(prefixInput, prefixOutput);
+
+        AStoreable sub = null;
 
         if (prefixOutput.actual.isArray()) {
             sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), value);
@@ -99,7 +102,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
                             location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
-                    index.analyze(scriptRoot, scope, new Input());
+                    index.analyze(classNode, scriptRoot, scope, new Input());
 
                     if (Map.class.isAssignableFrom(prefixOutput.actual)) {
                         sub = new PSubMapShortcut(location, prefixOutput.actual, index);
@@ -119,6 +122,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             }
         }
 
+        isDefOptimized = sub.isDefOptimized();
+
         if (nullSafe) {
             sub = new PSubNullSafeField(location, sub);
         }
@@ -128,34 +133,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
         subInput.read = input.read;
         subInput.expected = input.expected;
         subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, subInput);
         output.actual = subOutput.actual;
 
-        return output;
-    }
-
-    @Override
-    DotNode write(ClassNode classNode) {
         DotNode dotNode = new DotNode();
 
-        dotNode.setLeftNode(prefix.cast(prefix.write(classNode)));
-        dotNode.setRightNode(sub.write(classNode));
+        dotNode.setLeftNode(prefix.cast(prefixOutput));
+        dotNode.setRightNode(subOutput.expressionNode);
 
         dotNode.setLocation(location);
         dotNode.setExpressionType(output.actual);
 
-        return dotNode;
-    }
+        output.expressionNode = dotNode;
 
-    @Override
-    boolean isDefOptimized() {
-        return sub.isDefOptimized();
+        return output;
     }
 
     @Override
-    void updateActual(Class<?> actual) {
-        sub.updateActual(actual);
-        this.output.actual = actual;
+    boolean isDefOptimized() {
+        return isDefOptimized;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
index 43cdba046bd54..aaefa0717d3ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
@@ -30,10 +30,10 @@
 /**
  * Represents an array length field load.
  */
-final class PSubArrayLength extends AStoreable {
+public class PSubArrayLength extends AStoreable {
 
-    private final String type;
-    private final String value;
+    protected final String type;
+    protected final String value;
 
     PSubArrayLength(Location location, String type, String value) {
         super(location);
@@ -43,9 +43,8 @@ final class PSubArrayLength extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         if ("length".equals(value)) {
             if (input.write) {
@@ -57,27 +56,19 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Field [" + value + "] does not exist for type [" + type + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    DotSubArrayLengthNode write(ClassNode classNode) {
         DotSubArrayLengthNode dotSubArrayLengthNode = new DotSubArrayLengthNode();
 
         dotSubArrayLengthNode.setLocation(location);
         dotSubArrayLengthNode.setExpressionType(output.actual);
 
-        return dotSubArrayLengthNode;
-    }
+        output.expressionNode = dotSubArrayLengthNode;
 
-    @Override
-    boolean isDefOptimized() {
-        throw new IllegalStateException("Illegal tree structure.");
+        return output;
     }
 
     @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalStateException("Illegal tree structure.");
+    boolean isDefOptimized() {
+        return false;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
index 0303ce051f7f5..2a285d49acbe2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
@@ -30,10 +30,10 @@
 /**
  * Represents an array load/store.
  */
-final class PSubBrace extends AStoreable {
+public class PSubBrace extends AStoreable {
 
-    private final Class<?> clazz;
-    private AExpression index;
+    protected final Class<?> clazz;
+    protected final AExpression index;
 
     PSubBrace(Location location, Class<?> clazz, AExpression index) {
         super(location);
@@ -43,29 +43,26 @@ final class PSubBrace extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         Input indexInput = new Input();
         indexInput.expected = int.class;
-        index.analyze(scriptRoot, scope, indexInput);
-        index.cast();
+        Output indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+        index.cast(indexInput, indexOutput);
 
         output.actual = clazz.getComponentType();
 
-        return output;
-    }
-
-    BraceSubNode write(ClassNode classNode) {
         BraceSubNode braceSubNode = new BraceSubNode();
 
-        braceSubNode.setChildNode(index.cast(index.write(classNode)));
+        braceSubNode.setChildNode(index.cast(indexOutput));
 
         braceSubNode.setLocation(location);
         braceSubNode.setExpressionType(output.actual);
 
-        return braceSubNode;
+        output.expressionNode = braceSubNode;
+
+        return output;
     }
 
     @Override
@@ -73,11 +70,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw createError(new IllegalStateException("Illegal tree structure."));
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 7e509682f7d47..721e76cd3e500 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -26,30 +26,33 @@
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents a method call.
  */
-final class PSubCallInvoke extends AExpression {
+public class PSubCallInvoke extends AExpression {
 
-    private final PainlessMethod method;
-    private final Class<?> box;
-    private final List<AExpression> arguments;
+    protected final PainlessMethod method;
+    protected final Class<?> box;
+    protected final List<AExpression> arguments;
 
     PSubCallInvoke(Location location, PainlessMethod method, Class<?> box, List<AExpression> arguments) {
         super(location);
 
         this.method = Objects.requireNonNull(method);
         this.box = box;
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        List<Output> argumentOutputs = new ArrayList<>();
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
@@ -57,22 +60,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             Input expressionInput = new Input();
             expressionInput.expected = method.typeParameters.get(argument);
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            argumentOutputs.add(expressionOutput);
         }
 
         output.statement = true;
         output.actual = method.returnType;
 
-        return output;
-    }
-
-    @Override
-    CallSubNode write(ClassNode classNode) {
         CallSubNode callSubNode = new CallSubNode();
 
-        for (AExpression argument : arguments) {
-            callSubNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int argument = 0; argument < arguments.size(); ++ argument) {
+            callSubNode.addArgumentNode(arguments.get(argument).cast(argumentOutputs.get(argument)));
         }
 
         callSubNode.setLocation(location);
@@ -80,7 +79,9 @@ CallSubNode write(ClassNode classNode) {
         callSubNode.setMethod(method);
         callSubNode .setBox(box);
 
-        return callSubNode;
+        output.expressionNode = callSubNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index 060102ae8593f..00f09a493c0e9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -32,8 +32,9 @@
 /**
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
  */
-final class PSubDefArray extends AStoreable {
-    private AExpression index;
+public class PSubDefArray extends AStoreable {
+
+    protected AExpression index;
 
     PSubDefArray(Location location, AExpression index) {
         super(location);
@@ -42,30 +43,27 @@ final class PSubDefArray extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
-        Output indexOutput = index.analyze(scriptRoot, scope, new Input());
-        index.input.expected = indexOutput.actual;
-        index.cast();
+        Input indexInput = new Input();
+        Output indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+        indexInput.expected = indexOutput.actual;
+        index.cast(indexInput, indexOutput);
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
         output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
 
-        return output;
-    }
-
-    @Override
-    BraceSubDefNode write(ClassNode classNode) {
         BraceSubDefNode braceSubDefNode = new BraceSubDefNode();
 
-        braceSubDefNode.setChildNode(index.cast(index.write(classNode)));
+        braceSubDefNode.setChildNode(index.cast(indexOutput));
 
         braceSubDefNode.setLocation(location);
         braceSubDefNode.setExpressionType(output.actual);
 
-        return braceSubDefNode;
+        output.expressionNode = braceSubDefNode;
+
+        return output;
     }
 
     @Override
@@ -73,11 +71,6 @@ boolean isDefOptimized() {
         return true;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        this.output.actual = actual;
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index f810f1a634b6e..8339d161f2a84 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -28,51 +28,55 @@
 
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents a method call made on a def type. (Internal only.)
  */
-final class PSubDefCall extends AExpression {
+public class PSubDefCall extends AExpression {
 
-    private final String name;
-    private final List<AExpression> arguments;
-
-    private final StringBuilder recipe = new StringBuilder();
-    private final List<String> pointers = new ArrayList<>();
-    private final List<Class<?>> parameterTypes = new ArrayList<>();
+    protected final String name;
+    protected final List<AExpression> arguments;
 
     PSubDefCall(Location location, String name, List<AExpression> arguments) {
         super(location);
 
         this.name = Objects.requireNonNull(name);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        StringBuilder recipe = new StringBuilder();
+        List<String> pointers = new ArrayList<>();
+        List<Class<?>> parameterTypes = new ArrayList<>();
 
         parameterTypes.add(Object.class);
         int totalCaptures = 0;
 
+        List<Output> argumentOutputs = new ArrayList<>(arguments.size());
+
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
             Input expressionInput = new Input();
             expressionInput.internal = true;
-            Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            argumentOutputs.add(expressionOutput);
 
             if (expressionOutput.actual == void.class) {
                 throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "]."));
             }
 
-            expression.input.expected = expressionOutput.actual;
-            expression.cast();
+            expressionInput.expected = expressionOutput.actual;
+            expression.cast(expressionInput, expressionOutput);
             parameterTypes.add(expressionOutput.actual);
 
+            // TODO: #54015
             if (expression instanceof ILambda) {
                 ILambda lambda = (ILambda) expression;
                 pointers.add(lambda.getPointer());
@@ -87,15 +91,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
         output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
 
-        return output;
-    }
-
-    @Override
-    CallSubDefNode write(ClassNode classNode) {
         CallSubDefNode callSubDefNode = new CallSubDefNode();
 
-        for (AExpression argument : arguments) {
-            callSubDefNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int argument = 0; argument < arguments.size(); ++ argument) {
+            callSubDefNode.addArgumentNode(arguments.get(argument).cast(argumentOutputs.get(argument)));
         }
 
         callSubDefNode.setLocation(location);
@@ -105,7 +104,9 @@ CallSubDefNode write(ClassNode classNode) {
         callSubDefNode.getPointers().addAll(pointers);
         callSubDefNode.getTypeParameters().addAll(parameterTypes);
 
-        return callSubDefNode;
+        output.expressionNode = callSubDefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 75bfa5c5755cc..1cb8b2a0119e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -32,9 +32,9 @@
 /**
  * Represents a field load/store or shortcut on a def type.  (Internal only.)
  */
-final class PSubDefField extends AStoreable {
+public class PSubDefField extends AStoreable {
 
-    private final String value;
+    protected final String value;
 
     PSubDefField(Location location, String value) {
         super(location);
@@ -43,25 +43,21 @@ final class PSubDefField extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
         output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
 
-        return output;
-    }
-
-    @Override
-    DotSubDefNode write(ClassNode classNode) {
         DotSubDefNode dotSubDefNode = new DotSubDefNode();
 
         dotSubDefNode.setLocation(location);
         dotSubDefNode.setExpressionType(output.actual);
         dotSubDefNode.setValue(value);
 
-        return dotSubDefNode;
+        output.expressionNode = dotSubDefNode;
+
+        return output;
     }
 
     @Override
@@ -69,11 +65,6 @@ boolean isDefOptimized() {
         return true;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        this.output.actual = actual;
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, value);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index c9a438571bc36..26d2858f54afb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -33,9 +33,9 @@
 /**
  * Represents a field load/store.
  */
-final class PSubField extends AStoreable {
+public class PSubField extends AStoreable {
 
-    private final PainlessField field;
+    protected final PainlessField field;
 
     PSubField(Location location, PainlessField field) {
         super(location);
@@ -44,9 +44,8 @@ final class PSubField extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
          if (input.write && Modifier.isFinal(field.javaField.getModifiers())) {
              throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.javaField.getName() + "] " +
@@ -55,18 +54,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
 
          output.actual = field.typeParameter;
 
-         return output;
-    }
-
-    @Override
-    DotSubNode write(ClassNode classNode) {
         DotSubNode dotSubNode = new DotSubNode();
 
         dotSubNode.setLocation(location);
         dotSubNode.setExpressionType(output.actual);
         dotSubNode.setField(field);
 
-        return dotSubNode;
+        output.expressionNode = dotSubNode;
+
+        return output;
     }
 
     @Override
@@ -74,11 +70,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, field.javaField.getName());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 4dd3b46ceaa10..d23039453b3f8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -32,13 +32,10 @@
 /**
  * Represents a list load/store shortcut.  (Internal only.)
  */
-final class PSubListShortcut extends AStoreable {
+public class PSubListShortcut extends AStoreable {
 
-    private final Class<?> targetClass;
-    private AExpression index;
-
-    private PainlessMethod getter;
-    private PainlessMethod setter;
+    protected final Class<?> targetClass;
+    protected final AExpression index;
 
     PSubListShortcut(Location location, Class<?> targetClass, AExpression index) {
         super(location);
@@ -48,14 +45,13 @@ final class PSubListShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
-        getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
-        setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
+        PainlessMethod getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
+        PainlessMethod setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
 
         if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1 ||
             getter.typeParameters.get(0) != int.class)) {
@@ -71,32 +67,31 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
+        Output indexOutput = new Output();
+
         if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
             Input indexInput = new Input();
             indexInput.expected = int.class;
-            index.analyze(scriptRoot, scope, indexInput);
-            index.cast();
+            indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+            index.cast(indexInput, indexOutput);
 
             output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    ListSubShortcutNode write(ClassNode classNode) {
         ListSubShortcutNode listSubShortcutNode = new ListSubShortcutNode();
 
-        listSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
+        listSubShortcutNode.setChildNode(index.cast(indexOutput));
 
         listSubShortcutNode.setLocation(location);
         listSubShortcutNode.setExpressionType(output.actual);
         listSubShortcutNode.setGetter(getter);
         listSubShortcutNode.setSetter(setter);
 
-        return listSubShortcutNode;
+        output.expressionNode = listSubShortcutNode;
+
+        return output;
     }
 
     @Override
@@ -104,11 +99,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 707609e5a4947..4105bcec8b261 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -32,13 +32,10 @@
 /**
  * Represents a map load/store shortcut. (Internal only.)
  */
-final class PSubMapShortcut extends AStoreable {
+public class PSubMapShortcut extends AStoreable {
 
-    private final Class<?> targetClass;
-    private AExpression index;
-
-    private PainlessMethod getter;
-    private PainlessMethod setter;
+    protected final Class<?> targetClass;
+    protected final AExpression index;
 
     PSubMapShortcut(Location location, Class<?> targetClass, AExpression index) {
         super(location);
@@ -48,14 +45,13 @@ final class PSubMapShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
-        getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
-        setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
+        PainlessMethod getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
+        PainlessMethod setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
 
         if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1)) {
             throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "]."));
@@ -70,32 +66,31 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
+        Output indexOutput;
+
         if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
             Input indexInput = new Input();
             indexInput.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
-            index.analyze(scriptRoot, scope, indexInput);
-            index.cast();
+            indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+            index.cast(indexInput, indexOutput);
 
             output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    MapSubShortcutNode write(ClassNode classNode) {
         MapSubShortcutNode mapSubShortcutNode = new MapSubShortcutNode();
 
-        mapSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
+        mapSubShortcutNode.setChildNode(index.cast(indexOutput));
 
         mapSubShortcutNode.setLocation(location);
         mapSubShortcutNode.setExpressionType(output.actual);
         mapSubShortcutNode.setGetter(getter);
         mapSubShortcutNode.setSetter(setter);
 
-        return mapSubShortcutNode;
+        output.expressionNode = mapSubShortcutNode;
+
+        return output;
     }
 
     @Override
@@ -103,11 +98,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
index 83d5fded5649d..bc9a259a2e96c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
@@ -31,10 +31,11 @@
  * Implements a call who's value is null if the prefix is null rather than throwing an NPE.
  */
 public class PSubNullSafeCallInvoke extends AExpression {
+
     /**
-     * The expression gaurded by the null check. Required at construction time and replaced at analysis time.
+     * The expression guarded by the null check. Required at construction time and replaced at analysis time.
      */
-    private AExpression guarded;
+    protected final AExpression guarded;
 
     public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
         super(location);
@@ -42,29 +43,25 @@ public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
-        Output guardedOutput = guarded.analyze(scriptRoot, scope, new Input());
+        Output guardedOutput = guarded.analyze(classNode, scriptRoot, scope, new Input());
         output.actual = guardedOutput.actual;
         if (output.actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
 
-        return output;
-    }
-
-    @Override
-    NullSafeSubNode write(ClassNode classNode) {
         NullSafeSubNode nullSafeSubNode = new NullSafeSubNode();
 
-        nullSafeSubNode.setChildNode(guarded.write(classNode));
+        nullSafeSubNode.setChildNode(guardedOutput.expressionNode);
 
         nullSafeSubNode.setLocation(location);
         nullSafeSubNode.setExpressionType(output.actual);
 
-        return nullSafeSubNode;
+        output.expressionNode = nullSafeSubNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
index 536c8b15e83c6..22d4c91945273 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
@@ -29,7 +29,8 @@
  * Implements a field who's value is null if the prefix is null rather than throwing an NPE.
  */
 public class PSubNullSafeField extends AStoreable {
-    private AStoreable guarded;
+
+    protected final AStoreable guarded;
 
     public PSubNullSafeField(Location location, AStoreable guarded) {
         super(location);
@@ -37,44 +38,37 @@ public PSubNullSafeField(Location location, AStoreable guarded) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         if (input.write) {
             throw createError(new IllegalArgumentException("Can't write to null safe reference"));
         }
+
         Input guardedInput = new Input();
         guardedInput.read = input.read;
-        Output guardedOutput = guarded.analyze(scriptRoot, scope, guardedInput);
+        Output guardedOutput = guarded.analyze(classNode, scriptRoot, scope, guardedInput);
         output.actual = guardedOutput.actual;
+
         if (output.actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
 
-        return output;
-    }
-
-    @Override
-    boolean isDefOptimized() {
-        return guarded.isDefOptimized();
-    }
-
-    @Override
-    void updateActual(Class<?> actual) {
-        guarded.updateActual(actual);
-    }
-
-    @Override
-    NullSafeSubNode write(ClassNode classNode) {
         NullSafeSubNode nullSafeSubNode = new NullSafeSubNode();
 
-        nullSafeSubNode.setChildNode(guarded.write(classNode));
+        nullSafeSubNode.setChildNode(guardedOutput.expressionNode);
 
         nullSafeSubNode.setLocation(location);
         nullSafeSubNode.setExpressionType(output.actual);
 
-        return nullSafeSubNode;
+        output.expressionNode = nullSafeSubNode;
+
+        return output;
+    }
+
+    @Override
+    boolean isDefOptimized() {
+        return false;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index e88fbeaac0765..a55a5556bafca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -29,12 +29,12 @@
 /**
  * Represents a field load/store shortcut.  (Internal only.)
  */
-final class PSubShortcut extends AStoreable {
+public class PSubShortcut extends AStoreable {
 
-    private final String value;
-    private final String type;
-    private final PainlessMethod getter;
-    private final PainlessMethod setter;
+    protected final String value;
+    protected final String type;
+    protected final PainlessMethod getter;
+    protected final PainlessMethod setter;
 
     PSubShortcut(Location location, String value, String type, PainlessMethod getter, PainlessMethod setter) {
         super(location);
@@ -46,9 +46,8 @@ final class PSubShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) {
             throw createError(new IllegalArgumentException(
@@ -70,11 +69,6 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + type + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    DotSubShortcutNode write(ClassNode classNode) {
         DotSubShortcutNode dotSubShortcutNode = new DotSubShortcutNode();
 
         dotSubShortcutNode.setLocation(location);
@@ -82,7 +76,9 @@ DotSubShortcutNode write(ClassNode classNode) {
         dotSubShortcutNode.setGetter(getter);
         dotSubShortcutNode.setSetter(setter);
 
-        return dotSubShortcutNode;
+        output.expressionNode = dotSubShortcutNode;
+
+        return output;
     }
 
     @Override
@@ -90,11 +86,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, value);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 8e6db345e0b30..f662976c2e4b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -33,9 +34,9 @@
 /**
  * Represents a set of statements as a branch of control-flow.
  */
-public final class SBlock extends AStatement {
+public class SBlock extends AStatement {
 
-    final List<AStatement> statements;
+    protected final List<AStatement> statements;
 
     public SBlock(Location location, List<AStatement> statements) {
         super(location);
@@ -44,9 +45,8 @@ public SBlock(Location location, List<AStatement> statements) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
@@ -54,6 +54,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         AStatement last = statements.get(statements.size() - 1);
 
+        List<Output> statementOutputs = new ArrayList<>(statements.size());
+
         for (AStatement statement : statements) {
             // Note that we do not need to check after the last statement because
             // there is no statement that can be unreachable after the last.
@@ -65,7 +67,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             statementInput.inLoop = input.inLoop;
             statementInput.lastSource = input.lastSource && statement == last;
             statementInput.lastLoop = (input.beginLoop || input.lastLoop) && statement == last;
-            Output statementOutput = statement.analyze(scriptRoot, scope, statementInput);
+
+            Output statementOutput = statement.analyze(classNode, scriptRoot, scope, statementInput);
 
             output.methodEscape = statementOutput.methodEscape;
             output.loopEscape = statementOutput.loopEscape;
@@ -73,24 +76,23 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.anyContinue |= statementOutput.anyContinue;
             output.anyBreak |= statementOutput.anyBreak;
             output.statementCount += statementOutput.statementCount;
-        }
 
-        return output;
-    }
+            statementOutputs.add(statementOutput);
+        }
 
-    @Override
-    BlockNode write(ClassNode classNode) {
         BlockNode blockNode = new BlockNode();
 
-        for (AStatement statement : statements) {
-            blockNode.addStatementNode(statement.write(classNode));
+        for (Output statementOutput : statementOutputs) {
+            blockNode.addStatementNode(statementOutput.statementNode);
         }
 
         blockNode.setLocation(location);
         blockNode.setAllEscape(output.allEscape);
         blockNode.setStatementCount(output.statementCount);
 
-        return blockNode;
+        output.statementNode = blockNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 4b036153319ea..55d22a2097a9f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -28,16 +28,15 @@
 /**
  * Represents a break statement.
  */
-public final class SBreak extends AStatement {
+public class SBreak extends AStatement {
 
     public SBreak(Location location) {
         super(location);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
@@ -48,16 +47,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.anyBreak = true;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    BreakNode write(ClassNode classNode) {
         BreakNode breakNode = new BreakNode();
-
         breakNode.setLocation(location);
 
-        return breakNode;
+        output.statementNode = breakNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index f00c0b18a70fc..06dc1a06514fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -21,8 +21,10 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.CatchNode;
 import org.elasticsearch.painless.ir.ClassNode;
+import org.elasticsearch.painless.ir.DeclarationNode;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
@@ -31,11 +33,11 @@
 /**
  * Represents a catch block as part of a try-catch block.
  */
-public final class SCatch extends AStatement {
+public class SCatch extends AStatement {
 
-    private final DType baseException;
-    private final SDeclaration declaration;
-    private final SBlock block;
+    protected final DType baseException;
+    protected final SDeclaration declaration;
+    protected final SBlock block;
 
     public SCatch(Location location, DType baseException, SDeclaration declaration, SBlock block) {
         super(location);
@@ -46,11 +48,10 @@ public SCatch(Location location, DType baseException, SDeclaration declaration,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
-        declaration.analyze(scriptRoot, scope, new Input());
+        Output declarationOutput = declaration.analyze(classNode, scriptRoot, scope, new Input());
 
         Class<?> baseType = baseException.resolveType(scriptRoot.getPainlessLookup()).getType();
         Class<?> type = scope.getVariable(location, declaration.name).getType();
@@ -61,12 +62,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     "to [" + PainlessLookupUtility.typeToCanonicalTypeName(baseType) + "]"));
         }
 
+        Output blockOutput = null;
+
         if (block != null) {
             Input blockInput = new Input();
             blockInput.lastSource = input.lastSource;
             blockInput.inLoop = input.inLoop;
             blockInput.lastLoop = input.lastLoop;
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
             output.methodEscape = blockOutput.methodEscape;
             output.loopEscape = blockOutput.loopEscape;
@@ -76,19 +79,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.statementCount = blockOutput.statementCount;
         }
 
-        return output;
-    }
-
-    @Override
-    CatchNode write(ClassNode classNode) {
         CatchNode catchNode = new CatchNode();
-
-        catchNode.setDeclarationNode(declaration.write(classNode));
-        catchNode.setBlockNode(block == null ? null : block.write(classNode));
-
+        catchNode.setDeclarationNode((DeclarationNode)declarationOutput.statementNode);
+        catchNode.setBlockNode(blockOutput == null ? null : (BlockNode)blockOutput.statementNode);
         catchNode.setLocation(location);
 
-        return catchNode;
+        output.statementNode = catchNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java
index d946b9b1c69b5..1c03994d55d06 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.StatementNode;
 import org.elasticsearch.painless.symbol.FunctionTable;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 import org.objectweb.asm.util.Printer;
@@ -36,37 +35,25 @@
 /**
  * The root of all Painless trees.  Contains a series of statements.
  */
-public final class SClass extends ANode {
+public class SClass extends ANode {
 
-    private final ScriptClassInfo scriptClassInfo;
-    private final String name;
-    private final Printer debugStream;
-    private final List<SFunction> functions = new ArrayList<>();
-    private final List<SField> fields = new ArrayList<>();
-
-    private ScriptRoot scriptRoot;
-    private final String sourceText;
+    protected final ScriptClassInfo scriptClassInfo;
+    protected final String name;
+    protected final String sourceText;
+    protected final Printer debugStream;
+    protected final List<SFunction> functions = new ArrayList<>();
 
     public SClass(ScriptClassInfo scriptClassInfo, String name, String sourceText, Printer debugStream,
             Location location, List<SFunction> functions) {
         super(location);
         this.scriptClassInfo = Objects.requireNonNull(scriptClassInfo);
         this.name = Objects.requireNonNull(name);
+        this.sourceText = Objects.requireNonNull(sourceText);
         this.debugStream = debugStream;
         this.functions.addAll(Objects.requireNonNull(functions));
-        this.sourceText = Objects.requireNonNull(sourceText);
-    }
-
-    void addFunction(SFunction function) {
-        functions.add(function);
     }
 
-    void addField(SField field) {
-        fields.add(field);
-    }
-
-    public ScriptRoot analyze(ScriptRoot scriptRoot) {
-        this.scriptRoot = scriptRoot;
+    public ClassNode writeClass(ScriptRoot scriptRoot) {
         scriptRoot.addStaticConstant("$NAME", name);
         scriptRoot.addStaticConstant("$SOURCE", sourceText);
 
@@ -83,31 +70,10 @@ public ScriptRoot analyze(ScriptRoot scriptRoot) {
                     function.name, function.returnType, function.typeParameters, function.isInternal, function.isStatic);
         }
 
-        // copy protection is required because synthetic functions are
-        // added for lambdas/method references and analysis here is
-        // only for user-defined functions
-        List<SFunction> functions = new ArrayList<>(this.functions);
-        for (SFunction function : functions) {
-            function.analyze(scriptRoot);
-        }
-
-        return scriptRoot;
-    }
-
-    @Override
-    public StatementNode write(ClassNode classNode) {
-        throw new UnsupportedOperationException();
-    }
-
-    public ClassNode writeClass() {
         ClassNode classNode = new ClassNode();
 
-        for (SField field : fields) {
-            classNode.addFieldNode(field.write(classNode));
-        }
-
         for (SFunction function : functions) {
-            classNode.addFunctionNode(function.write(classNode));
+            classNode.addFunctionNode(function.writeFunction(classNode, scriptRoot));
         }
 
         classNode.setLocation(location);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 1f5c752f8ccef..830455274f9e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -28,16 +28,15 @@
 /**
  * Represents a continue statement.
  */
-public final class SContinue extends AStatement {
+public class SContinue extends AStatement {
 
     public SContinue(Location location) {
         super(location);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
@@ -51,16 +50,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.anyContinue = true;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ContinueNode write(ClassNode classNode) {
         ContinueNode continueNode = new ContinueNode();
-
         continueNode.setLocation(location);
 
-        return continueNode;
+        output.statementNode = continueNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 47bbe0d123ecb..4bbfcf6d1d505 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -23,8 +23,10 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DeclarationBlockNode;
+import org.elasticsearch.painless.ir.DeclarationNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -33,9 +35,9 @@
 /**
  * Represents a series of declarations.
  */
-public final class SDeclBlock extends AStatement {
+public class SDeclBlock extends AStatement {
 
-    private final List<SDeclaration> declarations;
+    protected final List<SDeclaration> declarations;
 
     public SDeclBlock(Location location, List<SDeclaration> declarations) {
         super(location);
@@ -44,30 +46,28 @@ public SDeclBlock(Location location, List<SDeclaration> declarations) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        List<Output> declarationOutputs = new ArrayList<>(declarations.size());
 
         for (SDeclaration declaration : declarations) {
-            declaration.analyze(scriptRoot, scope, new Input());
+            declarationOutputs.add(declaration.analyze(classNode, scriptRoot, scope, new Input()));
         }
 
         output.statementCount = declarations.size();
 
-        return output;
-    }
-
-    @Override
-    DeclarationBlockNode write(ClassNode classNode) {
         DeclarationBlockNode declarationBlockNode = new DeclarationBlockNode();
 
-        for (SDeclaration declaration : declarations) {
-            declarationBlockNode.addDeclarationNode(declaration.write(classNode));
+        for (Output declarationOutput : declarationOutputs) {
+            declarationBlockNode.addDeclarationNode((DeclarationNode)declarationOutput.statementNode);
         }
 
         declarationBlockNode.setLocation(location);
 
-        return declarationBlockNode;
+        output.statementNode = declarationBlockNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 2a28b4d029558..22b71eb931a79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DeclarationNode;
-import org.elasticsearch.painless.node.AExpression.Input;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,12 +30,12 @@
 /**
  * Represents a single variable declaration.
  */
-public final class SDeclaration extends AStatement {
+public class SDeclaration extends AStatement {
 
-    private DType type;
+    protected final DType type;
     protected final String name;
     protected final boolean requiresDefault;
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SDeclaration(Location location, DType type, String name, boolean requiresDefault, AExpression expression) {
         super(location);
@@ -48,37 +47,32 @@ public SDeclaration(Location location, DType type, String name, boolean requires
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         DResolvedType resolvedType = type.resolveType(scriptRoot.getPainlessLookup());
-        type = resolvedType;
+
+        AExpression.Output expressionOutput = null;
 
         if (expression != null) {
             AExpression.Input expressionInput = new AExpression.Input();
             expressionInput.expected = resolvedType.getType();
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
         }
 
         scope.defineVariable(location, resolvedType.getType(), name, false);
 
-        return output;
-    }
-
-    @Override
-    DeclarationNode write(ClassNode classNode) {
         DeclarationNode declarationNode = new DeclarationNode();
-
-        declarationNode.setExpressionNode(expression == null ? null : expression.cast(expression.write(classNode)));
-
+        declarationNode.setExpressionNode(expression == null ? null : expression.cast(expressionOutput));
         declarationNode.setLocation(location);
-        declarationNode.setDeclarationType(((DResolvedType)type).getType());
+        declarationNode.setDeclarationType(resolvedType.getType());
         declarationNode.setName(name);
         declarationNode.setRequiresDefault(requiresDefault);
 
-        return declarationNode;
+        output.statementNode = declarationNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 8d31f325a049f..f5f4b0636a370 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DoWhileLoopNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -30,12 +31,10 @@
 /**
  * Represents a do-while loop.
  */
-public final class SDo extends AStatement {
+public class SDo extends AStatement {
 
-    private final SBlock block;
-    private AExpression condition;
-
-    private boolean continuous = false;
+    protected final SBlock block;
+    protected final AExpression condition;
 
     public SDo(Location location, SBlock block, AExpression condition) {
         super(location);
@@ -45,10 +44,8 @@ public SDo(Location location, SBlock block, AExpression condition) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
         scope = scope.newLocalScope();
 
         if (block == null) {
@@ -58,7 +55,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input blockInput = new Input();
         blockInput.beginLoop = true;
         blockInput.inLoop = true;
-        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        Output blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
         if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
@@ -66,8 +63,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
+
+        boolean continuous = false;
 
         if (condition instanceof EBoolean) {
             continuous = ((EBoolean)condition).constant;
@@ -84,20 +83,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    DoWhileLoopNode write(ClassNode classNode) {
         DoWhileLoopNode doWhileLoopNode = new DoWhileLoopNode();
-
-        doWhileLoopNode.setConditionNode(condition.cast(condition.write(classNode)));
-        doWhileLoopNode.setBlockNode(block.write(classNode));
-
+        doWhileLoopNode.setConditionNode(condition.cast(conditionOutput));
+        doWhileLoopNode.setBlockNode((BlockNode)blockOutput.statementNode);
         doWhileLoopNode.setLocation(location);
         doWhileLoopNode.setContinuous(continuous);
 
-        return doWhileLoopNode;
+        output.statementNode = doWhileLoopNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index b5c439233b64d..ac8741cc551ec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -36,12 +36,10 @@
  */
 public class SEach extends AStatement {
 
-    private final String type;
-    private final String name;
-    private AExpression expression;
-    private final SBlock block;
-
-    private AStatement sub = null;
+    protected final String type;
+    protected final String name;
+    protected final AExpression expression;
+    protected final SBlock block;
 
     public SEach(Location location, String type, String name, AExpression expression, SBlock block) {
         super(location);
@@ -53,13 +51,14 @@ public SEach(Location location, String type, String name, AExpression expression
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
-        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, new AExpression.Input());
-        expression.input.expected = expressionOutput.actual;
-        expression.cast();
+        AExpression.Input expressionInput = new AExpression.Input();
+        AExpression.Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+        // TODO: no need to cast here
+        expressionInput.expected = expressionOutput.actual;
+        expression.cast(expressionInput, expressionOutput);
 
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
@@ -70,17 +69,6 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         scope = scope.newLocalScope();
         Variable variable = scope.defineVariable(location, clazz, name, true);
 
-        if (expressionOutput.actual.isArray()) {
-            sub = new SSubEachArray(location, variable, expression, block);
-        } else if (expressionOutput.actual == def.class || Iterable.class.isAssignableFrom(expressionOutput.actual)) {
-            sub = new SSubEachIterable(location, variable, expression, block);
-        } else {
-            throw createError(new IllegalArgumentException("Illegal for each type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
-        }
-
-        sub.analyze(scriptRoot, scope, input);
-
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous for each loop."));
         }
@@ -88,27 +76,36 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input blockInput = new Input();
         blockInput.beginLoop = true;
         blockInput.inLoop = true;
-        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        Output blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
         blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
 
         if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous for loop."));
         }
 
-        output.statementCount = 1;
+        AStatement sub;
 
-        return output;
-    }
+        if (expressionOutput.actual.isArray()) {
+            sub = new SSubEachArray(location, variable, expressionOutput, blockOutput);
+        } else if (expressionOutput.actual == def.class || Iterable.class.isAssignableFrom(expressionOutput.actual)) {
+            sub = new SSubEachIterable(location, variable, expressionOutput, blockOutput);
+        } else {
+            throw createError(new IllegalArgumentException("Illegal for each type " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
+        }
 
-    @Override
-    ForEachLoopNode write(ClassNode classNode) {
-        ForEachLoopNode forEachLoopNode = new ForEachLoopNode();
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, input);
 
-        forEachLoopNode.setConditionNode((ConditionNode)sub.write(classNode));
+        output.statementCount = 1;
 
+        ForEachLoopNode forEachLoopNode = new ForEachLoopNode();
+        forEachLoopNode.setConditionNode((ConditionNode)subOutput.statementNode);
         forEachLoopNode.setLocation(location);
 
-        return forEachLoopNode;
+        output.statementNode = forEachLoopNode;
+
+        return output;
+
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index f04cefa622acd..ace1be4b96736 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.ir.ReturnNode;
 import org.elasticsearch.painless.ir.StatementExpressionNode;
-import org.elasticsearch.painless.ir.StatementNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -33,9 +32,9 @@
 /**
  * Represents the top-level node for an expression as a statement.
  */
-public final class SExpression extends AStatement {
+public class SExpression extends AStatement {
 
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SExpression(Location location, AExpression expression) {
         super(location);
@@ -44,15 +43,13 @@ public SExpression(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         Class<?> rtnType = scope.getReturnType();
         boolean isVoid = rtnType == void.class;
 
         AExpression.Input expressionInput = new AExpression.Input();
         expressionInput.read = input.lastSource && !isVoid;
-        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
+        AExpression.Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
 
         if ((input.lastSource == false || isVoid) && expressionOutput.statement == false) {
             throw createError(new IllegalArgumentException("Not a statement."));
@@ -60,40 +57,33 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         boolean rtn = input.lastSource && isVoid == false && expressionOutput.actual != void.class;
 
-        expression.input.expected = rtn ? rtnType : expressionOutput.actual;
-        expression.input.internal = rtn;
-        expression.cast();
+        expressionInput.expected = rtn ? rtnType : expressionOutput.actual;
+        expressionInput.internal = rtn;
+        expression.cast(expressionInput, expressionOutput);
 
-        output = new Output();
+        Output output = new Output();
         output.methodEscape = rtn;
         output.loopEscape = rtn;
         output.allEscape = rtn;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    StatementNode write(ClassNode classNode) {
-        ExpressionNode expressionNode = expression.cast(expression.write(classNode));
+        ExpressionNode expressionNode = expression.cast(expressionOutput);
 
         if (output.methodEscape) {
             ReturnNode returnNode = new ReturnNode();
-
             returnNode.setExpressionNode(expressionNode);
-
             returnNode.setLocation(location);
 
-            return returnNode;
+            output.statementNode = returnNode;
         } else {
             StatementExpressionNode statementExpressionNode = new StatementExpressionNode();
-
             statementExpressionNode.setExpressionNode(expressionNode);
-
             statementExpressionNode.setLocation(location);
 
-            return statementExpressionNode;
+            output.statementNode = statementExpressionNode;
         }
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java
deleted file mode 100644
index 944802fb7ac39..0000000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.node;
-
-import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.FieldNode;
-
-/**
- * Represents a member field for its parent class (internal only).
- */
-public class SField extends ANode {
-
-    private final int modifiers;
-    private final String name;
-    private final Class<?> type;
-
-    /**
-     * Standard constructor.
-     * @param location original location in the source
-     * @param modifiers java modifiers for the field
-     * @param name name of the field
-     * @param type type of the field
-     */
-    public SField(Location location, int modifiers, String name, Class<?> type) {
-        super(location);
-
-        this.modifiers = modifiers;
-        this.name = name;
-        this.type = type;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    @Override
-    FieldNode write(ClassNode classNode) {
-        FieldNode fieldNode = new FieldNode();
-
-        fieldNode.setLocation(location);
-        fieldNode.setModifiers(modifiers);
-        fieldNode.setName(name);
-        fieldNode.setFieldType(type);
-
-        return fieldNode;
-    }
-
-    @Override
-    public String toString() {
-        return singleLineToString(name, type);
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 241c50b8fe2b9..ce43e9a434f1a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -21,8 +21,8 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.ir.ForLoopNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
@@ -33,14 +33,12 @@
 /**
  * Represents a for loop.
  */
-public final class SFor extends AStatement {
+public class SFor extends AStatement {
 
-    private ANode initializer;
-    private AExpression condition;
-    private AExpression afterthought;
-    private final SBlock block;
-
-    private boolean continuous = false;
+    protected final ANode initializer;
+    protected final AExpression condition;
+    protected final AExpression afterthought;
+    protected final SBlock block;
 
     public SFor(Location location, ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
         super(location);
@@ -52,37 +50,42 @@ public SFor(Location location, ANode initializer, AExpression condition, AExpres
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         scope = scope.newLocalScope();
 
+        Output initializerStatementOutput = null;
+        AExpression.Output initializerExpressionOutput = null;
+
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(scriptRoot, scope, new Input());
+                initializerStatementOutput = ((SDeclBlock)initializer).analyze(classNode, scriptRoot, scope, new Input());
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
                 AExpression.Input initializerInput = new AExpression.Input();
                 initializerInput.read = false;
-                AExpression.Output initializerOutput = initializer.analyze(scriptRoot, scope, initializerInput);
+                initializerExpressionOutput = initializer.analyze(classNode, scriptRoot, scope, initializerInput);
 
-                if (initializerOutput.statement == false) {
+                if (initializerExpressionOutput.statement == false) {
                     throw createError(new IllegalArgumentException("Not a statement."));
                 }
 
-                initializer.input.expected = initializerOutput.actual;
-                initializer.cast();
+                initializerInput.expected = initializerExpressionOutput.actual;
+                initializer.cast(initializerInput, initializerExpressionOutput);
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
+        boolean continuous = false;
+
+        AExpression.Output conditionOutput = null;
+
         if (condition != null) {
             AExpression.Input conditionInput = new AExpression.Input();
             conditionInput.expected = boolean.class;
-            condition.analyze(scriptRoot, scope, conditionInput);
-            condition.cast();
+            conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+            condition.cast(conditionInput, conditionOutput);
 
             if (condition instanceof EBoolean) {
                 continuous = ((EBoolean)condition).constant;
@@ -99,27 +102,30 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             continuous = true;
         }
 
+        AExpression.Output afterthoughtOutput = null;
+
         if (afterthought != null) {
             AExpression.Input afterthoughtInput = new AExpression.Input();
             afterthoughtInput.read = false;
-            AExpression.Output afterthoughtOutput = afterthought.analyze(scriptRoot, scope, afterthoughtInput);
+            afterthoughtOutput = afterthought.analyze(classNode, scriptRoot, scope, afterthoughtInput);
 
             if (afterthoughtOutput.statement == false) {
                 throw createError(new IllegalArgumentException("Not a statement."));
             }
 
-            afterthought.input.expected = afterthoughtOutput.actual;
-            afterthought.cast();
+            afterthoughtInput.expected = afterthoughtOutput.actual;
+            afterthought.cast(afterthoughtInput, afterthoughtOutput);
         }
 
-        output = new Output();
+        Output output = new Output();
+        Output blockOutput = null;
 
         if (block != null) {
             Input blockInput = new Input();
             blockInput.beginLoop = true;
             blockInput.inLoop = true;
 
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
             if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
@@ -135,24 +141,19 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ForLoopNode write(ClassNode classNode) {
         ForLoopNode forLoopNode = new ForLoopNode();
-
         forLoopNode.setInitialzerNode(initializer == null ? null : initializer instanceof AExpression ?
-                ((AExpression)initializer).cast((ExpressionNode)initializer.write(classNode)) :
-                initializer.write(classNode));
-        forLoopNode.setConditionNode(condition == null ? null : condition.cast(condition.write(classNode)));
-        forLoopNode.setAfterthoughtNode(afterthought == null ? null : afterthought.cast(afterthought.write(classNode)));
-        forLoopNode.setBlockNode(block == null ? null : block.write(classNode));
-
+                ((AExpression)initializer).cast(initializerExpressionOutput) :
+                initializerStatementOutput.statementNode);
+        forLoopNode.setConditionNode(condition == null ? null : condition.cast(conditionOutput));
+        forLoopNode.setAfterthoughtNode(afterthought == null ? null : afterthought.cast(afterthoughtOutput));
+        forLoopNode.setBlockNode(blockOutput == null ? null : (BlockNode)blockOutput.statementNode);
         forLoopNode.setLocation(location);
         forLoopNode.setContinuous(continuous);
 
-        return forLoopNode;
+        output.statementNode = forLoopNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 96bf040099a17..fad71982b4772 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -46,32 +46,28 @@
 /**
  * Represents a user-defined function.
  */
-public final class SFunction extends ANode {
+public class SFunction extends ANode {
 
-    private final String rtnTypeStr;
-    public final String name;
-    private final List<String> paramTypeStrs;
-    private final List<String> paramNameStrs;
-    private final SBlock block;
-    public final boolean isInternal;
-    public final boolean isStatic;
-    public final boolean synthetic;
+    protected final String rtnTypeStr;
+    protected final String name;
+    protected final List<String> paramTypeStrs;
+    protected final List<String> paramNameStrs;
+    protected final SBlock block;
+    protected final boolean isInternal;
+    protected final boolean isStatic;
+    protected final boolean synthetic;
 
     /**
      * If set to {@code true} default return values are inserted if
      * not all paths return a value.
      */
-    public final boolean isAutoReturnEnabled;
+    protected final boolean isAutoReturnEnabled;
 
-    private int maxLoopCounter;
+    protected Class<?> returnType;
+    protected List<Class<?>> typeParameters;
+    protected MethodType methodType;
 
-    Class<?> returnType;
-    List<Class<?>> typeParameters;
-    MethodType methodType;
-
-    org.objectweb.asm.commons.Method method;
-
-    private boolean methodEscape;
+    protected org.objectweb.asm.commons.Method method;
 
     public SFunction(Location location, String rtnType, String name,
             List<String> paramTypes, List<String> paramNames,
@@ -89,6 +85,7 @@ public SFunction(Location location, String rtnType, String name,
         this.isAutoReturnEnabled = isAutoReturnEnabled;
     }
 
+    // TODO: do this in class on add to remove need for mutable state
     void generateSignature(PainlessLookup painlessLookup) {
         returnType = painlessLookup.canonicalTypeNameToType(rtnTypeStr);
 
@@ -121,7 +118,7 @@ void generateSignature(PainlessLookup painlessLookup) {
                 PainlessLookupUtility.typeToJavaType(returnType), paramClasses).toMethodDescriptorString());
     }
 
-    void analyze(ScriptRoot scriptRoot) {
+    FunctionNode writeFunction(ClassNode classNode, ScriptRoot scriptRoot) {
         FunctionScope functionScope = newFunctionScope(returnType);
 
         for (int index = 0; index < typeParameters.size(); ++index) {
@@ -130,7 +127,7 @@ void analyze(ScriptRoot scriptRoot) {
             functionScope.defineVariable(location, typeParameter, parameterName, false);
         }
 
-        maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
+        int maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
 
         if (block.statements.isEmpty()) {
             throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
@@ -138,8 +135,8 @@ void analyze(ScriptRoot scriptRoot) {
 
         Input blockInput = new Input();
         blockInput.lastSource = true;
-        Output blockOutput = block.analyze(scriptRoot, functionScope.newLocalScope(), blockInput);
-        methodEscape = blockOutput.methodEscape;
+        Output blockOutput = block.analyze(classNode, scriptRoot, functionScope.newLocalScope(), blockInput);
+        boolean methodEscape = blockOutput.methodEscape;
 
         if (methodEscape == false && isAutoReturnEnabled == false && returnType != void.class) {
             throw createError(new IllegalArgumentException("not all paths provide a return value " +
@@ -152,11 +149,8 @@ void analyze(ScriptRoot scriptRoot) {
             scriptRoot.setUsedVariables(functionScope.getUsedVariables());
         }
         // TODO: end
-    }
 
-    @Override
-    public FunctionNode write(ClassNode classNode) {
-        BlockNode blockNode = block.write(classNode);
+        BlockNode blockNode = (BlockNode)blockOutput.statementNode;
 
         if (methodEscape == false) {
             ExpressionNode expressionNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index a7d738670ec6f..44b6e0d6105c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.IfNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -30,10 +31,10 @@
 /**
  * Represents an if block.
  */
-public final class SIf extends AStatement {
+public class SIf extends AStatement {
 
-    AExpression condition;
-    final SBlock ifblock;
+    protected final AExpression condition;
+    protected final SBlock ifblock;
 
     public SIf(Location location, AExpression condition, SBlock ifblock) {
         super(location);
@@ -43,14 +44,13 @@ public SIf(Location location, AExpression condition, SBlock ifblock) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
 
         if (condition instanceof EBoolean) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
@@ -65,25 +65,20 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         ifblockInput.inLoop = input.inLoop;
         ifblockInput.lastLoop = input.lastLoop;
 
-        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
+        Output ifblockOutput = ifblock.analyze(classNode, scriptRoot, scope.newLocalScope(), ifblockInput);
 
         output.anyContinue = ifblockOutput.anyContinue;
         output.anyBreak = ifblockOutput.anyBreak;
         output.statementCount = ifblockOutput.statementCount;
 
-        return output;
-    }
-
-    @Override
-    IfNode write(ClassNode classNode) {
         IfNode ifNode = new IfNode();
-
-        ifNode.setConditionNode(condition.cast(condition.write(classNode)));
-        ifNode.setBlockNode(ifblock.write(classNode));
-
+        ifNode.setConditionNode(condition.cast(conditionOutput));
+        ifNode.setBlockNode((BlockNode)ifblockOutput.statementNode);
         ifNode.setLocation(location);
 
-        return ifNode;
+        output.statementNode = ifNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index f433db860ccfb..6ae366ad8428a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.IfElseNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -33,11 +34,11 @@
 /**
  * Represents an if/else block.
  */
-public final class SIfElse extends AStatement {
+public class SIfElse extends AStatement {
 
-    private AExpression condition;
-    private final SBlock ifblock;
-    private final SBlock elseblock;
+    protected final AExpression condition;
+    protected final SBlock ifblock;
+    protected final SBlock elseblock;
 
     public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock elseblock) {
         super(location);
@@ -48,14 +49,13 @@ public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
 
         if (condition instanceof EBoolean) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
@@ -70,7 +70,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         ifblockInput.inLoop = input.inLoop;
         ifblockInput.lastLoop = input.lastLoop;
 
-        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
+        Output ifblockOutput = ifblock.analyze(classNode, scriptRoot, scope.newLocalScope(), ifblockInput);
 
         output.anyContinue = ifblockOutput.anyContinue;
         output.anyBreak = ifblockOutput.anyBreak;
@@ -85,7 +85,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         elseblockInput.inLoop = input.inLoop;
         elseblockInput.lastLoop = input.lastLoop;
 
-        Output elseblockOutput = elseblock.analyze(scriptRoot, scope.newLocalScope(), elseblockInput);
+        Output elseblockOutput = elseblock.analyze(classNode, scriptRoot, scope.newLocalScope(), elseblockInput);
 
         output.methodEscape = ifblockOutput.methodEscape && elseblockOutput.methodEscape;
         output.loopEscape = ifblockOutput.loopEscape && elseblockOutput.loopEscape;
@@ -94,20 +94,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.anyBreak |= elseblockOutput.anyBreak;
         output.statementCount = Math.max(ifblockOutput.statementCount, elseblockOutput.statementCount);
 
-        return output;
-    }
-
-    @Override
-    IfElseNode write(ClassNode classNode) {
         IfElseNode ifElseNode = new IfElseNode();
-
-        ifElseNode.setConditionNode(condition.cast(condition.write(classNode)));
-        ifElseNode.setBlockNode(ifblock.write(classNode));
-        ifElseNode.setElseBlockNode(elseblock.write(classNode));
-
+        ifElseNode.setConditionNode(condition.cast(conditionOutput));
+        ifElseNode.setBlockNode((BlockNode)ifblockOutput.statementNode);
+        ifElseNode.setElseBlockNode((BlockNode)elseblockOutput.statementNode);
         ifElseNode.setLocation(location);
 
-        return ifElseNode;
+        output.statementNode = ifElseNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 23a5d4183edbc..6ab6c6590e350 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -29,9 +29,9 @@
 /**
  * Represents a return statement.
  */
-public final class SReturn extends AStatement {
+public class SReturn extends AStatement {
 
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SReturn(Location location, AExpression expression) {
         super(location);
@@ -40,9 +40,10 @@ public SReturn(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        AExpression.Output expressionOutput = null;
 
         if (expression == null) {
             if (scope.getReturnType() != void.class) {
@@ -54,8 +55,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             AExpression.Input expressionInput = new AExpression.Input();
             expressionInput.expected = scope.getReturnType();
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
         }
 
         output.methodEscape = true;
@@ -64,18 +65,13 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ReturnNode write(ClassNode classNode) {
         ReturnNode returnNode = new ReturnNode();
-
-        returnNode.setExpressionNode(expression == null ? null : expression.cast(expression.write(classNode)));
-
+        returnNode.setExpressionNode(expression == null ? null : expression.cast(expressionOutput));
         returnNode.setLocation(location);
 
-        return returnNode;
+        output.statementNode = returnNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index 5e1846457d515..fef81dfed6968 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.Scope.Variable;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ForEachSubArrayNode;
 import org.elasticsearch.painless.lookup.PainlessCast;
@@ -33,46 +34,34 @@
 /**
  * Represents a for-each loop for arrays.
  */
-final class SSubEachArray extends AStatement {
-    private final Variable variable;
-    private AExpression expression;
-    private final SBlock block;
+public class SSubEachArray extends AStatement {
 
-    private PainlessCast cast = null;
-    private Variable array = null;
-    private Variable index = null;
-    private Class<?> indexed = null;
+    protected final Variable variable;
+    protected final AExpression.Output expressionOutput;
+    protected final Output blockOutput;
 
-    SSubEachArray(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachArray(Location location, Variable variable, AExpression.Output expressionOutput, Output blockOutput) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
-        this.expression = Objects.requireNonNull(expression);
-        this.block = block;
+        this.expressionOutput = Objects.requireNonNull(expressionOutput);
+        this.blockOutput = blockOutput;
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
-        array = scope.defineInternalVariable(location, expression.output.actual, "array" + location.getOffset(), true);
-        index = scope.defineInternalVariable(location, int.class, "index" + location.getOffset(), true);
-        indexed = expression.output.actual.getComponentType();
-        cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
+        Variable array = scope.defineVariable(location, expressionOutput.actual, "#array" + location.getOffset(), true);
+        Variable index = scope.defineVariable(location, int.class, "#index" + location.getOffset(), true);
+        Class<?> indexed = expressionOutput.actual.getComponentType();
+        PainlessCast cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
 
-        return output;
-    }
-
-    @Override
-    ForEachSubArrayNode write(ClassNode classNode) {
         ForEachSubArrayNode forEachSubArrayNode = new ForEachSubArrayNode();
-
-        forEachSubArrayNode.setConditionNode(expression.write(classNode));
-        forEachSubArrayNode.setBlockNode(block.write(classNode));
-
+        forEachSubArrayNode.setConditionNode(expressionOutput.expressionNode);
+        forEachSubArrayNode.setBlockNode((BlockNode)blockOutput.statementNode);
         forEachSubArrayNode.setLocation(location);
         forEachSubArrayNode.setVariableType(variable.getType());
         forEachSubArrayNode.setVariableName(variable.getName());
@@ -84,11 +73,14 @@ ForEachSubArrayNode write(ClassNode classNode) {
         forEachSubArrayNode.setIndexedType(indexed);
         forEachSubArrayNode.setContinuous(false);
 
-        return forEachSubArrayNode;
+        output.statementNode = forEachSubArrayNode;
+
+        return output;
     }
 
     @Override
     public String toString() {
-        return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        //return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        return null;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 4198452ff8e79..83561431db1b1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.Scope.Variable;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ForEachSubIterableNode;
 import org.elasticsearch.painless.lookup.PainlessCast;
@@ -38,56 +39,46 @@
 /**
  * Represents a for-each loop for iterables.
  */
-final class SSubEachIterable extends AStatement {
+public class SSubEachIterable extends AStatement {
 
-    private AExpression expression;
-    private final SBlock block;
-    private final Variable variable;
+    protected final Variable variable;
+    protected final AExpression.Output expressionOutput;
+    protected final Output blockOutput;
 
-    private PainlessCast cast = null;
-    private Variable iterator = null;
-    private PainlessMethod method = null;
-
-    SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachIterable(Location location, Variable variable, AExpression.Output expressionOutput, Output blockOutput) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
-        this.expression = Objects.requireNonNull(expression);
-        this.block = block;
+        this.expressionOutput = Objects.requireNonNull(expressionOutput);
+        this.blockOutput = blockOutput;
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
-        iterator = scope.defineInternalVariable(location, Iterator.class, "itr" + location.getOffset(), true);
+        Variable iterator = scope.defineVariable(location, Iterator.class, "#itr" + location.getOffset(), true);
+
+        PainlessMethod method;
 
-        if (expression.output.actual == def.class) {
+        if (expressionOutput.actual == def.class) {
             method = null;
         } else {
-            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expression.output.actual, false, "iterator", 0);
+            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expressionOutput.actual, false, "iterator", 0);
 
             if (method == null) {
                     throw createError(new IllegalArgumentException(
-                            "method [" + typeToCanonicalTypeName(expression.output.actual) + ", iterator/0] not found"));
+                            "method [" + typeToCanonicalTypeName(expressionOutput.actual) + ", iterator/0] not found"));
             }
         }
 
-        cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
-
-        return output;
-    }
+        PainlessCast cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
 
-    @Override
-    ForEachSubIterableNode write(ClassNode classNode) {
         ForEachSubIterableNode forEachSubIterableNode = new ForEachSubIterableNode();
-
-        forEachSubIterableNode.setConditionNode(expression.write(classNode));
-        forEachSubIterableNode.setBlockNode(block.write(classNode));
-
+        forEachSubIterableNode.setConditionNode(expressionOutput.expressionNode);
+        forEachSubIterableNode.setBlockNode((BlockNode)blockOutput.statementNode);
         forEachSubIterableNode.setLocation(location);
         forEachSubIterableNode.setVariableType(variable.getType());
         forEachSubIterableNode.setVariableName(variable.getName());
@@ -97,11 +88,14 @@ ForEachSubIterableNode write(ClassNode classNode) {
         forEachSubIterableNode.setMethod(method);
         forEachSubIterableNode.setContinuous(false);
 
-        return forEachSubIterableNode;
+        output.statementNode = forEachSubIterableNode;
+
+        return output;
     }
 
     @Override
     public String toString() {
-        return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        //return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        return null;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 86b40fafa378b..3fff9615b12d9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -30,9 +30,9 @@
 /**
  * Represents a throw statement.
  */
-public final class SThrow extends AStatement {
+public class SThrow extends AStatement {
 
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SThrow(Location location, AExpression expression) {
         super(location);
@@ -41,32 +41,26 @@ public SThrow(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         AExpression.Input expressionInput = new AExpression.Input();
         expressionInput.expected = Exception.class;
-        expression.analyze(scriptRoot, scope, expressionInput);
-        expression.cast();
+        AExpression.Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+        expression.cast(expressionInput, expressionOutput);
 
         output.methodEscape = true;
         output.loopEscape = true;
         output.allEscape = true;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ThrowNode write(ClassNode classNode) {
         ThrowNode throwNode = new ThrowNode();
-
-        throwNode.setExpressionNode(expression.cast(expression.write(classNode)));
-
+        throwNode.setExpressionNode(expression.cast(expressionOutput));
         throwNode.setLocation(location);
 
-        return throwNode;
+        output.statementNode = throwNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 7798fd13b13a2..b9fd59f6b3fc1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -21,10 +21,13 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
+import org.elasticsearch.painless.ir.CatchNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.TryNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -33,10 +36,10 @@
 /**
  * Represents the try block as part of a try-catch block.
  */
-public final class STry extends AStatement {
+public class STry extends AStatement {
 
-    private final SBlock block;
-    private final List<SCatch> catches;
+    protected final SBlock block;
+    protected final List<SCatch> catches;
 
     public STry(Location location, SBlock block, List<SCatch> catches) {
         super(location);
@@ -46,9 +49,8 @@ public STry(Location location, SBlock block, List<SCatch> catches) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
@@ -59,7 +61,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         blockInput.inLoop = input.inLoop;
         blockInput.lastLoop = input.lastLoop;
 
-        Output blockOutput = block.analyze(scriptRoot, scope.newLocalScope(), blockInput);
+        Output blockOutput = block.analyze(classNode, scriptRoot, scope.newLocalScope(), blockInput);
 
         output.methodEscape = blockOutput.methodEscape;
         output.loopEscape = blockOutput.loopEscape;
@@ -69,13 +71,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         int statementCount = 0;
 
+        List<Output> catchOutputs = new ArrayList<>();
+
         for (SCatch catc : catches) {
             Input catchInput = new Input();
             catchInput.lastSource = input.lastSource;
             catchInput.inLoop = input.inLoop;
             catchInput.lastLoop = input.lastLoop;
 
-            Output catchOutput = catc.analyze(scriptRoot, scope.newLocalScope(), catchInput);
+            Output catchOutput = catc.analyze(classNode, scriptRoot, scope.newLocalScope(), catchInput);
 
             output.methodEscape &= catchOutput.methodEscape;
             output.loopEscape &= catchOutput.loopEscape;
@@ -83,27 +87,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.anyContinue |= catchOutput.anyContinue;
             output.anyBreak |= catchOutput.anyBreak;
 
+            catchOutputs.add(catchOutput);
+
             statementCount = Math.max(statementCount, catchOutput.statementCount);
         }
 
         output.statementCount = blockOutput.statementCount + statementCount;
 
-        return output;
-    }
-
-    @Override
-    TryNode write(ClassNode classNode) {
         TryNode tryNode = new TryNode();
 
-        for (SCatch catc : catches) {
-            tryNode.addCatchNode(catc.write(classNode));
+        for (Output catchOutput : catchOutputs) {
+            tryNode.addCatchNode((CatchNode)catchOutput.statementNode);
         }
 
-        tryNode.setBlockNode(block.write(classNode));
-
+        tryNode.setBlockNode((BlockNode)blockOutput.statementNode);
         tryNode.setLocation(location);
 
-        return tryNode;
+        output.statementNode = tryNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index eead51786e2d3..9fa3139b87eed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.WhileNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -30,12 +31,10 @@
 /**
  * Represents a while loop.
  */
-public final class SWhile extends AStatement {
+public class SWhile extends AStatement {
 
-    private AExpression condition;
-    private final SBlock block;
-
-    private boolean continuous = false;
+    protected final AExpression condition;
+    protected final SBlock block;
 
     public SWhile(Location location, AExpression condition, SBlock block) {
         super(location);
@@ -45,16 +44,16 @@ public SWhile(Location location, AExpression condition, SBlock block) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
         scope = scope.newLocalScope();
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
+
+        boolean continuous = false;
 
         if (condition instanceof EBoolean) {
             continuous = ((EBoolean)condition).constant;
@@ -68,12 +67,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
         }
 
+        Output blockOutput = null;
+
         if (block != null) {
             Input blockInput = new Input();
             blockInput.beginLoop = true;
             blockInput.inLoop = true;
 
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
             if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
@@ -89,20 +90,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    WhileNode write(ClassNode classNode) {
         WhileNode whileNode = new WhileNode();
-
-        whileNode.setConditionNode(condition.cast(condition.write(classNode)));
-        whileNode.setBlockNode(block == null ? null : block.write(classNode));
-
+        whileNode.setConditionNode(condition.cast(conditionOutput));
+        whileNode.setBlockNode(blockOutput == null ? null : (BlockNode)blockOutput.statementNode);
         whileNode.setLocation(location);
         whileNode.setContinuous(continuous);
 
-        return whileNode;
+        output.statementNode = whileNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 0a7c0cdcd0d7e..0cee95adef5fd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 public class DefOptimizationTests extends ScriptTestCase {
+
     public void testIntBraceArrayOptiLoad() {
         final String script = "int x = 0; def y = new int[1]; y[0] = 5; x = y[0]; return x;";
         assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I");
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index bdc683125eb68..70e754a4168f5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.painless.FeatureTestObject;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Scope.Variable;
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.action.PainlessExecuteAction;
 import org.elasticsearch.painless.antlr.Walker;
@@ -749,30 +748,6 @@ public void testSIfElse() {
               + "}");
     }
 
-    public void testSSubEachArray() {
-        Location l = new Location(getTestName(), 0);
-        Variable v = new Variable(int.class, "test", false);
-        AExpression e = new ENewArray(l, "int", Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)), true);
-        SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5))));
-        SSubEachArray node = new SSubEachArray(l, v, e, b);
-        assertEquals(
-                "(SSubEachArray int test (ENewArray int init (Args (EConstant Integer 1) (EConstant Integer 2) (EConstant Integer 3))) "
-              + "(SBlock (SReturn (EConstant Integer 5))))",
-                node.toString());
-    }
-
-    public void testSSubEachIterable() {
-        Location l = new Location(getTestName(), 0);
-        Variable v = new Variable(int.class, "test", false);
-        AExpression e = new EListInit(l, Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)));
-        SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5))));
-        SSubEachIterable node = new SSubEachIterable(l, v, e, b);
-        assertEquals(
-                  "(SSubEachIterable int test (EListInit (EConstant Integer 1) (EConstant Integer 2) (EConstant Integer 3)) (SBlock "
-                + "(SReturn (EConstant Integer 5))))",
-                  node.toString());
-    }
-
     public void testSThrow() {
         assertToString("(SClass (SThrow (ENewObj RuntimeException)))", "throw new RuntimeException()");
     }

From 0bce5d2602bbace21971e1214675ebf10ec1c966 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Mar 2020 19:21:58 -0400
Subject: [PATCH 249/285] Reenable BWC tests after backport of #53793 (#54018)

* Reenable BWC tests after backport of #53793
---
 build.gradle                                  |  4 +-
 .../common/io/stream/DelayableWriteable.java  |  7 +--
 .../io/stream/DelayableWriteableTests.java    | 45 ++++++++++++++-----
 3 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/build.gradle b/build.gradle
index 4ae244c9ecf8f..a23614bafa975 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53793" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
index e75b81fff4ef4..b27010017e5ee 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.common.bytes.BytesReference;
-
 import java.io.IOException;
 import java.util.function.Supplier;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+
 /**
  * A holder for {@link Writeable}s that can delays reading the underlying
  * {@linkplain Writeable} when it is read from a remote node.
@@ -60,6 +60,7 @@ private static class Referencing<T extends Writeable> extends DelayableWriteable
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             try (BytesStreamOutput buffer = new BytesStreamOutput()) {
+                buffer.setVersion(out.getVersion());
                 reference.writeTo(buffer);
                 out.writeBytesReference(buffer.bytes());
             }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
index fbd8702608ead..d12a15c1a7207 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
@@ -19,25 +19,25 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
 
 import java.io.IOException;
 
-import static java.util.Collections.singletonList;
-import static org.hamcrest.Matchers.equalTo;
+import org.elasticsearch.Version;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 
 public class DelayableWriteableTests extends ESTestCase {
     // NOTE: we don't use AbstractWireSerializingTestCase because we don't implement equals and hashCode.
-    public static class Example implements NamedWriteable {
+    private static class Example implements NamedWriteable {
         private final String s;
 
-        public Example(String s) {
+        Example(String s) {
             this.s = s;
         }
 
-        public Example(StreamInput in) throws IOException {
+        Example(StreamInput in) throws IOException {
             s = in.readString();
         }
 
@@ -66,14 +66,14 @@ public int hashCode() {
         }
     }
 
-    public static class NamedHolder implements Writeable {
+    private static class NamedHolder implements Writeable {
         private final Example e;
 
-        public NamedHolder(Example e) {
+        NamedHolder(Example e) {
             this.e = e;
         }
 
-        public NamedHolder(StreamInput in) throws IOException {
+        NamedHolder(StreamInput in) throws IOException {
             e = in.readNamedWriteable(Example.class);
         }
 
@@ -97,6 +97,23 @@ public int hashCode() {
         }
     }
 
+    private static class SneakOtherSideVersionOnWire implements Writeable {
+        private final Version version;
+
+        SneakOtherSideVersionOnWire() {
+            version = Version.CURRENT;
+        }
+
+        SneakOtherSideVersionOnWire(StreamInput in) throws IOException {
+            version = Version.readVersion(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            Version.writeVersion(out.getVersion(), out);
+        }
+    }
+
     public void testRoundTripFromReferencing() throws IOException {
         Example e = new Example(randomAlphaOfLength(5));
         DelayableWriteable<Example> original = DelayableWriteable.referencing(e);
@@ -139,6 +156,12 @@ public void testRoundTripFromDelayedFromOldVersionWithNamedWriteable() throws IO
         roundTripTestCase(original, NamedHolder::new);
     }
 
+    public void testSerializesWithRemoteVersion() throws IOException {
+        Version remoteVersion = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+        DelayableWriteable<SneakOtherSideVersionOnWire> original = DelayableWriteable.referencing(new SneakOtherSideVersionOnWire());
+        assertThat(roundTrip(original, SneakOtherSideVersionOnWire::new, remoteVersion).get().version, equalTo(remoteVersion));
+    }
+
     private <T extends Writeable> void roundTripTestCase(DelayableWriteable<T> original, Writeable.Reader<T> reader) throws IOException {
         DelayableWriteable<T> roundTripped = roundTrip(original, reader, Version.CURRENT);
         assertTrue(roundTripped.isDelayed());

From d0cd387ac5de1bd5ed316ae4920924054308d594 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Mar 2020 19:24:34 -0400
Subject: [PATCH 250/285] Remove ceremony declaring aggs (#53990)

This removes some more ceremony when declaring agg parsers. You no
longer need a static `parse` method, instead you can just make the
`PARSER` public in most cases.

There are still a few aggs with the `parse` method, but those `parse`
methods are a little more complex to untangle.
---
 .../elasticsearch/search/SearchModule.java    | 20 +++++------
 .../CompositeValuesSourceParserHelper.java    |  2 +-
 .../DateHistogramValuesSourceBuilder.java     | 17 ++++-----
 .../geogrid/GeoGridAggregationBuilder.java    | 14 ++++----
 .../GeoHashGridAggregationBuilder.java        | 14 +++-----
 .../GeoTileGridAggregationBuilder.java        | 14 +++-----
 .../AutoDateHistogramAggregationBuilder.java  | 25 ++++++-------
 .../DateHistogramAggregationBuilder.java      | 36 ++++++++-----------
 .../bucket/histogram/DateIntervalWrapper.java |  2 +-
 .../HistogramAggregationBuilder.java          | 19 ++++------
 .../range/DateRangeAggregationBuilder.java    |  7 ++--
 .../range/IpRangeAggregationBuilder.java      | 26 ++++++--------
 .../bucket/range/RangeAggregationBuilder.java |  8 ++---
 .../metrics/GeoBoundsAggregationBuilder.java  | 19 ++++------
 .../GeoCentroidAggregationBuilder.java        | 15 +++-----
 .../aggregations/bucket/DateRangeTests.java   |  2 +-
 .../aggregations/bucket/RangeTests.java       |  4 +--
 .../geogrid/GeoHashGridParserTests.java       | 26 +++++++-------
 .../geogrid/GeoTileGridParserTests.java       | 16 ++++-----
 19 files changed, 119 insertions(+), 167 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 3ecf9e2371682..dea00a9b3012d 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -396,23 +396,23 @@ private void registerAggregations(List<SearchPlugin> plugins) {
         registerAggregation(new AggregationSpec(SignificantTextAggregationBuilder.NAME, SignificantTextAggregationBuilder::new,
                 SignificantTextAggregationBuilder::parse));
         registerAggregation(new AggregationSpec(RangeAggregationBuilder.NAME, RangeAggregationBuilder::new,
-                RangeAggregationBuilder::parse).addResultReader(InternalRange::new));
+                RangeAggregationBuilder.PARSER).addResultReader(InternalRange::new));
         registerAggregation(new AggregationSpec(DateRangeAggregationBuilder.NAME, DateRangeAggregationBuilder::new,
-                DateRangeAggregationBuilder::parse).addResultReader(InternalDateRange::new));
+                DateRangeAggregationBuilder.PARSER).addResultReader(InternalDateRange::new));
         registerAggregation(new AggregationSpec(IpRangeAggregationBuilder.NAME, IpRangeAggregationBuilder::new,
-                IpRangeAggregationBuilder::parse).addResultReader(InternalBinaryRange::new));
+                IpRangeAggregationBuilder.PARSER).addResultReader(InternalBinaryRange::new));
         registerAggregation(new AggregationSpec(HistogramAggregationBuilder.NAME, HistogramAggregationBuilder::new,
-                HistogramAggregationBuilder::parse).addResultReader(InternalHistogram::new));
+                HistogramAggregationBuilder.PARSER).addResultReader(InternalHistogram::new));
         registerAggregation(new AggregationSpec(DateHistogramAggregationBuilder.NAME, DateHistogramAggregationBuilder::new,
-                DateHistogramAggregationBuilder::parse).addResultReader(InternalDateHistogram::new));
+                DateHistogramAggregationBuilder.PARSER).addResultReader(InternalDateHistogram::new));
         registerAggregation(new AggregationSpec(AutoDateHistogramAggregationBuilder.NAME, AutoDateHistogramAggregationBuilder::new,
-                AutoDateHistogramAggregationBuilder::parse).addResultReader(InternalAutoDateHistogram::new));
+                AutoDateHistogramAggregationBuilder.PARSER).addResultReader(InternalAutoDateHistogram::new));
         registerAggregation(new AggregationSpec(GeoDistanceAggregationBuilder.NAME, GeoDistanceAggregationBuilder::new,
                 GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new));
         registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new,
-                GeoHashGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new));
+                GeoHashGridAggregationBuilder.PARSER).addResultReader(InternalGeoHashGrid::new));
         registerAggregation(new AggregationSpec(GeoTileGridAggregationBuilder.NAME, GeoTileGridAggregationBuilder::new,
-                GeoTileGridAggregationBuilder::parse).addResultReader(InternalGeoTileGrid::new));
+                GeoTileGridAggregationBuilder.PARSER).addResultReader(InternalGeoTileGrid::new));
         registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new,
                 NestedAggregationBuilder::parse).addResultReader(InternalNested::new));
         registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new,
@@ -420,9 +420,9 @@ private void registerAggregations(List<SearchPlugin> plugins) {
         registerAggregation(new AggregationSpec(TopHitsAggregationBuilder.NAME, TopHitsAggregationBuilder::new,
                 TopHitsAggregationBuilder::parse).addResultReader(InternalTopHits::new));
         registerAggregation(new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new,
-                GeoBoundsAggregationBuilder::parse).addResultReader(InternalGeoBounds::new));
+                GeoBoundsAggregationBuilder.PARSER).addResultReader(InternalGeoBounds::new));
         registerAggregation(new AggregationSpec(GeoCentroidAggregationBuilder.NAME, GeoCentroidAggregationBuilder::new,
-                GeoCentroidAggregationBuilder::parse).addResultReader(InternalGeoCentroid::new));
+                GeoCentroidAggregationBuilder.PARSER).addResultReader(InternalGeoCentroid::new));
         registerAggregation(new AggregationSpec(ScriptedMetricAggregationBuilder.NAME, ScriptedMetricAggregationBuilder::new,
                 ScriptedMetricAggregationBuilder.PARSER).addResultReader(InternalScriptedMetric::new));
         registerAggregation((new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java
index cf8c951a6c5de..60092f1d353c5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java
@@ -116,7 +116,7 @@ public static CompositeValuesSourceBuilder<?> fromXContent(XContentParser parser
                 builder = TermsValuesSourceBuilder.parse(name, parser);
                 break;
             case DateHistogramValuesSourceBuilder.TYPE:
-                builder = DateHistogramValuesSourceBuilder.parse(name, parser);
+                builder = DateHistogramValuesSourceBuilder.PARSER.parse(parser, name);
                 break;
             case HistogramValuesSourceBuilder.TYPE:
                 builder = HistogramValuesSourceBuilder.parse(name, parser);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
index 2c76872479074..dfac9ee524c0b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
+import java.io.IOException;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.Objects;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Rounding;
@@ -41,11 +46,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
-import java.io.IOException;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.util.Objects;
-
 /**
  * A {@link CompositeValuesSourceBuilder} that builds a {@link RoundingValuesSource} from a {@link Script} or
  * a field name using the provided interval.
@@ -54,9 +54,9 @@ public class DateHistogramValuesSourceBuilder
     extends CompositeValuesSourceBuilder<DateHistogramValuesSourceBuilder> implements DateIntervalConsumer {
     static final String TYPE = "date_histogram";
 
-    private static final ObjectParser<DateHistogramValuesSourceBuilder, Void> PARSER;
+    static final ObjectParser<DateHistogramValuesSourceBuilder, String> PARSER =
+            ObjectParser.fromBuilder(TYPE, DateHistogramValuesSourceBuilder::new);
     static {
-        PARSER = new ObjectParser<>(DateHistogramValuesSourceBuilder.TYPE);
         PARSER.declareString(DateHistogramValuesSourceBuilder::format, new ParseField("format"));
         DateIntervalWrapper.declareIntervalFields(PARSER);
         PARSER.declareField(DateHistogramValuesSourceBuilder::offset, p -> {
@@ -75,9 +75,6 @@ public class DateHistogramValuesSourceBuilder
         }, new ParseField("time_zone"), ObjectParser.ValueType.LONG);
         CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER, ValueType.NUMERIC);
     }
-    static DateHistogramValuesSourceBuilder parse(String name, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new DateHistogramValuesSourceBuilder(name), null);
-    }
 
     private ZoneId timeZone = null;
     private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 0a7c918231c73..71a79818828ed 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
@@ -42,10 +47,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoGridAggregationBuilder>
         implements MultiBucketAggregationBuilder {
     /* recognized field names in JSON */
@@ -64,8 +65,9 @@ protected interface PrecisionParser {
         int parse(XContentParser parser) throws IOException;
     }
 
-    public static ObjectParser<GeoGridAggregationBuilder, Void> createParser(String name, PrecisionParser precisionParser) {
-        ObjectParser<GeoGridAggregationBuilder, Void> parser = new ObjectParser<>(name);
+    public static <T extends GeoGridAggregationBuilder> ObjectParser<T, String> createParser(
+            String name, PrecisionParser precisionParser, Function<String, T> ctor) {
+        ObjectParser<T, String> parser = ObjectParser.fromBuilder(name, ctor);
         ValuesSourceParserHelper.declareGeoFields(parser, false, false);
         parser.declareField((p, builder, context) -> builder.precision(precisionParser.parse(p)), FIELD_PRECISION,
             org.elasticsearch.common.xcontent.ObjectParser.ValueType.INT);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
index acc9cde113164..e685a82b00e86 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -32,15 +34,13 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
-import java.io.IOException;
-import java.util.Map;
-
 public class GeoHashGridAggregationBuilder extends GeoGridAggregationBuilder {
     public static final String NAME = "geohash_grid";
     public static final int DEFAULT_PRECISION = 5;
     public static final int DEFAULT_MAX_NUM_CELLS = 10000;
 
-    private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoUtils::parsePrecision);
+    public static final ObjectParser<GeoHashGridAggregationBuilder, String> PARSER =
+            createParser(NAME, GeoUtils::parsePrecision, GeoHashGridAggregationBuilder::new);
 
     public GeoHashGridAggregationBuilder(String name) {
         super(name);
@@ -79,10 +79,6 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu
         return new GeoHashGridAggregationBuilder(this, factoriesBuilder, metaData);
     }
 
-    public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoHashGridAggregationBuilder(aggregationName), null);
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
index 595c6cab6e718..01a670caf6cfe 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
@@ -19,10 +19,12 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -31,15 +33,13 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
-import java.io.IOException;
-import java.util.Map;
-
 public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder {
     public static final String NAME = "geotile_grid";
     public static final int DEFAULT_PRECISION = 7;
     private static final int DEFAULT_MAX_NUM_CELLS = 10000;
 
-    private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoTileUtils::parsePrecision);
+    public static final ObjectParser<GeoTileGridAggregationBuilder, String> PARSER =
+        createParser(NAME, GeoTileUtils::parsePrecision, GeoTileGridAggregationBuilder::new);
 
     public GeoTileGridAggregationBuilder(String name) {
         super(name);
@@ -77,10 +77,6 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu
         return new GeoTileGridAggregationBuilder(this, factoriesBuilder, metaData);
     }
 
-    public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoTileGridAggregationBuilder(aggregationName), null);
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
index 313d34bbbad17..06cf86d852e11 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -19,6 +19,14 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import static java.util.Map.entry;
+
+import java.io.IOException;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Rounding;
@@ -28,7 +36,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
@@ -43,14 +50,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.time.ZoneId;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
-
-import static java.util.Map.entry;
-
 public class AutoDateHistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, AutoDateHistogramAggregationBuilder> {
 
@@ -59,9 +58,9 @@ public class AutoDateHistogramAggregationBuilder
     private static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
     private static final ParseField MINIMUM_INTERVAL_FIELD = new ParseField("minimum_interval");
 
-    private static final ObjectParser<AutoDateHistogramAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<AutoDateHistogramAggregationBuilder, String> PARSER =
+        ObjectParser.fromBuilder(NAME, AutoDateHistogramAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(AutoDateHistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
         PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD);
         PARSER.declareStringOrNull(AutoDateHistogramAggregationBuilder::setMinimumIntervalExpression, MINIMUM_INTERVAL_FIELD);
@@ -110,10 +109,6 @@ static RoundingInfo[] buildRoundings(ZoneId timeZone, String minimumInterval) {
         return Arrays.copyOfRange(roundings, indexToSliceFrom, roundings.length);
     }
 
-    public static AutoDateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new AutoDateHistogramAggregationBuilder(aggregationName), null);
-    }
-
     private int numBuckets = 10;
 
     private String minimumIntervalExpression;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index c60f9c7f52cef..03aaaa89f3757 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -19,6 +19,17 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import static java.util.Map.entry;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.zone.ZoneOffsetTransition;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
@@ -26,14 +37,12 @@
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
@@ -55,17 +64,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.time.zone.ZoneOffsetTransition;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static java.util.Map.entry;
-
 /**
  * A builder for histograms on date fields.
  */
@@ -73,7 +71,6 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil
         implements MultiBucketAggregationBuilder, DateIntervalConsumer {
 
     public static final String NAME = "date_histogram";
-    private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser();
 
     public static final Map<String, Rounding.DateTimeUnit> DATE_FIELD_UNITS = Map.ofEntries(
             entry("year", Rounding.DateTimeUnit.YEAR_OF_CENTURY),
@@ -93,11 +90,10 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil
             entry("second", Rounding.DateTimeUnit.SECOND_OF_MINUTE),
             entry("1s", Rounding.DateTimeUnit.SECOND_OF_MINUTE));
 
-    private static final ObjectParser<DateHistogramAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<DateHistogramAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, DateHistogramAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(DateHistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareAnyFields(PARSER, true, true, true);
-
         DateIntervalWrapper.declareIntervalFields(PARSER);
 
         PARSER.declareField(DateHistogramAggregationBuilder::offset, p -> {
@@ -119,10 +115,6 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil
                 Histogram.ORDER_FIELD);
     }
 
-    public static DateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new DateHistogramAggregationBuilder(aggregationName), null);
-    }
-
     private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper();
     private long offset = 0;
     private ExtendedBounds extendedBounds;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
index 18e7d8720607e..04874c592c967 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
@@ -86,7 +86,7 @@ public String value() {
     private DateHistogramInterval dateHistogramInterval;
     private IntervalTypeEnum intervalType = IntervalTypeEnum.NONE;
 
-    public static <T extends DateIntervalConsumer> void declareIntervalFields(ObjectParser<T, Void> parser) {
+    public static <T extends DateIntervalConsumer> void declareIntervalFields(ObjectParser<T, String> parser) {
 
         // NOTE: this field is deprecated and will be removed
         parser.declareField((wrapper, interval) -> {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index 11a792ed25d6f..1a5a2cce36c86 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -43,11 +47,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 /**
  * A builder for histograms on numeric fields.  This builder can operate on either base numeric fields, or numeric range fields.  IP range
  * fields are unsupported, and will throw at the factory layer.
@@ -64,9 +63,9 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder<
         EXTENDED_BOUNDS_PARSER.declareDouble((bounds, d) -> bounds[1] = d, new ParseField("max"));
     }
 
-    private static final ObjectParser<HistogramAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<HistogramAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, HistogramAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(HistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareAnyFields(PARSER, true, true);
 
         PARSER.declareDouble(HistogramAggregationBuilder::interval, Histogram.INTERVAL_FIELD);
@@ -85,10 +84,6 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder<
             Histogram.ORDER_FIELD);
     }
 
-    public static HistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new HistogramAggregationBuilder(aggregationName), null);
-    }
-
     private double interval;
     private double offset = 0;
     private double minBound = Double.POSITIVE_INFINITY;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
index fcefc81b69845..92880c2011df0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
@@ -38,9 +38,9 @@
 public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeAggregationBuilder, RangeAggregator.Range> {
     public static final String NAME = "date_range";
 
-    private static final ObjectParser<DateRangeAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<DateRangeAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME,  DateRangeAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(DateRangeAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
         PARSER.declareBoolean(DateRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
 
@@ -51,9 +51,6 @@ public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeA
         }, (p, c) -> DateRangeAggregationBuilder.parseRange(p), RangeAggregator.RANGES_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new DateRangeAggregationBuilder(aggregationName), null);
-    }
 
     private static RangeAggregator.Range parseRange(XContentParser parser) throws IOException {
         return RangeAggregator.Range.fromXContent(parser);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
index e0b3fad96f6bc..eda8c0524e6bf 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
@@ -18,6 +18,15 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
@@ -44,24 +53,15 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 
 public final class IpRangeAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, IpRangeAggregationBuilder> {
     public static final String NAME = "ip_range";
     private static final ParseField MASK_FIELD = new ParseField("mask");
 
-    private static final ObjectParser<IpRangeAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<IpRangeAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, IpRangeAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(IpRangeAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareBytesFields(PARSER, false, false);
 
         PARSER.declareBoolean(IpRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
@@ -71,10 +71,6 @@ public final class IpRangeAggregationBuilder
         }, (p, c) -> IpRangeAggregationBuilder.parseRange(p), RangeAggregator.RANGES_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new IpRangeAggregationBuilder(aggregationName), null);
-    }
-
     private static Range parseRange(XContentParser parser) throws IOException {
         String key = null;
         String from = null;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index 6a95447287799..7744cae764c57 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -38,9 +38,9 @@
 public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregationBuilder, Range> {
     public static final String NAME = "range";
 
-    private static final ObjectParser<RangeAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<RangeAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, RangeAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(RangeAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
         PARSER.declareBoolean(RangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
 
@@ -51,10 +51,6 @@ public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregati
         }, (p, c) -> RangeAggregationBuilder.parseRange(p), RangeAggregator.RANGES_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new RangeAggregationBuilder(aggregationName), null);
-    }
-
     private static Range parseRange(XContentParser parser) throws IOException {
         return Range.fromXContent(parser);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
index 8b9f89fd54209..80658243a0794 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
@@ -19,40 +19,35 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
-import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
 
 public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoBoundsAggregationBuilder> {
     public static final String NAME = "geo_bounds";
 
-    private static final ObjectParser<GeoBoundsAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<GeoBoundsAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, GeoBoundsAggregationBuilder::new); 
     static {
-        PARSER = new ObjectParser<>(GeoBoundsAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareGeoFields(PARSER, false, false);
         PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoBoundsAggregationBuilder(aggregationName), null);
-    }
-
     private boolean wrapLongitude = true;
 
     public GeoBoundsAggregationBuilder(String name) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
index daf357186f36c..afa0329433a6e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
@@ -35,23 +37,16 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.util.Map;
-
 public class GeoCentroidAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregationBuilder> {
     public static final String NAME = "geo_centroid";
 
-    private static final ObjectParser<GeoCentroidAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<GeoCentroidAggregationBuilder, String> PARSER =
+        ObjectParser.fromBuilder(NAME, GeoCentroidAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(GeoCentroidAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareGeoFields(PARSER, true, false);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoCentroidAggregationBuilder(aggregationName), null);
-    }
-
     public GeoCentroidAggregationBuilder(String name) {
         super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
index 10f89af509234..882ce66a24ba3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
@@ -80,7 +80,7 @@ public void testParsingRangeStrict() throws IOException {
             "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, rangeAggregation);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> DateRangeAggregationBuilder.parse("aggregationName", parser));
+                () -> DateRangeAggregationBuilder.PARSER.parse(parser, "aggregationName"));
         assertThat(ex.getCause(), notNullValue());
         assertThat(ex.getCause().getMessage(), containsString("badField"));
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
index 6acca86bffa14..b3afa9b25a92b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
@@ -76,7 +76,7 @@ public void testParsingRangeStrict() throws IOException {
             "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, rangeAggregation);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> RangeAggregationBuilder.parse("aggregationName", parser));
+                () -> RangeAggregationBuilder.PARSER.parse(parser, "aggregationName"));
         assertThat(ex.getCause(), notNullValue());
         assertThat(ex.getCause().getMessage(), containsString("badField"));
     }
@@ -92,7 +92,7 @@ public void testParsingNull() throws IOException {
                 "]\n" +
             "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, rangeAggregation);
-        RangeAggregationBuilder aggregationBuilder = (RangeAggregationBuilder) RangeAggregationBuilder.parse("aggregationName", parser);
+        RangeAggregationBuilder aggregationBuilder = RangeAggregationBuilder.PARSER.parse(parser, "aggregationName");
         assertEquals(1, aggregationBuilder.ranges().size());
         assertEquals(Double.NEGATIVE_INFINITY, aggregationBuilder.ranges().get(0).getFrom(), 0.0);
         assertEquals(Double.POSITIVE_INFINITY, aggregationBuilder.ranges().get(0).getTo(), 0.0);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
index 5075ed9f02aa2..71f962a205233 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -26,11 +31,6 @@
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
 public class GeoHashGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(1, 12);
@@ -39,7 +39,7 @@ public void testParseValidFromInts() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+        assertNotNull(GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
     }
 
     public void testParseValidFromStrings() throws Exception {
@@ -49,7 +49,7 @@ public void testParseValidFromStrings() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+        assertNotNull(GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
     }
 
     public void testParseDistanceUnitPrecision() throws Exception {
@@ -64,7 +64,7 @@ public void testParseDistanceUnitPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        GeoGridAggregationBuilder builder = GeoHashGridAggregationBuilder.parse("geohash_grid", stParser);
+        GeoGridAggregationBuilder builder = GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid");
         assertNotNull(builder);
         assertThat(builder.precision(), greaterThanOrEqualTo(0));
         assertThat(builder.precision(), lessThanOrEqualTo(12));
@@ -76,7 +76,7 @@ public void testParseInvalidUnitPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+                () -> GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
         assertThat(ex.getMessage(), containsString("[geohash_grid] failed to parse field [precision]"));
         assertThat(ex.getCause(), instanceOf(NumberFormatException.class));
         assertEquals("For input string: \"10kg\"", ex.getCause().getMessage());
@@ -88,7 +88,7 @@ public void testParseDistanceUnitPrecisionTooSmall() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+                () -> GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
         assertThat(ex.getMessage(), containsString("[geohash_grid] failed to parse field [precision]"));
         assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
         assertEquals("precision too high [1cm]", ex.getCause().getMessage());
@@ -99,7 +99,7 @@ public void testParseErrorOnBooleanPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException e = expectThrows(XContentParseException.class,
-                () -> GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+                () -> GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
         assertThat(e.getMessage(), containsString("[geohash_grid] precision doesn't support values of type: VALUE_BOOLEAN"));
     }
 
@@ -108,7 +108,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
-            GeoHashGridAggregationBuilder.parse("geohash_grid", stParser);
+            GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid");
             fail();
         } catch (XContentParseException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
@@ -128,6 +128,6 @@ public void testParseValidBounds() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+        assertNotNull(GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
index aa5253564fb49..a3a31ff85e077 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -26,9 +29,6 @@
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-
 public class GeoTileGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM);
@@ -37,7 +37,7 @@ public void testParseValidFromInts() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+        assertNotNull(GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
     }
 
     public void testParseValidFromStrings() throws Exception {
@@ -47,7 +47,7 @@ public void testParseValidFromStrings() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+        assertNotNull(GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
     }
 
     public void testParseErrorOnBooleanPrecision() throws Exception {
@@ -55,7 +55,7 @@ public void testParseErrorOnBooleanPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException e = expectThrows(XContentParseException.class,
-                () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+                () -> GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
         assertThat(ExceptionsHelper.stackTrace(e),
                 containsString("[geotile_grid] precision doesn't support values of type: VALUE_BOOLEAN"));
     }
@@ -65,7 +65,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
-            GeoTileGridAggregationBuilder.parse("geotile_grid", stParser);
+            GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid");
             fail();
         } catch (XContentParseException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
@@ -85,6 +85,6 @@ public void testParseValidBounds() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+        assertNotNull(GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
     }
 }

From 1ec31274c6acb7904d217380d22a415eadfb09e5 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Mar 2020 20:42:08 -0400
Subject: [PATCH 251/285] Prepare to backport #53873 (#54040)

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index a23614bafa975..9593b53b442aa 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/54009" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 50e81360c8a2a59da227be2f51d1001e85bb40b2 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 24 Mar 2020 02:04:48 +0100
Subject: [PATCH 252/285] adapt version after backport (#54009)

---
 .../java/org/elasticsearch/action/search/SearchRequest.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index eae980cd90f67..1c01da64d45f1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -201,7 +201,7 @@ public SearchRequest(StreamInput in) throws IOException {
         requestCache = in.readOptionalBoolean();
         batchedReduceSize = in.readVInt();
         maxConcurrentShardRequests = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
             preFilterShardSize = in.readOptionalVInt();
         } else {
             preFilterShardSize = in.readVInt();
@@ -235,7 +235,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalBoolean(requestCache);
         out.writeVInt(batchedReduceSize);
         out.writeVInt(maxConcurrentShardRequests);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
             out.writeOptionalVInt(preFilterShardSize);
         } else {
             out.writeVInt(preFilterShardSize == null ? DEFAULT_BATCHED_REDUCE_SIZE : preFilterShardSize);

From 408311dc0f33e323d7e67ecfc4c5348be9c025ab Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 23 Mar 2020 18:39:56 -0700
Subject: [PATCH 253/285] add tests to StatsAggregatorTests (#53768)

Adds tests for supported ValuesSourceTypes, unmapped fields, scripting,
and the missing param. The tests for unmapped fields and scripting are
migrated from the StatsIT integration test
---
 .../metrics/StatsAggregatorTests.java         | 364 ++++++++++++++++--
 .../search/aggregations/metrics/StatsIT.java  | 259 -------------
 .../search/aggregations/metrics/SumIT.java    |  36 --
 .../metrics/AbstractNumericTestCase.java      |  30 +-
 4 files changed, 341 insertions(+), 348 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
index 28b1514545506..ccd01c94de2f3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
@@ -22,30 +22,62 @@
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
 
 public class StatsAggregatorTests extends AggregatorTestCase {
-    static final double TOLERANCE = 1e-10;
+
+    private static final double TOLERANCE = 1e-10;
+    private static final String VALUE_SCRIPT_NAME = "value_script";
+    private static final String FIELD_SCRIPT_NAME = "field_script";
 
     public void testEmpty() throws IOException {
-        MappedFieldType ft =
-            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
         ft.setName("field");
-        testCase(ft, iw -> {},
+        testCase(
+            stats("_name").field(ft.name()),
+            iw -> {},
             stats -> {
                 assertEquals(0d, stats.getCount(), 0);
                 assertEquals(0d, stats.getSum(), 0);
@@ -53,16 +85,17 @@ public void testEmpty() throws IOException {
                 assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0);
                 assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0);
                 assertFalse(AggregationInspectionHelper.hasValue(stats));
-            }
+            },
+            singleton(ft)
         );
     }
 
     public void testRandomDoubles() throws IOException {
-        MappedFieldType ft =
-            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.DOUBLE);
         ft.setName("field");
         final SimpleStatsAggregator expected = new SimpleStatsAggregator();
-        testCase(ft,
+        testCase(
+            stats("_name").field(ft.name()),
             iw -> {
                 int numDocs = randomIntBetween(10, 50);
                 for (int i = 0; i < numDocs; i++) {
@@ -71,7 +104,7 @@ public void testRandomDoubles() throws IOException {
                     for (int j = 0; j < numValues; j++) {
                         double value = randomDoubleBetween(-100d, 100d, true);
                         long valueAsLong = NumericUtils.doubleToSortableLong(value);
-                        doc.add(new SortedNumericDocValuesField("field", valueAsLong));
+                        doc.add(new SortedNumericDocValuesField(ft.name(), valueAsLong));
                         expected.add(value);
                     }
                     iw.addDocument(doc);
@@ -84,30 +117,16 @@ public void testRandomDoubles() throws IOException {
                 assertEquals(expected.max, stats.getMax(), 0);
                 assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
                 assertTrue(AggregationInspectionHelper.hasValue(stats));
-            }
+            },
+            singleton(ft)
         );
     }
 
     public void testRandomLongs() throws IOException {
-        MappedFieldType ft =
-            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
-        ft.setName("field");
-        final SimpleStatsAggregator expected = new SimpleStatsAggregator();
-        testCase(ft,
-            iw -> {
-                int numDocs = randomIntBetween(10, 50);
-                for (int i = 0; i < numDocs; i++) {
-                    Document doc = new Document();
-                    int numValues = randomIntBetween(1, 5);
-                    for (int j = 0; j < numValues; j++) {
-                        long value = randomIntBetween(-100, 100);
-                        doc.add(new SortedNumericDocValuesField("field", value));
-                        expected.add(value);
-                    }
-                    iw.addDocument(doc);
-                }
-            },
-            stats -> {
+        randomLongsTestCase(
+            randomIntBetween(1, 5),
+            stats("_name").field("field"),
+            (expected, stats) -> {
                 assertEquals(expected.count, stats.getCount(), 0);
                 assertEquals(expected.sum, stats.getSum(), TOLERANCE);
                 assertEquals(expected.min, stats.getMin(), 0);
@@ -151,7 +170,7 @@ public void testSummationAccuracy() throws IOException {
 
     private void verifySummationOfDoubles(double[] values, double expectedSum,
                                           double expectedAvg, double delta) throws IOException {
-        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.DOUBLE);
         ft.setName("field");
 
         double max = Double.NEGATIVE_INFINITY;
@@ -162,10 +181,11 @@ private void verifySummationOfDoubles(double[] values, double expectedSum,
         }
         double expectedMax = max;
         double expectedMin = min;
-        testCase(ft,
+        testCase(
+            stats("_name").field(ft.name()),
             iw -> {
                 for (double value : values) {
-                    iw.addDocument(singleton(new NumericDocValuesField("field", NumericUtils.doubleToSortableLong(value))));
+                    iw.addDocument(singleton(new NumericDocValuesField(ft.name(), NumericUtils.doubleToSortableLong(value))));
                 }
             },
             stats -> {
@@ -174,20 +194,251 @@ private void verifySummationOfDoubles(double[] values, double expectedSum,
                 assertEquals(expectedSum, stats.getSum(), delta);
                 assertEquals(expectedMax, stats.getMax(), 0d);
                 assertEquals(expectedMin, stats.getMin(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            },
+            singleton(ft)
+        );
+    }
+
+    public void testUnmapped() throws IOException {
+        randomLongsTestCase(
+            randomIntBetween(1, 5),
+            stats("_name").field("unmapped_field"),
+            (expected, stats) -> {
+                assertEquals(0d, stats.getCount(), 0);
+                assertEquals(0d, stats.getSum(), 0);
+                assertEquals(Float.NaN, stats.getAvg(), 0);
+                assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0);
+                assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0);
+                assertFalse(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testPartiallyUnmapped() throws IOException {
+        try (Directory mappedDirectory = newDirectory();
+             Directory unmappedDirectory = newDirectory();
+             RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory);
+             RandomIndexWriter unmappedWriter = new RandomIndexWriter(random(), unmappedDirectory)) {
+
+            final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+            ft.setName("field");
+            final SimpleStatsAggregator expected = new SimpleStatsAggregator();
+            final int numDocs = randomIntBetween(10, 50);
+            for (int i = 0; i < numDocs; i++) {
+                final long value = randomLongBetween(-100, 100);
+                mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(ft.name(), value)));
+                expected.add(value);
+            }
+            final StatsAggregationBuilder builder = stats("_name").field(ft.name());
+
+            try (IndexReader mappedReader = mappedWriter.getReader();
+                 IndexReader unmappedReader = unmappedWriter.getReader();
+                 MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) {
+
+                final IndexSearcher searcher = new IndexSearcher(multiReader);
+                final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, ft);
+
+                assertEquals(expected.count, stats.getCount(), 0);
+                assertEquals(expected.sum, stats.getSum(), TOLERANCE);
+                assertEquals(expected.max, stats.getMax(), 0);
+                assertEquals(expected.min, stats.getMin(), 0);
+                assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        }
+    }
+
+    public void testValueScriptSingleValuedField() throws IOException {
+        randomLongsTestCase(
+            1,
+            stats("_name")
+                .field("field")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap())),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testValueScriptMultiValuedField() throws IOException {
+        final int valuesPerField = randomIntBetween(2, 5);
+        randomLongsTestCase(
+            valuesPerField,
+            stats("_name")
+                .field("field")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap())),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testFieldScriptSingleValuedField() throws IOException {
+        randomLongsTestCase(
+            1,
+            stats("_name")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", "field"))),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testFieldScriptMultiValuedField() throws IOException {
+        final int valuesPerField = randomIntBetween(2, 5);
+        randomLongsTestCase(
+            valuesPerField,
+            stats("_name")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", "field"))),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testMissing() throws IOException {
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+
+        final long missingValue = randomIntBetween(-100, 100);
+
+        final int numDocs = randomIntBetween(100, 200);
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        final SimpleStatsAggregator expected = new SimpleStatsAggregator();
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                final long value = randomLongBetween(-100, 100);
+                docs.add(singleton(new SortedNumericDocValuesField(ft.name(), value)));
+                expected.add(value);
+            } else {
+                docs.add(emptySet());
+                expected.add(missingValue);
+            }
+        }
+
+        testCase(
+            stats("_name")
+                .field(ft.name())
+                .missing(missingValue),
+            iw -> iw.addDocuments(docs),
+            stats -> {
+                assertEquals(expected.count, stats.getCount(), 0);
+                assertEquals(expected.sum, stats.getSum(), TOLERANCE);
+                assertEquals(expected.max, stats.getMax(), 0);
+                assertEquals(expected.min, stats.getMin(), 0);
+                assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            },
+            singleton(ft)
+        );
+    }
+
+    public void testMissingUnmapped() throws IOException {
+        final int valuesPerField = randomIntBetween(1, 5);
+        final long missingValue = randomLongBetween(-100, 100);
+        randomLongsTestCase(
+            valuesPerField,
+            stats("_name")
+                .field("unknown_field")
+                .missing(missingValue),
+            (expected, stats) -> {
+                final long numDocs = expected.count / valuesPerField;
+                assertEquals(numDocs, stats.getCount());
+                assertEquals(numDocs * missingValue, stats.getSum(), TOLERANCE);
+                assertEquals(missingValue, stats.getMax(), 0);
+                assertEquals(missingValue, stats.getMin(), 0);
+                assertEquals(missingValue, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
             }
         );
     }
 
-    public void testCase(MappedFieldType ft,
-                         CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
-                         Consumer<InternalStats> verify) throws IOException {
+    private void randomLongsTestCase(int valuesPerField,
+                                     StatsAggregationBuilder builder,
+                                     BiConsumer<SimpleStatsAggregator, InternalStats> verify) throws IOException {
+
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+
+        final int numDocs = randomIntBetween(10, 50);
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        final SimpleStatsAggregator expected = new SimpleStatsAggregator();
+        for (int iDoc = 0; iDoc < numDocs; iDoc++) {
+            List<Long> values = randomList(valuesPerField, valuesPerField, () -> randomLongBetween(-100, 100));
+            docs.add(values.stream()
+                .map(value -> new SortedNumericDocValuesField(ft.name(), value))
+                .collect(toSet()));
+            values.forEach(expected::add);
+        }
+
+        testCase(
+            builder,
+            iw -> iw.addDocuments(docs),
+            stats -> verify.accept(expected, stats),
+            singleton(ft)
+        );
+    }
+
+    private void testCase(StatsAggregationBuilder builder,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalStats> verify,
+                          Collection<MappedFieldType> fieldTypes) throws IOException {
         try (Directory directory = newDirectory();
-             RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+            RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
             buildIndex.accept(indexWriter);
             try (IndexReader reader = indexWriter.getReader()) {
                 IndexSearcher searcher = new IndexSearcher(reader);
-                StatsAggregationBuilder aggBuilder = new StatsAggregationBuilder("my_agg").field("field");
-                InternalStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft);
+                final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]);
+                final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, fieldTypesArray);
                 verify.accept(stats);
             }
         }
@@ -199,6 +450,15 @@ static class SimpleStatsAggregator {
         double max = Long.MIN_VALUE;
         double sum = 0;
 
+        SimpleStatsAggregator() {}
+
+        SimpleStatsAggregator(long count, double min, double max, double sum) {
+            this.count = count;
+            this.min = min;
+            this.max = max;
+            this.sum = sum;
+        }
+
         void add(double value) {
             count ++;
             if (Double.compare(value, min) < 0) {
@@ -210,4 +470,32 @@ void add(double value) {
             sum += value;
         }
     }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return singletonList(CoreValuesSourceType.NUMERIC);
+    }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new StatsAggregationBuilder("_name")
+            .field(fieldName);
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        final Map<String, Function<Map<String, Object>, Object>> scripts = Map.of(
+            VALUE_SCRIPT_NAME, vars -> ((Number) vars.get("_value")).doubleValue() + 1,
+            FIELD_SCRIPT_NAME, vars -> {
+                final String fieldName = (String) vars.get("field");
+                final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc");
+                return lookup.get(fieldName).stream()
+                    .map(value -> ((Number) value).longValue() + 1)
+                    .collect(toList());
+            }
+        );
+        final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap());
+        final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
index d393b4f0b2c64..7a084c21d80f5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
@@ -35,11 +35,8 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
-import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -86,40 +83,6 @@ public void testEmptyAggregation() throws Exception {
         assertThat(Double.isNaN(stats.getAvg()), is(true));
     }
 
-    @Override
-    public void testUnmapped() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx_unmapped")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").field("value"))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo(Double.NaN));
-        assertThat(stats.getMin(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(stats.getMax(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(stats.getSum(), equalTo(0.0));
-        assertThat(stats.getCount(), equalTo(0L));
-    }
-
-    public void testPartiallyUnmapped() {
-        Stats s1 = client().prepareSearch("idx")
-                .addAggregation(stats("stats").field("value")).get()
-                .getAggregations().get("stats");
-        Stats s2 = client().prepareSearch("idx", "idx_unmapped")
-                .addAggregation(stats("stats").field("value")).get()
-                .getAggregations().get("stats");
-        assertEquals(s1.getAvg(), s2.getAvg(), 1e-10);
-        assertEquals(s1.getCount(), s2.getCount());
-        assertEquals(s1.getMin(), s2.getMin(), 0d);
-        assertEquals(s1.getMax(), s2.getMax(), 0d);
-    }
-
     @Override
     public void testSingleValuedField() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -199,77 +162,6 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat((double) ((InternalAggregation)global).getProperty("stats.count"), equalTo((double) expectedCountValue));
     }
 
-    @Override
-    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").field("value"))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(10.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testSingleValuedFieldWithValueScript() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("value")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", emptyMap())))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("inc", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("value")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
     @Override
     public void testMultiValuedField() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -291,157 +183,6 @@ public void testMultiValuedField() throws Exception {
         assertThat(stats.getCount(), equalTo(20L));
     }
 
-    @Override
-    public void testMultiValuedFieldWithValueScript() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("values")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11) / 20));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
-    @Override
-    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("values")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params)))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11) / 20));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
-    @Override
-    public void testScriptSingleValued() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap())))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(10.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testScriptSingleValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("inc", 1);
-
-        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value + inc", params);
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").script(script))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testScriptMultiValued() throws Exception {
-        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values']", emptyMap());
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").script(script))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12) / 20));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(12.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
-    @Override
-    public void testScriptMultiValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
-
-        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "[ doc['value'].value, doc['value'].value - dec ]",
-            params);
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .script(script))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10+0+1+2+3+4+5+6+7+8+9) / 20));
-        assertThat(stats.getMin(), equalTo(0.0));
-        assertThat(stats.getMax(), equalTo(10.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10+0+1+2+3+4+5+6+7+8+9));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
     @Override
     public void testOrderByEmptyAggregation() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index fb6b021eda0bf..42ace8a25f4bf 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -164,34 +164,6 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat((double) ((InternalAggregation)sum).getProperty("value"), equalTo(expectedSumValue));
     }
 
-    /** This test has been moved to {@link SumAggregatorTests#testPartiallyUnmapped()} */
-    @Override
-    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptSingleValuedField()} */
-    @Override
-    public void testSingleValuedFieldWithValueScript() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptSingleValuedField()} */
-    @Override
-    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptSingleValuedField()} */
-    @Override
-    public void testScriptSingleValued() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptSingleValuedField()} */
-    @Override
-    public void testScriptSingleValuedWithParams() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptMultiValuedField()} */
-    @Override
-    public void testScriptMultiValued() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptMultiValuedField()} */
-    @Override
-    public void testScriptMultiValuedWithParams() throws Exception {}
-
     @Override
     public void testMultiValuedField() throws Exception {
 
@@ -208,14 +180,6 @@ public void testMultiValuedField() throws Exception {
         assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12));
     }
 
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptMultiValuedField()} */
-    @Override
-    public void testMultiValuedFieldWithValueScript() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptMultiValuedField()} */
-    @Override
-    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {}
-
     @Override
     public void testOrderByEmptyAggregation() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
index d3d54643f9461..0fb1af419edcc 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
@@ -67,33 +67,33 @@ public void setupSuiteScopeCluster() throws Exception {
         ensureSearchable();
     }
 
-    public abstract void testEmptyAggregation() throws Exception;
+    public void testEmptyAggregation() throws Exception {}
 
-    public abstract void testUnmapped() throws Exception;
+    public void testUnmapped() throws Exception {}
 
-    public abstract void testSingleValuedField() throws Exception;
+    public void testSingleValuedField() throws Exception {}
 
-    public abstract void testSingleValuedFieldGetProperty() throws Exception;
+    public void testSingleValuedFieldGetProperty() throws Exception {}
 
-    public abstract void testSingleValuedFieldPartiallyUnmapped() throws Exception;
+    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {}
 
-    public abstract void testSingleValuedFieldWithValueScript() throws Exception;
+    public void testSingleValuedFieldWithValueScript() throws Exception {}
 
-    public abstract void testSingleValuedFieldWithValueScriptWithParams() throws Exception;
+    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {}
 
-    public abstract void testMultiValuedField() throws Exception;
+    public void testMultiValuedField() throws Exception {}
 
-    public abstract void testMultiValuedFieldWithValueScript() throws Exception;
+    public void testMultiValuedFieldWithValueScript() throws Exception {}
 
-    public abstract void testMultiValuedFieldWithValueScriptWithParams() throws Exception;
+    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {}
 
-    public abstract void testScriptSingleValued() throws Exception;
+    public void testScriptSingleValued() throws Exception {}
 
-    public abstract void testScriptSingleValuedWithParams() throws Exception;
+    public void testScriptSingleValuedWithParams() throws Exception {}
 
-    public abstract void testScriptMultiValued() throws Exception;
+    public void testScriptMultiValued() throws Exception {}
 
-    public abstract void testScriptMultiValuedWithParams() throws Exception;
+    public void testScriptMultiValuedWithParams() throws Exception {}
 
-    public abstract void testOrderByEmptyAggregation() throws Exception;
+    public void testOrderByEmptyAggregation() throws Exception {}
 }

From f39aa7515b02e68fb31bfb394e6e2cc11945b8d4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 22:50:43 -0400
Subject: [PATCH 254/285] Use onlyIf for build Docker image task execution
 (#54047)

This commit switches to using an onlyIf to determine if a build Docker
image task execution should occur. This is preferred since it means that
the determination is performed at task execution time, rather than
during configuration.
---
 distribution/docker/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index b9ccfe95ac401..f67e520ce76a4 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -191,8 +191,8 @@ void addBuildDockerImage(final String architecture, final boolean oss) {
       ]
     }
   }
+  buildDockerImageTask.onlyIf { Architecture.current().name().toLowerCase().equals(architecture) }
   assemble.dependsOn(buildDockerImageTask)
-  buildDockerImageTask.enabled = Architecture.current().name().toLowerCase().equals(architecture)
 }
 
 for (final String architecture : ["aarch64", "x64"]) {

From 94e81b0aa0be5bab9ea2b77f9a9b101e979b8350 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Mar 2020 22:53:51 -0400
Subject: [PATCH 255/285] Introduce autoscaling decisions (#53934)

This is the first in a series of commits that will introduce the
autoscaling deciders framework. This commit introduces the basic
framework for representing autoscaling decisions.
---
 .../apis/get-autoscaling-decision.asciidoc    |  2 +-
 .../autoscaling/AutoscalingDecision.java      | 85 +++++++++++++++++++
 .../autoscaling/AutoscalingDecisionType.java  | 73 ++++++++++++++++
 .../autoscaling/AutoscalingDecisions.java     | 77 +++++++++++++++++
 .../action/GetAutoscalingDecisionAction.java  | 26 +++++-
 ...TransportGetAutoscalingDecisionAction.java |  4 +-
 .../autoscaling/AutoscalingDecisionTests.java | 31 +++++++
 ...alingDecisionTypeWireSerializingTests.java | 54 ++++++++++++
 ...toscalingDecisionWireSerializingTests.java | 24 ++++++
 .../AutoscalingDecisionsTests.java            | 35 ++++++++
 ...oscalingDecisionsWireSerializingTests.java | 24 ++++++
 .../autoscaling/AutoscalingTestCase.java      | 56 ++++++++++++
 12 files changed, 485 insertions(+), 6 deletions(-)
 create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java
 create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java
 create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java

diff --git a/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc
index aa66e3a0d034f..dfa14ac180636 100644
--- a/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc
+++ b/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc
@@ -47,6 +47,6 @@ The API returns the following result:
 [source,console-result]
 --------------------------------------------------
 {
-
+  decisions: []
 }
 --------------------------------------------------
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java
new file mode 100644
index 0000000000000..4f4a3ce70981b
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Represents an autoscaling decision.
+ */
+public class AutoscalingDecision implements ToXContent, Writeable {
+
+    private final String name;
+
+    public String name() {
+        return name;
+    }
+
+    private final AutoscalingDecisionType type;
+
+    public AutoscalingDecisionType type() {
+        return type;
+    }
+
+    private final String reason;
+
+    public String reason() {
+        return reason;
+    }
+
+    public AutoscalingDecision(final String name, final AutoscalingDecisionType type, final String reason) {
+        this.name = Objects.requireNonNull(name);
+        this.type = Objects.requireNonNull(type);
+        this.reason = Objects.requireNonNull(reason);
+    }
+
+    public AutoscalingDecision(final StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.type = AutoscalingDecisionType.readFrom(in);
+        this.reason = in.readString();
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeString(name);
+        type.writeTo(out);
+        out.writeString(reason);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field("name", name);
+            builder.field("type", type);
+            builder.field("reason", reason);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final AutoscalingDecision that = (AutoscalingDecision) o;
+        return name.equals(that.name) && type == that.type && reason.equals(that.reason);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, type, reason);
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java
new file mode 100644
index 0000000000000..8590e26426291
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/**
+ * Represents the type of an autoscaling decision: to indicating if a scale down, no scaling event, or a scale up is needed.
+ */
+public enum AutoscalingDecisionType implements Writeable, ToXContentFragment {
+
+    /**
+     * Indicates that a scale down event is needed.
+     */
+    SCALE_DOWN((byte) 0),
+
+    /**
+     * Indicates that no scaling event is needed.
+     */
+    NO_SCALE((byte) 1),
+
+    /**
+     * Indicates that a scale up event is needed.
+     */
+    SCALE_UP((byte) 2);
+
+    private final byte id;
+
+    byte id() {
+        return id;
+    }
+
+    AutoscalingDecisionType(final byte id) {
+        this.id = id;
+    }
+
+    public static AutoscalingDecisionType readFrom(final StreamInput in) throws IOException {
+        final byte id = in.readByte();
+        switch (id) {
+            case 0:
+                return SCALE_DOWN;
+            case 1:
+                return NO_SCALE;
+            case 2:
+                return SCALE_UP;
+            default:
+                throw new IllegalArgumentException("unexpected value [" + id + "] for autoscaling decision type");
+        }
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeByte(id);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.value(name().toLowerCase(Locale.ROOT));
+        return builder;
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java
new file mode 100644
index 0000000000000..5e7e23fcc35dd
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+
+/**
+ * Represents a collection of individual autoscaling decisions that can be aggregated into a single autoscaling decision.
+ */
+public class AutoscalingDecisions implements ToXContent, Writeable {
+
+    private final Collection<AutoscalingDecision> decisions;
+
+    public AutoscalingDecisions(final Collection<AutoscalingDecision> decisions) {
+        Objects.requireNonNull(decisions);
+        if (decisions.isEmpty()) {
+            throw new IllegalArgumentException("decisions can not be empty");
+        }
+        this.decisions = decisions;
+    }
+
+    public AutoscalingDecisions(final StreamInput in) throws IOException {
+        this.decisions = in.readList(AutoscalingDecision::new);
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeCollection(decisions);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        for (final AutoscalingDecision decision : decisions) {
+            decision.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+    public AutoscalingDecisionType type() {
+        if (decisions.stream().anyMatch(p -> p.type() == AutoscalingDecisionType.SCALE_UP)) {
+            // if any deciders say to scale up
+            return AutoscalingDecisionType.SCALE_UP;
+        } else if (decisions.stream().allMatch(p -> p.type() == AutoscalingDecisionType.SCALE_DOWN)) {
+            // if all deciders say to scale down
+            return AutoscalingDecisionType.SCALE_DOWN;
+        } else {
+            // otherwise, do not scale
+            return AutoscalingDecisionType.NO_SCALE;
+        }
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final org.elasticsearch.xpack.autoscaling.AutoscalingDecisions that = (org.elasticsearch.xpack.autoscaling.AutoscalingDecisions) o;
+        return decisions.equals(that.decisions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(decisions);
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java
index 88ffc0fbc91c7..8e97eb0d3f64e 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java
@@ -14,8 +14,13 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.autoscaling.AutoscalingDecisions;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 public class GetAutoscalingDecisionAction extends ActionType<GetAutoscalingDecisionAction.Response> {
 
@@ -60,24 +65,37 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        public Response() {
+        private final SortedMap<String, AutoscalingDecisions> decisions;
 
+        public Response(final SortedMap<String, AutoscalingDecisions> decisions) {
+            this.decisions = Objects.requireNonNull(decisions);
         }
 
         public Response(final StreamInput in) throws IOException {
             super(in);
+            decisions = new TreeMap<>(in.readMap(StreamInput::readString, AutoscalingDecisions::new));
         }
 
         @Override
-        public void writeTo(final StreamOutput out) {
-
+        public void writeTo(final StreamOutput out) throws IOException {
+            out.writeMap(decisions, StreamOutput::writeString, (o, decision) -> decision.writeTo(o));
         }
 
         @Override
         public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
             builder.startObject();
             {
-
+                builder.startArray("decisions");
+                {
+                    for (final Map.Entry<String, AutoscalingDecisions> decision : decisions.entrySet()) {
+                        builder.startObject();
+                        {
+                            builder.field(decision.getKey(), decision.getValue());
+                        }
+                        builder.endObject();
+                    }
+                }
+                builder.endArray();
             }
             builder.endObject();
             return builder;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java
index 5f6b79358756d..ea1687239b74f 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.TreeMap;
 
 public class TransportGetAutoscalingDecisionAction extends TransportMasterNodeAction<
     GetAutoscalingDecisionAction.Request,
@@ -61,7 +63,7 @@ protected void masterOperation(
         final ClusterState state,
         final ActionListener<GetAutoscalingDecisionAction.Response> listener
     ) {
-        listener.onResponse(new GetAutoscalingDecisionAction.Response());
+        listener.onResponse(new GetAutoscalingDecisionAction.Response(Collections.unmodifiableSortedMap(new TreeMap<>())));
     }
 
     @Override
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java
new file mode 100644
index 0000000000000..c6c8dfa94d114
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoscalingDecisionTests extends AutoscalingTestCase {
+
+    public void testAutoscalingDecisionType() {
+        final AutoscalingDecisionType type = randomFrom(AutoscalingDecisionType.values());
+        final AutoscalingDecision decision = randomAutoscalingDecisionOfType(type);
+        assertThat(decision.type(), equalTo(type));
+    }
+
+    public void testAutoscalingDecisionTypeSerialization() throws IOException {
+        final AutoscalingDecisionType before = randomFrom(AutoscalingDecisionType.values());
+        final BytesStreamOutput out = new BytesStreamOutput();
+        before.writeTo(out);
+        final AutoscalingDecisionType after = AutoscalingDecisionType.readFrom(out.bytes().streamInput());
+        assertThat(after, equalTo(before));
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java
new file mode 100644
index 0000000000000..caa5c6440bbb4
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoscalingDecisionTypeWireSerializingTests extends AbstractWireSerializingTestCase<AutoscalingDecisionType> {
+
+    @Override
+    protected Writeable.Reader<AutoscalingDecisionType> instanceReader() {
+        return AutoscalingDecisionType::readFrom;
+    }
+
+    @Override
+    protected AutoscalingDecisionType createTestInstance() {
+        return randomFrom(AutoscalingDecisionType.values());
+    }
+
+    @Override
+    protected void assertEqualInstances(final AutoscalingDecisionType expectedInstance, final AutoscalingDecisionType newInstance) {
+        assertSame(expectedInstance, newInstance);
+        assertEquals(expectedInstance, newInstance);
+        assertEquals(expectedInstance.hashCode(), newInstance.hashCode());
+    }
+
+    public void testInvalidAutoscalingDecisionTypeSerialization() throws IOException {
+        final BytesStreamOutput out = new BytesStreamOutput();
+        final Set<Byte> values = Arrays.stream(AutoscalingDecisionType.values())
+            .map(AutoscalingDecisionType::id)
+            .collect(Collectors.toSet());
+        final byte value = randomValueOtherThanMany(values::contains, ESTestCase::randomByte);
+        out.writeByte(value);
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> AutoscalingDecisionType.readFrom(out.bytes().streamInput())
+        );
+        assertThat(e.getMessage(), equalTo("unexpected value [" + value + "] for autoscaling decision type"));
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java
new file mode 100644
index 0000000000000..f24dbcabdd090
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class AutoscalingDecisionWireSerializingTests extends AbstractWireSerializingTestCase<AutoscalingDecision> {
+
+    @Override
+    protected Writeable.Reader<AutoscalingDecision> instanceReader() {
+        return AutoscalingDecision::new;
+    }
+
+    @Override
+    protected AutoscalingDecision createTestInstance() {
+        return AutoscalingTestCase.randomAutoscalingDecision();
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java
new file mode 100644
index 0000000000000..c397f8650f7b6
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoscalingDecisionsTests extends AutoscalingTestCase {
+
+    public void testAutoscalingDecisionsRejectsEmptyDecisions() {
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AutoscalingDecisions(List.of()));
+        assertThat(e.getMessage(), equalTo("decisions can not be empty"));
+    }
+
+    public void testAutoscalingDecisionsTypeDown() {
+        final AutoscalingDecisions decisions = randomAutoscalingDecisions(randomIntBetween(1, 8), 0, 0);
+        assertThat(decisions.type(), equalTo(AutoscalingDecisionType.SCALE_DOWN));
+    }
+
+    public void testAutoscalingDecisionsTypeNo() {
+        final AutoscalingDecisions decision = randomAutoscalingDecisions(randomIntBetween(0, 8), randomIntBetween(1, 8), 0);
+        assertThat(decision.type(), equalTo(AutoscalingDecisionType.NO_SCALE));
+    }
+
+    public void testAutoscalingDecisionsTypeUp() {
+        final AutoscalingDecisions decision = randomAutoscalingDecisions(0, 0, randomIntBetween(1, 8));
+        assertThat(decision.type(), equalTo(AutoscalingDecisionType.SCALE_UP));
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java
new file mode 100644
index 0000000000000..c058d7c81d5cf
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class AutoscalingDecisionsWireSerializingTests extends AbstractWireSerializingTestCase<AutoscalingDecisions> {
+
+    @Override
+    protected Writeable.Reader<AutoscalingDecisions> instanceReader() {
+        return AutoscalingDecisions::new;
+    }
+
+    @Override
+    protected AutoscalingDecisions createTestInstance() {
+        return AutoscalingTestCase.randomAutoscalingDecisions();
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java
new file mode 100644
index 0000000000000..29b9a3c41bda9
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class AutoscalingTestCase extends ESTestCase {
+
+    static AutoscalingDecision randomAutoscalingDecision() {
+        return randomAutoscalingDecisionOfType(randomFrom(AutoscalingDecisionType.values()));
+    }
+
+    static AutoscalingDecision randomAutoscalingDecisionOfType(final AutoscalingDecisionType type) {
+        return new AutoscalingDecision(randomAlphaOfLength(8), type, randomAlphaOfLength(8));
+    }
+
+    static AutoscalingDecisions randomAutoscalingDecisions() {
+        final int numberOfDecisions = 1 + randomIntBetween(1, 8);
+        final List<AutoscalingDecision> decisions = new ArrayList<>(numberOfDecisions);
+        for (int i = 0; i < numberOfDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.SCALE_DOWN));
+        }
+        final int numberOfDownDecisions = randomIntBetween(0, 8);
+        final int numberOfNoDecisions = randomIntBetween(0, 8);
+        final int numberOfUpDecisions = randomIntBetween(numberOfDownDecisions + numberOfNoDecisions == 0 ? 1 : 0, 8);
+        return randomAutoscalingDecisions(numberOfDownDecisions, numberOfNoDecisions, numberOfUpDecisions);
+    }
+
+    static AutoscalingDecisions randomAutoscalingDecisions(
+        final int numberOfDownDecisions,
+        final int numberOfNoDecisions,
+        final int numberOfUpDecisions
+    ) {
+        final List<AutoscalingDecision> decisions = new ArrayList<>(numberOfDownDecisions + numberOfNoDecisions + numberOfUpDecisions);
+        for (int i = 0; i < numberOfDownDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.SCALE_DOWN));
+        }
+        for (int i = 0; i < numberOfNoDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.NO_SCALE));
+        }
+        for (int i = 0; i < numberOfUpDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.SCALE_UP));
+        }
+        Randomness.shuffle(decisions);
+        return new AutoscalingDecisions(decisions);
+    }
+
+}

From 7989d676ec105117d7a342232b7d5bb6a7244698 Mon Sep 17 00:00:00 2001
From: muachilin <32566798+muachilin@users.noreply.github.com>
Date: Tue, 24 Mar 2020 11:22:50 +0800
Subject: [PATCH 256/285] Deprecate alternatives to the hot threads API
 (#52930)

This commit deprecates various undocumented alternatives to the hot
threads API.
---
 .../cluster/RestNodesHotThreadsAction.java    | 57 ++++++++++++++++---
 1 file changed, 49 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index 0b3d8b698cf58..ea7f97c3815f7 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -34,23 +34,64 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-
 public class RestNodesHotThreadsAction extends BaseRestHandler {
 
+    private static final String formatDeprecatedMessageWithoutNodeID = "[%s] is a deprecated endpoint. " + 
+        "Please use [/_nodes/hot_threads] instead.";
+    private static final String formatDeprecatedMessageWithNodeID = "[%s] is a deprecated endpoint. " + 
+        "Please use [/_nodes/{nodeId}/hot_threads] instead.";
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithoutNodeID, 
+        "/_cluster/nodes/hot_threads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithNodeID, 
+        "/_cluster/nodes/{nodeId}/hot_threads"
+    );
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithoutNodeID, 
+        "/_cluster/nodes/hotthreads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithNodeID, 
+        "/_cluster/nodes/{nodeId}/hotthreads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_NODES_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithoutNodeID, 
+        "/_nodes/hotthreads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithNodeID, 
+        "/_nodes/{nodeId}/hotthreads"
+    );
+
+    @Override
+    public List<DeprecatedRoute> deprecatedRoutes() {
+        return List.of(
+            new DeprecatedRoute(GET, "/_cluster/nodes/hot_threads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS),
+            new DeprecatedRoute(GET, "/_cluster/nodes/{nodeId}/hot_threads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS),
+            new DeprecatedRoute(GET, "/_cluster/nodes/hotthreads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS),
+            new DeprecatedRoute(GET, "/_cluster/nodes/{nodeId}/hotthreads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS),
+            new DeprecatedRoute(GET, "/_nodes/hotthreads",
+                    DEPRECATED_MESSAGE_NODES_HOTTHREADS),
+            new DeprecatedRoute(GET, "/_nodes/{nodeId}/hotthreads",
+                    DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS));
+    }
+
     @Override
     public List<Route> routes() {
         return List.of(
-            new Route(GET, "/_cluster/nodes/hotthreads"),
-            new Route(GET, "/_cluster/nodes/hot_threads"),
-            new Route(GET, "/_cluster/nodes/{nodeId}/hotthreads"),
-            new Route(GET, "/_cluster/nodes/{nodeId}/hot_threads"),
-            new Route(GET, "/_nodes/hotthreads"),
             new Route(GET, "/_nodes/hot_threads"),
-            new Route(GET, "/_nodes/{nodeId}/hotthreads"),
-            new Route(GET, "/_nodes/{nodeId}/hot_threads"));
+            new Route(GET, "/_nodes/{nodeId}/hot_threads")
+        );
     }
 
     @Override

From b56b9fb67409859acb609cfd4e6158b57d43c1a4 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 23 Mar 2020 15:15:30 -0700
Subject: [PATCH 257/285] Mute
 BlobStoreIncrementalityIT.testIncrementalBehaviorOnPrimaryFailover

---
 .../org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
index 897e75a2e0d9f..21e7d3fe0cc2e 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
@@ -46,6 +46,7 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase {
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54034")
     public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedException, ExecutionException, IOException {
         internalCluster().startMasterOnlyNode();
         final String primaryNode = internalCluster().startDataOnlyNode();

From 4ff42331ffa4778610fc11131eead2bee7bc05bd Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 23 Mar 2020 21:31:04 -0700
Subject: [PATCH 258/285] Fix nasty errors when importing into IntelliJ

---
 .gitignore        |  1 +
 gradle/ide.gradle | 23 +----------------------
 2 files changed, 2 insertions(+), 22 deletions(-)

diff --git a/.gitignore b/.gitignore
index 309342f3b611d..ff9e964289f37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ out/
 # include shared intellij config
 !.idea/scopes/x_pack.xml
 !.idea/inspectionProfiles/Project_Default.xml
+!.idea/runConfigurations/Debug_Elasticsearch.xml
 
 # These files are generated in the main tree by IntelliJ
 benchmarks/src/main/generated/*
diff --git a/gradle/ide.gradle b/gradle/ide.gradle
index 62c0caa37ec42..c73d6e5a1bfdd 100644
--- a/gradle/ide.gradle
+++ b/gradle/ide.gradle
@@ -35,27 +35,6 @@ tasks.register('configureIdeaGradleJvm') {
   }
 }
 
-tasks.register('configureIdeaRunConfigs') {
-  group = 'ide'
-  description = 'Configures default run configuration settings'
-
-  doLast {
-    modifyXml('.idea/workspace.xml') { xml ->
-      def runManager = xml.component.find { it.'@name' == 'RunManager' }
-      if (runManager == null) {
-        throw new GradleException("IntelliJ 'RunManager' configuration is missing from workspace.xml. You may need to refresh your Gradle project.")
-      }
-
-      def debugConfig = runManager.configuration.find { it.'@name' == 'Debug Elasticsearch' }
-      // Enable "auto restart" on remote debug run configuration
-      if (debugConfig.option.any { it.'@name' == 'AUTO_RESTART' && it.'@value' == 'true'} == false) {
-        def restart = new NodeBuilder().option(name: 'AUTO_RESTART', value: 'true')
-        debugConfig.append(restart)
-      }
-    }
-  }
-}
-
 idea {
   project {
     vcs = 'Git'
@@ -67,7 +46,7 @@ idea {
         testRunner = 'choose_per_test'
       }
       taskTriggers {
-        afterSync tasks.named('configureIdeaGradleJvm'), tasks.named('configureIdeaRunConfigs')
+        afterSync tasks.named('configureIdeaGradleJvm')
       }
       codeStyle {
         java {

From 20736b21b98a4f6bdb084834c6a8968c6d0a0e0a Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 23 Mar 2020 21:32:11 -0700
Subject: [PATCH 259/285] Add remote debug run configuration for IntelliJ

---
 .idea/runConfigurations/Debug_Elasticsearch.xml | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 .idea/runConfigurations/Debug_Elasticsearch.xml

diff --git a/.idea/runConfigurations/Debug_Elasticsearch.xml b/.idea/runConfigurations/Debug_Elasticsearch.xml
new file mode 100644
index 0000000000000..685c59c1846a4
--- /dev/null
+++ b/.idea/runConfigurations/Debug_Elasticsearch.xml
@@ -0,0 +1,11 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Debug Elasticsearch" type="Remote">
+    <option name="USE_SOCKET_TRANSPORT" value="true" />
+    <option name="SERVER_MODE" value="true" />
+    <option name="SHMEM_ADDRESS" />
+    <option name="HOST" value="localhost" />
+    <option name="PORT" value="5005" />
+    <option name="AUTO_RESTART" value="false" />
+    <method v="2" />
+  </configuration>
+</component>
\ No newline at end of file

From f9526f9e1ed2f2ae38781624eb846a00b1a5c9c2 Mon Sep 17 00:00:00 2001
From: Christos Soulios <1561376+csoulios@users.noreply.github.com>
Date: Tue, 24 Mar 2020 08:42:52 +0200
Subject: [PATCH 260/285] Add supported-type tests to avg aggregation (#54011)

---
 .../aggregations/metrics/AvgAggregatorTests.java   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
index 46f08d786a856..b34132a51e00a 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
@@ -52,7 +52,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
@@ -709,4 +711,16 @@ public void testScriptCaching() throws IOException {
         directory.close();
         unmappedDirectory.close();
     }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(
+            CoreValuesSourceType.NUMERIC
+        );
+    }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new AvgAggregationBuilder("foo").field(fieldName);
+    }
 }

From d0464893f6ad30440cc5162277538663702d7a7d Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Tue, 24 Mar 2020 18:25:43 +1100
Subject: [PATCH 261/285] Validate role templates before saving role mapping
 (#52636)

Role names are now compiled from role templates before role mapping is saved.
This serves as validation for role templates to prevent malformed and invalid scripts
to be persisted, which could later break authentication.

Resolves: #48773
---
 .../security/create-role-mappings.asciidoc    |  54 ++++----
 .../support/mapper/TemplateRoleName.java      |  12 ++
 .../support/mapper/TemplateRoleNameTests.java | 117 ++++++++++++++++++
 .../mapper/NativeRoleMappingStore.java        |   5 +
 .../mapper/NativeRoleMappingStoreTests.java   |  15 +++
 5 files changed, 176 insertions(+), 27 deletions(-)

diff --git a/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc b/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc
index e80ff662c8305..51ccb39a69eca 100644
--- a/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc
+++ b/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc
@@ -23,7 +23,7 @@ Creates and updates role mappings.
 [[security-api-put-role-mapping-desc]]
 ==== {api-description-title}
 
-Role mappings define which roles are assigned to each user. Each mapping has 
+Role mappings define which roles are assigned to each user. Each mapping has
 _rules_ that identify users and a list of _roles_ that are granted to those users.
 
 The role mapping APIs are generally the preferred way to manage role mappings
@@ -37,6 +37,31 @@ roles API>> or <<roles-management-file,roles files>>.
 
 For more information, see <<mapping-roles>>.
 
+[[_role_templates]]
+===== Role templates
+
+The most common use for role mappings is to create a mapping from a known value
+on the user to a fixed role name. For example, all users in the
+`cn=admin,dc=example,dc=com` LDAP group should be given the `superuser` role in
+{es}. The `roles` field is used for this purpose.
+
+For more complex needs, it is possible to use Mustache templates to dynamically
+determine the names of the roles that should be granted to the user. The
+`role_templates` field is used for this purpose.
+
+NOTE: To use role templates successfully, the relevant scripting feature must be
+enabled. Otherwise, all attempts to create a role mapping with role templates
+fail. See <<allowed-script-types-setting>>.
+
+All of the <<role-mapping-resources,user fields>> that are available in the
+role mapping `rules` are also available in the role templates. Thus it is possible
+to assign a user to a role that reflects their `username`, their `groups`, or the
+name of the `realm` to which they authenticated.
+
+By default a template is evaluated to produce a single string that is the name
+of the role which should be assigned to the user. If the `format` of the template
+is set to `"json"` then the template is expected to produce a JSON string or an
+array of JSON strings for the role names.
 
 [[security-api-put-role-mapping-path-params]]
 ==== {api-path-parms-title}
@@ -77,31 +102,7 @@ _Exactly one of `roles` or `role_templates` must be specified_.
 `rules`::
 (Required, object) The rules that determine which users should be matched by the
 mapping. A rule is a logical condition that is expressed by using a JSON DSL.
-See  <<role-mapping-resources>>. 
-
-==== Role Templates
-
-The most common use for role mappings is to create a mapping from a known value
-on the user to a fixed role name.
-For example, all users in the `cn=admin,dc=example,dc=com` LDAP group should be
-given the `superuser` role in {es}.
-The `roles` field is used for this purpose.
-
-For more complex needs it is possible to use Mustache templates to dynamically
-determine the names of the roles that should be granted to the user.
-The `role_templates` field is used for this purpose.
-
-All of the <<role-mapping-resources,user fields>> that are available in the
-role mapping `rules` are also available in the role templates. Thus it is possible
-to assign a user to a role that reflects their `username`, their `groups` or the
-name of the `realm` to which they authenticated.
-
-By default a template is evaluated to produce a single string that is the name
-of the role which should be assigned to the user. If the `format` of the template
-is set to `"json"` then the template is expected to produce a JSON string, or an
-array of JSON strings for the role name(s).
-
-The Examples section below demonstrates the use of templated role names.
+See  <<role-mapping-resources>>.
 
 [[security-api-put-role-mapping-example]]
 ==== {api-examples-title}
@@ -339,4 +340,3 @@ POST /_security/role_mapping/mapping9
 <1> Because it is not possible to specify both `roles` and `role_templates` in
     the same role mapping, we can apply a "fixed name" role by using a template
     that has no substitutions.
-
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java
index 59f9eafec1c00..e5a039b89a1ad 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java
@@ -97,6 +97,18 @@ public List<String> getRoleNames(ScriptService scriptService, ExpressionModel mo
         }
     }
 
+    public void validate(ScriptService scriptService) {
+        try {
+            parseTemplate(scriptService, Collections.emptyMap());
+        } catch (IllegalArgumentException e) {
+            throw e;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (Exception e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
     private List<String> convertJsonToList(String evaluation) throws IOException {
         final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY,
             LoggingDeprecationHandler.INSTANCE, evaluation);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
index cab10ca728323..19433d132f1ed 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
@@ -6,6 +6,9 @@
 
 package org.elasticsearch.xpack.core.security.authc.support.mapper;
 
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -17,8 +20,11 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.ScriptMetaData;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.StoredScriptSource;
 import org.elasticsearch.script.mustache.MustacheScriptEngine;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
@@ -31,8 +37,11 @@
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TemplateRoleNameTests extends ESTestCase {
 
@@ -116,4 +125,112 @@ public void tryEquals(TemplateRoleName original) {
         };
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(original, copy, mutate);
     }
+
+    public void testValidate() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+
+        final TemplateRoleName plainString = new TemplateRoleName(new BytesArray("{ \"source\":\"heroes\" }"), Format.STRING);
+        plainString.validate(scriptService);
+
+        final TemplateRoleName user = new TemplateRoleName(new BytesArray("{ \"source\":\"_user_{{username}}\" }"), Format.STRING);
+        user.validate(scriptService);
+
+        final TemplateRoleName groups = new TemplateRoleName(new BytesArray("{ \"source\":\"{{#tojson}}groups{{/tojson}}\" }"),
+            Format.JSON);
+        groups.validate(scriptService);
+
+        final TemplateRoleName notObject = new TemplateRoleName(new BytesArray("heroes"), Format.STRING);
+        expectThrows(IllegalArgumentException.class, () -> notObject.validate(scriptService));
+
+        final TemplateRoleName invalidField = new TemplateRoleName(new BytesArray("{ \"foo\":\"heroes\" }"), Format.STRING);
+        expectThrows(IllegalArgumentException.class, () -> invalidField.validate(scriptService));
+    }
+
+    public void testValidateWillPassWithEmptyContext() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+
+        final BytesReference template = new BytesArray("{ \"source\":\"" +
+            "{{username}}/{{dn}}/{{realm}}/{{metadata}}" +
+            "{{#realm}}" +
+            "  {{name}}/{{type}}" +
+            "{{/realm}}" +
+            "{{#toJson}}groups{{/toJson}}" +
+            "{{^groups}}{{.}}{{/groups}}" +
+            "{{#metadata}}" +
+            "  {{#first}}" +
+            "    <li><strong>{{name}}</strong></li>" +
+            "  {{/first}}" +
+            "  {{#link}}" +
+            "    <li><a href=\\\"{{url}}\\\">{{name}}</a></li>" +
+            "  {{/link}}" +
+            "  {{#toJson}}subgroups{{/toJson}}" +
+            "  {{something-else}}" +
+            "{{/metadata}}\" }");
+        final TemplateRoleName templateRoleName = new TemplateRoleName(template, Format.STRING);
+        templateRoleName.validate(scriptService);
+    }
+
+    public void testValidateWillFailForSyntaxError() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+
+        final BytesReference template = new BytesArray("{ \"source\":\" {{#not-closed}} {{other-variable}} \" }");
+
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(template, Format.STRING).validate(scriptService));
+        assertTrue(e.getCause() instanceof ScriptException);
+    }
+
+    public void testValidationWillFailWhenInlineScriptIsNotEnabled() {
+        final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build();
+        final ScriptService scriptService = new ScriptService(settings,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+        final BytesReference inlineScript = new BytesArray("{ \"source\":\"\" }");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(inlineScript, Format.STRING).validate(scriptService));
+        assertThat(e.getMessage(), containsString("[inline]"));
+    }
+
+    public void testValidateWillFailWhenStoredScriptIsNotEnabled() {
+        final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build();
+        final ScriptService scriptService = new ScriptService(settings,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+        final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class);
+        final ClusterState clusterState = mock(ClusterState.class);
+        final MetaData metaData = mock(MetaData.class);
+        final StoredScriptSource storedScriptSource = mock(StoredScriptSource.class);
+        final ScriptMetaData scriptMetaData = new ScriptMetaData.Builder(null).storeScript("foo", storedScriptSource).build();
+        when(clusterChangedEvent.state()).thenReturn(clusterState);
+        when(clusterState.metaData()).thenReturn(metaData);
+        when(metaData.custom(ScriptMetaData.TYPE)).thenReturn(scriptMetaData);
+        when(storedScriptSource.getLang()).thenReturn("mustache");
+        when(storedScriptSource.getSource()).thenReturn("");
+        when(storedScriptSource.getOptions()).thenReturn(Collections.emptyMap());
+        scriptService.applyClusterState(clusterChangedEvent);
+
+        final BytesReference storedScript = new BytesArray("{ \"id\":\"foo\" }");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(storedScript, Format.STRING).validate(scriptService));
+        assertThat(e.getMessage(), containsString("[stored]"));
+    }
+
+    public void testValidateWillFailWhenStoredScriptIsNotFound() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+        final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class);
+        final ClusterState clusterState = mock(ClusterState.class);
+        final MetaData metaData = mock(MetaData.class);
+        final ScriptMetaData scriptMetaData = new ScriptMetaData.Builder(null).build();
+        when(clusterChangedEvent.state()).thenReturn(clusterState);
+        when(clusterState.metaData()).thenReturn(metaData);
+        when(metaData.custom(ScriptMetaData.TYPE)).thenReturn(scriptMetaData);
+        scriptService.applyClusterState(clusterChangedEvent);
+
+        final BytesReference storedScript = new BytesArray("{ \"id\":\"foo\" }");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(storedScript, Format.STRING).validate(scriptService));
+        assertThat(e.getMessage(), containsString("unable to find script"));
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
index 2bd02e6decb1f..9586ddb47ab20 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.TemplateRoleName;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
@@ -165,6 +166,10 @@ protected ExpressionRoleMapping buildMapping(String id, BytesReference source) {
      * Stores (create or update) a single mapping in the index
      */
     public void putRoleMapping(PutRoleMappingRequest request, ActionListener<Boolean> listener) {
+        // Validate all templates before storing the role mapping
+        for (TemplateRoleName templateRoleName : request.getRoleTemplates()) {
+            templateRoleName.validate(scriptService);
+        }
         modifyMapping(request.getName(), this::innerPutMapping, request, listener);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
index 9d96f1e115869..37490d0ddec11 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
@@ -210,6 +211,20 @@ public void testCacheIsNotClearedIfNoRealmsAreAttached() {
         assertEquals(0, numInvalidation.get());
     }
 
+    public void testPutRoleMappingWillValidateTemplateRoleNamesBeforeSave() {
+        final PutRoleMappingRequest putRoleMappingRequest = mock(PutRoleMappingRequest.class);
+        final TemplateRoleName templateRoleName = mock(TemplateRoleName.class);
+        final ScriptService scriptService = mock(ScriptService.class);
+        when(putRoleMappingRequest.getRoleTemplates()).thenReturn(Collections.singletonList(templateRoleName));
+        doAnswer(invocationOnMock -> {
+            throw new IllegalArgumentException();
+        }).when(templateRoleName).validate(scriptService);
+
+        final NativeRoleMappingStore nativeRoleMappingStore =
+            new NativeRoleMappingStore(Settings.EMPTY, mock(Client.class), mock(SecurityIndexManager.class), scriptService);
+        expectThrows(IllegalArgumentException.class, () -> nativeRoleMappingStore.putRoleMapping(putRoleMappingRequest, null));
+    }
+
     private NativeRoleMappingStore buildRoleMappingStoreForInvalidationTesting(AtomicInteger invalidationCounter, boolean attachRealm) {
         final Settings settings = Settings.builder().put("path.home", createTempDir()).build();
 

From 6b5752aa011430c78bf8a006231e20b21a2941d9 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 24 Mar 2020 09:11:34 +0100
Subject: [PATCH 262/285] Re-enable bwc tests (#54045)

This change re-enables bwc tests now that #54007 is merged in 7x.
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 9593b53b442aa..a23614bafa975 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/54009" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From 894714e6733f397e26fff7d9d2b9ee00dcbf2711 Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Tue, 24 Mar 2020 09:21:40 +0100
Subject: [PATCH 263/285] [Transform] add another reason for disabling BWC
 (#54054)

disable BWC in preparation for #54027
---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index a23614bafa975..792672d6021ff 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/54027" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From f4bdcd9b5897af5d43c7ae71bc9f5b74c9620210 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Tue, 24 Mar 2020 10:25:04 +0200
Subject: [PATCH 264/285] [ML] Mute data frame analytics BWC tests (#54030)

Until #53998 is backported to 7.x.
---
 .../test/mixed_cluster/90_ml_data_frame_analytics_crud.yml   | 5 +++++
 .../test/old_cluster/90_ml_data_frame_analytics_crud.yml     | 3 +++
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml     | 5 +++++
 3 files changed, 13 insertions(+)

diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index 5049f3360a42e..38c78803b6f61 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53998"
+
 ---
 "Get old outlier_detection job":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index 923a56395e8a4..3f5c68ab93d28 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53998"
 
   - do:
       index:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index 3af10eb33e286..4a98e6440c662 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,3 +1,8 @@
+setup:
+  - skip:
+      version: "all"
+      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53998"
+
 ---
 "Get old cluster outlier_detection job":
 

From ff067e1be437e6bd9bfa797f81cae8dde95f3dcf Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Tue, 24 Mar 2020 09:47:34 +0100
Subject: [PATCH 265/285] Fix BlobStoreIncrementalityIT (#54055)

The snapshot stats response list of snapshot statuses is not ordered according to the
given list of snapshot names so randomly we could mix up snapshot1 and snapshot2
when asserting on the stats.
Fixed by getting each snapshot's stats individually.

Closes #54034
---
 .../snapshots/BlobStoreIncrementalityIT.java   | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
index 21e7d3fe0cc2e..373760e301a56 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats;
-import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -46,7 +46,6 @@
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase {
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54034")
     public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedException, ExecutionException, IOException {
         internalCluster().startMasterOnlyNode();
         final String primaryNode = internalCluster().startDataOnlyNode();
@@ -174,10 +173,8 @@ public void testForceMergeCausesFullSnapshot() throws Exception {
         client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get();
 
         logger.info("--> asserting that the two snapshots refer to different files in the repository");
-        final SnapshotsStatusResponse response =
-            client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot2).get();
         final SnapshotStats secondSnapshotShardStatus =
-            response.getSnapshots().get(0).getIndices().get(indexName).getShards().get(0).getStats();
+            getStats(repo, snapshot2).getIndices().get(indexName).getShards().get(0).getStats();
         assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), greaterThan(0));
     }
 
@@ -191,19 +188,20 @@ private void assertCountInIndexThenDelete(String index, long expectedCount) thro
     private void assertTwoIdenticalShardSnapshots(String repo, String indexName, String snapshot1, String snapshot2) {
         logger.info(
             "--> asserting that snapshots [{}] and [{}] are referring to the same files in the repository", snapshot1, snapshot2);
-        final SnapshotsStatusResponse response =
-            client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot1, snapshot2).get();
-        final SnapshotStats firstSnapshotShardStatus =
-            response.getSnapshots().get(0).getIndices().get(indexName).getShards().get(0).getStats();
+        final SnapshotStats firstSnapshotShardStatus = getStats(repo, snapshot1).getIndices().get(indexName).getShards().get(0).getStats();
         final int totalFilesInShard = firstSnapshotShardStatus.getTotalFileCount();
         assertThat(totalFilesInShard, greaterThan(0));
         assertThat(firstSnapshotShardStatus.getIncrementalFileCount(), is(totalFilesInShard));
         final SnapshotStats secondSnapshotShardStatus =
-            response.getSnapshots().get(1).getIndices().get(indexName).getShards().get(0).getStats();
+            getStats(repo, snapshot2).getIndices().get(indexName).getShards().get(0).getStats();
         assertThat(secondSnapshotShardStatus.getTotalFileCount(), is(totalFilesInShard));
         assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), is(0));
     }
 
+    private SnapshotStatus getStats(String repository, String snapshot) {
+        return client().admin().cluster().prepareSnapshotStatus(repository).setSnapshots(snapshot).get().getSnapshots().get(0);
+    }
+
     private void ensureRestoreSingleShardSuccessfully(String repo, String indexName, String snapshot, String indexSuffix) {
         logger.info("--> restoring [{}]", snapshot);
         final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot)

From 9a64ae051446b9df305b282c00ab25e77ff28785 Mon Sep 17 00:00:00 2001
From: Albert Zaharovits <albert.zaharovits@elastic.co>
Date: Tue, 24 Mar 2020 11:09:46 +0200
Subject: [PATCH 266/285] Mute IndicesRequestCacheIT#testQueryRewrite

Relates #32827
---
 .../java/org/elasticsearch/indices/IndicesRequestCacheIT.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index 4274644217400..b76620c50db60 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -101,6 +101,7 @@ public void testCacheAggs() throws Exception {
     @TestIssueLogging(
         value = "org.elasticsearch.indices.IndicesRequestCache:TRACE",
         issueUrl = "https://github.com/elastic/elasticsearch/issues/32827")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32827")
     public void testQueryRewrite() throws Exception {
         Client client = client();
         assertAcked(client.admin().indices().prepareCreate("index").setMapping("s", "type=date")

From 5e637c410b952d7720084a2179b8588ad39cfdbb Mon Sep 17 00:00:00 2001
From: Christos Soulios <1561376+csoulios@users.noreply.github.com>
Date: Tue, 24 Mar 2020 11:32:28 +0200
Subject: [PATCH 267/285] Add supported type tests to min aggregation (#54021)

---
 .../metrics/MinAggregatorTests.java           | 29 +++++++------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
index 60a7bdc944d5e..cc58333d379bc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
@@ -27,7 +27,6 @@
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
@@ -45,7 +44,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.CheckedConsumer;
@@ -55,7 +53,6 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -84,10 +81,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.FieldContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 
@@ -278,22 +277,6 @@ public void testUnmappedWithMissingField() throws IOException {
         }, fieldType);
     }
 
-    public void testUnsupportedType() {
-        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("not_a_number");
-
-        MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
-        fieldType.setName("not_a_number");
-        fieldType.setHasDocValues(true);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
-                iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foo"))));
-            }, (Consumer<InternalMin>) min -> {
-                fail("Should have thrown exception");
-            }, fieldType));
-        assertEquals(e.getMessage(), "Expected numeric type on field [not_a_number], but got [keyword]");
-    }
-
     public void testBadMissingField() {
         MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("number").missing("not_a_number");
 
@@ -936,5 +919,13 @@ private <T extends AggregationBuilder, V extends InternalAggregation> void testC
         }
     }
 
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return Collections.singletonList(CoreValuesSourceType.NUMERIC);
+    }
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new MinAggregationBuilder("foo").field(fieldName);
+    }
 }

From 2537e02a7dbfc7805b9e0387ac225069cad1c68a Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Tue, 24 Mar 2020 09:35:29 +0000
Subject: [PATCH 268/285] Wildcard field - add normalizer support (#53851)

* Add support for normalisation to wildcard field

* Tidied imports

* Added docs about params

* Fix outdated error message

* Avoid normaliser butchering wildcard query special characters

* Fix broken test expectations

* Fix wrong toString method

* Address review comments - common method for normalising wildcard patterns and checkCompatibility

* Remove unused import
---
 .../reference/mapping/types/wildcard.asciidoc |  17 +++
 .../index/mapper/MappedFieldType.java         |   4 +-
 .../index/mapper/StringFieldType.java         |  57 +++++-----
 .../index/query/QueryBuilders.java            |   2 +-
 .../index/query/PrefixQueryBuilderTests.java  |   2 +-
 .../query/QueryStringQueryBuilderTests.java   |   2 +-
 .../test/wildcard/10_wildcard_basic.yml       |  66 +++++++++---
 .../wildcard/mapper/WildcardFieldMapper.java  | 101 +++++++++++++++++-
 8 files changed, 206 insertions(+), 45 deletions(-)

diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
index 9d01f4820a894..aa2de7db87afc 100644
--- a/docs/reference/mapping/types/wildcard.asciidoc
+++ b/docs/reference/mapping/types/wildcard.asciidoc
@@ -48,6 +48,23 @@ POST my_index/_doc/_search
 --------------------------------------------------
 
 
+[[wildcard-params]]
+==== Parameters for wildcard fields
+
+The following parameters are accepted by `wildcard` fields:
+
+[horizontal]
+
+<<ignore-above,`ignore_above`>>::
+
+    Do not index any string longer than this value.  Defaults to `2147483647`
+    so that all values would be accepted.
+
+<<normalizer,`normalizer`>>::
+
+    How to pre-process the value prior to indexing. Defaults to `null`,
+    meaning the value is kept as-is.
+
 ==== Limitations
 
 * `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 67bbdaf79ce76..1a80aa80a30b0 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -368,14 +368,14 @@ public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int
     }
 
     public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use prefix queries on keyword and text fields - not on [" + name
+        throw new QueryShardException(context, "Can only use prefix queries on keyword, text and wildcard fields - not on [" + name
             + "] which is of type [" + typeName() + "]");
     }
 
     public Query wildcardQuery(String value,
                                @Nullable MultiTermQuery.RewriteMethod method,
                                QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use wildcard queries on keyword and text fields - not on [" + name
+        throw new QueryShardException(context, "Can only use wildcard queries on keyword, text and wildcard fields - not on [" + name
             + "] which is of type [" + typeName() + "]");
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 05bf6b61d1de1..37c20b236ed56 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
@@ -93,6 +94,36 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
         return query;
     }
 
+    public static final String normalizeWildcardPattern(String fieldname, String value, Analyzer normalizer)  {
+        if (normalizer == null) {
+            return value;
+        }
+        // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
+        // is a char_filter that would otherwise remove them
+        Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
+        BytesRefBuilder sb = new BytesRefBuilder();
+        int last = 0;
+
+        while (wildcardMatcher.find()) {
+            if (wildcardMatcher.start() > 0) {
+                String chunk = value.substring(last, wildcardMatcher.start());
+
+                BytesRef normalized = normalizer.normalize(fieldname, chunk);
+                sb.append(normalized);
+            }
+            // append the matched group - without normalizing
+            sb.append(new BytesRef(wildcardMatcher.group()));
+
+            last = wildcardMatcher.end();
+        }
+        if (last < value.length()) {
+            String chunk = value.substring(last);
+            BytesRef normalized = normalizer.normalize(fieldname, chunk);
+            sb.append(normalized);
+        }
+        return sb.toBytesRef().utf8ToString();
+    }    
+    
     @Override
     public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
         failIfNotIndexed();
@@ -103,30 +134,8 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, Qu
 
         Term term;
         if (searchAnalyzer() != null) {
-            // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
-            // is a char_filter that would otherwise remove them
-            Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
-            BytesRefBuilder sb = new BytesRefBuilder();
-            int last = 0;
-
-            while (wildcardMatcher.find()) {
-                if (wildcardMatcher.start() > 0) {
-                    String chunk = value.substring(last, wildcardMatcher.start());
-
-                    BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
-                    sb.append(normalized);
-                }
-                // append the matched group - without normalizing
-                sb.append(new BytesRef(wildcardMatcher.group()));
-
-                last = wildcardMatcher.end();
-            }
-            if (last < value.length()) {
-                String chunk = value.substring(last);
-                BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
-                sb.append(normalized);
-            }
-            term = new Term(name(), sb.toBytesRef());
+            value = normalizeWildcardPattern(name(), value, searchAnalyzer());
+            term = new Term(name(), value);
         } else {
             term = new Term(name(), indexedValueForSearch(value));
         }
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index 3aa6118936e4d..bb9a007c60ca4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -239,7 +239,7 @@ public static RangeQueryBuilder rangeQuery(String name) {
      * which matches any single character. Note this query can be slow, as it
      * needs to iterate over many terms. In order to prevent extremely slow WildcardQueries,
      * a Wildcard term should not start with one of the wildcards {@code *} or
-     * {@code ?}.
+     * {@code ?}. (The wildcard field type however, is optimised for leading wildcards)
      *
      * @param name  The field name
      * @param query The wildcard query string
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 94596ffd6c58d..4253eaded3056 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -116,7 +116,7 @@ public void testNumeric() throws Exception {
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
                 () -> query.toQuery(context));
-        assertEquals("Can only use prefix queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+        assertEquals("Can only use prefix queries on keyword, text and wildcard fields - not on [mapped_int] which is of type [integer]",
                 e.getMessage());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 85453e774fe6e..7724e82a9e083 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -813,7 +813,7 @@ public void testPrefixNumeric() throws Exception {
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
                 () -> query.toQuery(context));
-        assertEquals("Can only use prefix queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+        assertEquals("Can only use prefix queries on keyword, text and wildcard fields - not on [mapped_int] which is of type [integer]",
                 e.getMessage());
         query.lenient(true);
         query.toQuery(context); // no exception
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
index c67a79c8218da..486082ef3ab59 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.9.99"
-      reason: "wildcard fields were added from 8.0"
+      version: " - 7.6.99"
+      reason: "wildcard fields were added from 7.7"
 
   - do:
       indices.create:
@@ -10,10 +10,20 @@ setup:
         body:
           settings:
             number_of_replicas: 0
+            analysis:
+              normalizer:
+                lowercase:
+                  type: custom
+                  char_filter: []
+                  filter: ["lowercase"]
           mappings:
             properties:
               my_wildcard:
                  type: wildcard
+                 normalizer: lowercase
+                 fields:
+                   case_sensitive:
+                     type: wildcard
   - do:
       index:
         index: test-index
@@ -26,6 +36,12 @@ setup:
         id: 2
         body:
           my_wildcard: goodbye world
+  - do:
+      index:
+        index: test-index
+        id: 3
+        body:
+          my_wildcard: cAsE iNsEnSiTiVe World
 
   - do:
       indices.refresh: {}
@@ -80,6 +96,31 @@ setup:
               my_wildcard: {value: "*ello worl*" }
 
 
+  - match: {hits.total.value: 1}
+---
+"Case insensitive query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*Worl*" }
+
+
+  - match: {hits.total.value: 3}
+
+---
+"Case sensitive query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard.case_sensitive: {value: "*Worl*" }
+
+
   - match: {hits.total.value: 1}
 
 ---
@@ -93,7 +134,7 @@ setup:
               my_wildcard: {value: "*ld" }
 
 
-  - match: {hits.total.value: 2}
+  - match: {hits.total.value: 3}
 
 ---
 "Long suffix query":
@@ -188,8 +229,8 @@ setup:
               terms: {field: "my_wildcard" }
 
 
-  - match: {hits.total.value: 2}
-  - length: { aggregations.top_vals.buckets: 2 }
+  - match: {hits.total.value: 3}
+  - length: { aggregations.top_vals.buckets: 3 }
 
 ---
 "Sort works":
@@ -199,10 +240,11 @@ setup:
           track_total_hits: true
           sort: [ { "my_wildcard": "desc" } ]
 
-  - match: { hits.total.value: 2 }
-  - length: { hits.hits: 2 }
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
   - match: { hits.hits.0._id: "1" }
   - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "3" }
 
   - do:
       search:
@@ -210,9 +252,9 @@ setup:
           track_total_hits: true
           sort: [ { "my_wildcard": "asc" } ]
 
-  - match: { hits.total.value: 2 }
-  - length: { hits.hits: 2 }
-  - match: { hits.hits.0._id: "2" }
-  - match: { hits.hits.1._id: "1" }
-
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "1" }
 
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index e489d8a35bb9f..568e4f502dbe0 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -53,6 +54,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -64,6 +66,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
@@ -100,6 +103,9 @@ public static class Defaults {
 
     public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
         protected int ignoreAbove = Defaults.IGNORE_ABOVE;
+        private IndexAnalyzers indexAnalyzers;
+        private String normalizerName;
+        
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -164,10 +170,23 @@ protected void setupFieldType(BuilderContext context) {
         public WildcardFieldType fieldType() {
             return (WildcardFieldType) super.fieldType();
         }
+        
+        public Builder normalizer(IndexAnalyzers indexAnalyzers, String name) {
+            this.indexAnalyzers = indexAnalyzers;
+            this.normalizerName = name;
+            return builder;
+        }        
 
         @Override
         public WildcardFieldMapper build(BuilderContext context) {
-            setupFieldType(context);            
+            setupFieldType(context);   
+            if (normalizerName != null) {
+                NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
+                if (normalizer == null) {
+                    throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                }
+                fieldType().setNormalizer(normalizer);
+            }            
             return new WildcardFieldMapper(
                     name, fieldType, defaultFieldType, ignoreAbove, 
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
@@ -188,6 +207,11 @@ public static class TypeParser implements Mapper.TypeParser {
                 if (propName.equals("ignore_above")) {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
+                } else if (propName.equals("normalizer")) {
+                    if (propNode != null) {
+                        builder.normalizer(parserContext.getIndexAnalyzers(), propNode.toString());
+                    }
+                    iterator.remove();
                 }
             }            
             
@@ -198,6 +222,8 @@ public static class TypeParser implements Mapper.TypeParser {
      public static final char TOKEN_START_OR_END_CHAR = 0;    
     
      public static final class WildcardFieldType extends MappedFieldType {
+         
+        private NamedAnalyzer normalizer = null;
 
         public WildcardFieldType() {            
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
@@ -206,6 +232,7 @@ public WildcardFieldType() {
 
         protected WildcardFieldType(WildcardFieldType ref) {
             super(ref);
+            this.normalizer = ref.normalizer;
         }
 
         public WildcardFieldType clone() {
@@ -213,7 +240,39 @@ public WildcardFieldType clone() {
             return result;
         }
         
-                
+
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            WildcardFieldType other = (WildcardFieldType) o;
+            return Objects.equals(normalizer, other.normalizer);
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * super.hashCode() + Objects.hash(normalizer);
+        }       
+        
+        private NamedAnalyzer normalizer() {
+            return normalizer;
+        }
+
+        public void setNormalizer(NamedAnalyzer normalizer) {
+            checkIfFrozen();
+            this.normalizer = normalizer;
+        }                
+
+        @Override
+        public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts) {
+            super.checkCompatibility(otherFT, conflicts);
+            WildcardFieldType other = (WildcardFieldType) otherFT;
+            if (Objects.equals(normalizer, other.normalizer) == false) {
+                conflicts.add("mapper [" + name() + "] has different [normalizer]");
+            }
+        }
+        
         // Holds parsed information about the wildcard pattern
         static class PatternStructure {
             boolean openStart, openEnd, hasSymbols;            
@@ -327,6 +386,9 @@ public boolean equals(Object obj) {
 
         @Override
         public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
+            if (normalizer != null) {
+                wildcardPattern = StringFieldType.normalizeWildcardPattern(name(), wildcardPattern, normalizer);
+            }
             PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
             ArrayList<String> tokens = new ArrayList<>();
 
@@ -467,7 +529,32 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
                         CircuitBreakerService breakerService, MapperService mapperService) {
                     return new WildcardBytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
                 }};
-        }        
+        }
+
+
+        String normalize(String value) throws IOException {
+            if (normalizer != null) {
+                try (TokenStream ts = normalizer.tokenStream(name(), value)) {
+                    final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+                    ts.reset();
+                    if (ts.incrementToken() == false) {
+                      throw new IllegalStateException("The normalization token stream is "
+                          + "expected to produce exactly 1 token, but got 0 for analyzer "
+                          + normalizer + " and input \"" + value + "\"");
+                    }
+                    final String newValue = termAtt.toString();
+                    if (ts.incrementToken()) {
+                      throw new IllegalStateException("The normalization token stream is "
+                          + "expected to produce exactly 1 token, but got 2+ for analyzer "
+                          + normalizer + " and input \"" + value + "\"");
+                    }
+                    ts.end();
+                    return newValue;
+                }
+            }
+            return value;
+        }       
+        
     }
      
     static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
@@ -521,6 +608,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
+        if (fieldType().normalizer() != null) {
+            builder.field("normalizer", fieldType().normalizer().name());
+        } else if (includeDefaults) {
+            builder.nullField("normalizer");
+        }        
     }
     
     @Override
@@ -544,10 +636,11 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
     // For internal use by Lucene only - used to define ngram index
     final MappedFieldType ngramFieldType;
     
-    void createFields(String value, Document parseDoc, List<IndexableField>fields) {
+    void createFields(String value, Document parseDoc, List<IndexableField>fields) throws IOException {
         if (value == null || value.length() > ignoreAbove) {
             return;
         }
+        value = fieldType().normalize(value);
         String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
         Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
         fields.add(ngramField);

From 077ffa235c013e80832789d330bdf580e081481b Mon Sep 17 00:00:00 2001
From: Hendrik Muhs <hendrik.muhs@elastic.co>
Date: Tue, 24 Mar 2020 10:59:28 +0100
Subject: [PATCH 269/285] [Transform ] reenable BWC after backport (#54061)

finish and reenable BWC after backport of #54027
---
 build.gradle                                                  | 4 ++--
 .../elasticsearch/xpack/core/indexing/IndexerJobStats.java    | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.gradle b/build.gradle
index 792672d6021ff..a23614bafa975 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/54027" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
index 57aac7f839e4b..927c71ce8e10c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
@@ -73,7 +73,7 @@ public IndexerJobStats(StreamInput in) throws IOException {
         this.indexFailures = in.readVLong();
         this.searchFailures = in.readVLong();
 
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo V_7_7_0
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
             this.processingTime = in.readVLong();
             this.processingTotal = in.readVLong();
         }
@@ -194,7 +194,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(searchTotal);
         out.writeVLong(indexFailures);
         out.writeVLong(searchFailures);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
             out.writeVLong(processingTime);
             out.writeVLong(processingTotal);
         }

From cbe063a074787960d198c12f9f93886745f01618 Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Tue, 24 Mar 2020 10:01:13 +0000
Subject: [PATCH 270/285] [ML] Introduce a "starting" datafeed state for lazy
 jobs (#53918)

It is possible for ML jobs to open lazily if the "allow_lazy_open"
option in the job config is set to true.  Such jobs wait in the
"opening" state until a node has sufficient capacity to run them.

This commit fixes the bug that prevented datafeeds for jobs lazily
waiting assignment from being started.  The state of such datafeeds
is "starting", and they can be stopped by the stop datafeed API
while in this state with or without force.

Fixes #53763
---
 docs/reference/ml/ml-shared.asciidoc          |  3 +
 .../elasticsearch/xpack/core/ml/MlTasks.java  | 12 ++--
 .../xpack/core/ml/MlTasksTests.java           |  3 +-
 .../ml/action/TransportOpenJobAction.java     |  6 +-
 .../action/TransportStartDatafeedAction.java  | 17 ++++--
 .../action/TransportStopDatafeedAction.java   |  8 +--
 .../ml/datafeed/DatafeedNodeSelector.java     | 10 +++-
 .../integration/BasicDistributedJobsIT.java   | 55 +++++++++++++++++++
 .../xpack/ml/support/BaseMlIntegTestCase.java |  5 ++
 .../rest-api-spec/test/ml/jobs_crud.yml       |  2 +-
 10 files changed, 97 insertions(+), 24 deletions(-)

diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index 16c9f486d8d63..75dc0a946d636 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -1388,7 +1388,10 @@ tag::state-datafeed[]
 The status of the {dfeed}, which can be one of the following values:
 +
 --
+* `starting`: The {dfeed} has been requested to start but has not yet started.
 * `started`: The {dfeed} is actively receiving data.
+* `stopping`: The {dfeed} has been requested to stop gracefully and is
+completing its final action.
 * `stopped`: The {dfeed} is stopped and will not receive data until it is
 re-started.
 --
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
index 5be6ffe2878d2..fa20e805cc589 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
@@ -132,14 +132,18 @@ public static JobState getJobStateModifiedForReassignments(@Nullable PersistentT
 
     public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
         PersistentTasksCustomMetaData.PersistentTask<?> task = getDatafeedTask(datafeedId, tasks);
-        // TODO: report (task != null && task.getState() == null) as STARTING in version 8, and fix side effects
-        if (task != null && task.getState() != null) {
-            return (DatafeedState) task.getState();
-        } else {
+        if (task == null) {
             // If we haven't started a datafeed then there will be no persistent task,
             // which is the same as if the datafeed was't started
             return DatafeedState.STOPPED;
         }
+        DatafeedState taskState = (DatafeedState) task.getState();
+        if (taskState == null) {
+            // If we haven't set a state yet then the task has never been assigned, so
+            // report that it's starting
+            return DatafeedState.STARTING;
+        }
+        return taskState;
     }
 
     public static DataFrameAnalyticsState getDataFrameAnalyticsState(String analyticsId, @Nullable PersistentTasksCustomMetaData tasks) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
index 902df8cd44fa2..a0b7d6a035f75 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
@@ -55,7 +55,8 @@ public void testGetDatefeedState() {
         tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME,
                 new StartDatafeedAction.DatafeedParams("foo", 0L),
                 new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
-        assertEquals(DatafeedState.STOPPED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
+        // A task with no state means the datafeed is starting
+        assertEquals(DatafeedState.STARTING, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
 
         tasksBuilder.updateTaskState(MlTasks.datafeedTaskId("foo"), DatafeedState.STARTED);
         assertEquals(DatafeedState.STARTED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index 44bf8c483dfd9..6fc8000b0a559 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -289,7 +289,7 @@ public void onResponse(PersistentTasksCustomMetaData.PersistentTask<OpenJobActio
                         listener.onFailure(predicate.exception);
                     }
                 } else {
-                    listener.onResponse(new AcknowledgedResponse(predicate.opened));
+                    listener.onResponse(new AcknowledgedResponse(true));
                 }
             }
 
@@ -537,9 +537,8 @@ void closeJob(String reason) {
      * Important: the methods of this class must NOT throw exceptions.  If they did then the callers
      * of endpoints waiting for a condition tested by this predicate would never get a response.
      */
-    private class JobPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
+    private static class JobPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
 
-        private volatile boolean opened;
         private volatile Exception exception;
         private volatile boolean shouldCancel;
 
@@ -586,7 +585,6 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 case CLOSED:
                     return false;
                 case OPENED:
-                    opened = true;
                     return true;
                 case CLOSING:
                     exception = ExceptionsHelper.conflictStatusException("The job has been " + JobState.CLOSED + " while waiting to be "
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index 40c4cea806f6f..28dbd4ba4d683 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -477,7 +477,7 @@ public void isolate() {
      * Important: the methods of this class must NOT throw exceptions.  If they did then the callers
      * of endpoints waiting for a condition tested by this predicate would never get a response.
      */
-    private class DatafeedPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
+    private static class DatafeedPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
 
         private volatile Exception exception;
 
@@ -487,12 +487,17 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 return false;
             }
             PersistentTasksCustomMetaData.Assignment assignment = persistentTask.getAssignment();
-            if (assignment != null && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false &&
-                    assignment.isAssigned() == false) {
-                // Assignment has failed despite passing our "fast fail" validation
-                exception = new ElasticsearchStatusException("Could not start datafeed, allocation explanation [" +
+            if (assignment != null) {
+                // This means we are awaiting the datafeed's job to be assigned to a node
+                if (assignment.equals(DatafeedNodeSelector.AWAITING_JOB_ASSIGNMENT)) {
+                    return true;
+                }
+                if (assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false && assignment.isAssigned() == false) {
+                    // Assignment has failed despite passing our "fast fail" validation
+                    exception = new ElasticsearchStatusException("Could not start datafeed, allocation explanation [" +
                         assignment.getExplanation() + "]", RestStatus.TOO_MANY_REQUESTS);
-                return true;
+                    return true;
+                }
             }
             DatafeedState datafeedState = (DatafeedState) persistentTask.getState();
             return datafeedState == DatafeedState.STARTED;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
index 903733e09970c..db94905a1bdb5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
@@ -93,14 +93,8 @@ private static void addDatafeedTaskIdAccordingToState(String datafeedId,
                                                           List<String> stoppingDatafeedIds,
                                                           List<String> notStoppedDatafeedIds) {
         switch (datafeedState) {
+            // Treat STARTING like STARTED for stop API behaviour.
             case STARTING:
-                // The STARTING state is not used anywhere at present, so this should never happen.
-                // At present datafeeds that have a persistent task that hasn't yet been assigned
-                // a state are reported as STOPPED (which is not great).  It could be considered a
-                // breaking change to introduce the STARTING state though, so let's aim to do it in
-                // version 8.  Also consider treating STARTING like STARTED for stop API behaviour.
-                notStoppedDatafeedIds.add(datafeedId);
-                break;
             case STARTED:
                 startedDatafeedIds.add(datafeedId);
                 notStoppedDatafeedIds.add(datafeedId);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
index 03c8272a979d8..a4c9748dd64c7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
@@ -32,6 +32,9 @@ public class DatafeedNodeSelector {
 
     private static final Logger LOGGER = LogManager.getLogger(DatafeedNodeSelector.class);
 
+    public static final PersistentTasksCustomMetaData.Assignment AWAITING_JOB_ASSIGNMENT =
+        new PersistentTasksCustomMetaData.Assignment(null, "datafeed awaiting job assignment.");
+
     private final String datafeedId;
     private final String jobId;
     private final List<String> datafeedIndices;
@@ -76,9 +79,14 @@ public PersistentTasksCustomMetaData.Assignment selectNode() {
 
         AssignmentFailure assignmentFailure = checkAssignment();
         if (assignmentFailure == null) {
-            return new PersistentTasksCustomMetaData.Assignment(jobTask.getExecutorNode(), "");
+            String jobNode = jobTask.getExecutorNode();
+            if (jobNode == null) {
+                return AWAITING_JOB_ASSIGNMENT;
+            }
+            return new PersistentTasksCustomMetaData.Assignment(jobNode, "");
         }
         LOGGER.debug(assignmentFailure.reason);
+        assert assignmentFailure.reason.isEmpty() == false;
         return new PersistentTasksCustomMetaData.Assignment(null, assignmentFailure.reason);
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 23275958b3c10..e0370066c047c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
+import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
@@ -424,6 +425,60 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception {
         assertBusy(() -> assertJobTask(jobId, JobState.OPENED, true));
     }
 
+    public void testCloseUnassignedLazyJobAndDatafeed() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(3);
+        ensureStableCluster(3);
+
+        String jobId = "test-lazy-stop";
+        String datafeedId = jobId + "-datafeed";
+        // Assume the test machine won't have space to assign a 2TB job
+        Job.Builder job = createJob(jobId, new ByteSizeValue(2, ByteSizeUnit.TB), true);
+        PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
+        client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
+
+        client().admin().indices().prepareCreate("data").setMapping("time", "type=date").get();
+
+        DatafeedConfig config = createDatafeed(datafeedId, jobId, Collections.singletonList("data"));
+        PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(config);
+        client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).actionGet();
+
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
+        OpenJobAction.Request openJobRequest = new OpenJobAction.Request(jobId);
+        client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
+
+        // Job state should be opening because it won't fit anyway, but is allowed to open lazily
+        GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(jobId);
+        GetJobsStatsAction.Response jobStatsResponse = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet();
+        assertEquals(JobState.OPENING, jobStatsResponse.getResponse().results().get(0).getState());
+
+        StartDatafeedAction.Request startDataFeedRequest = new StartDatafeedAction.Request(config.getId(), 0L);
+        client().execute(StartDatafeedAction.INSTANCE, startDataFeedRequest).actionGet();
+
+        // Datafeed state should be starting while it waits for job assignment
+        GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(datafeedId);
+        GetDatafeedsStatsAction.Response datafeedStatsResponse =
+            client().execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest).actionGet();
+        assertEquals(DatafeedState.STARTING, datafeedStatsResponse.getResponse().results().get(0).getDatafeedState());
+
+        // A starting datafeed can be stopped normally or by force
+        StopDatafeedAction.Request stopDatafeedRequest = new StopDatafeedAction.Request(datafeedId);
+        stopDatafeedRequest.setForce(randomBoolean());
+        StopDatafeedAction.Response stopDatafeedResponse = client().execute(StopDatafeedAction.INSTANCE, stopDatafeedRequest).actionGet();
+        assertTrue(stopDatafeedResponse.isStopped());
+
+        datafeedStatsResponse = client().execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest).actionGet();
+        assertEquals(DatafeedState.STOPPED, datafeedStatsResponse.getResponse().results().get(0).getDatafeedState());
+
+        // An opening job can also be stopped normally or by force
+        CloseJobAction.Request closeJobRequest = new CloseJobAction.Request(jobId);
+        closeJobRequest.setForce(randomBoolean());
+        CloseJobAction.Response closeJobResponse = client().execute(CloseJobAction.INSTANCE, closeJobRequest).actionGet();
+        assertTrue(closeJobResponse.isClosed());
+
+        jobStatsResponse = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet();
+        assertEquals(JobState.CLOSED, jobStatsResponse.getResponse().results().get(0).getState());
+    }
+
     private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) {
         ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
         PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
index 887cd17eb6b50..6ae0d3fbd57be 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
@@ -127,6 +127,10 @@ protected Job.Builder createJob(String id) {
     }
 
     protected Job.Builder createJob(String id, ByteSizeValue modelMemoryLimit) {
+        return createJob(id, modelMemoryLimit, false);
+    }
+
+    protected Job.Builder createJob(String id, ByteSizeValue modelMemoryLimit, boolean allowLazyOpen) {
         DataDescription.Builder dataDescription = new DataDescription.Builder();
         dataDescription.setFormat(DataDescription.DataFormat.XCONTENT);
         dataDescription.setTimeFormat(DataDescription.EPOCH_MS);
@@ -141,6 +145,7 @@ protected Job.Builder createJob(String id, ByteSizeValue modelMemoryLimit) {
         }
         builder.setAnalysisConfig(analysisConfig);
         builder.setDataDescription(dataDescription);
+        builder.setAllowLazyOpen(allowLazyOpen);
         return builder;
     }
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
index 7b365dbe92dc2..c566bfed6284c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -142,7 +142,7 @@
   - do:
       ml.open_job:
         job_id: job-model-memory-limit-as-string
-  - match: { opened: false }
+  - match: { opened: true }
 
   - do:
       headers:

From 4cf1d1ad059eeffbe3d7bf9d9782fbaeec0e3880 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Tue, 24 Mar 2020 12:09:25 +0200
Subject: [PATCH 271/285] [ML] No refresh on indexing DFA stats (#53977)

When we index data frame analytics stats docs we do not
need to refresh immediately.
---
 .../process/AnalyticsProcessManager.java         | 16 ++++++++++------
 .../xpack/ml/dataframe/stats/StatsPersister.java |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
index c1f09ff3fb34f..cc3852f32f051 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -178,7 +179,7 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
             processContext.setFailureReason(resultProcessor.getFailure());
 
             refreshDest(config);
-            refreshStateIndex(config.getId());
+            refreshIndices(config.getId());
             LOGGER.info("[{}] Result processor has completed", config.getId());
         } catch (Exception e) {
             if (task.isStopping()) {
@@ -316,12 +317,15 @@ private void refreshDest(DataFrameAnalyticsConfig config) {
             () -> client.execute(RefreshAction.INSTANCE, new RefreshRequest(config.getDest().getIndex())).actionGet());
     }
 
-    private void refreshStateIndex(String jobId) {
-        String indexName = AnomalyDetectorsIndex.jobStateIndexPattern();
-        LOGGER.debug("[{}] Refresh index {}", jobId, indexName);
-
-        RefreshRequest refreshRequest = new RefreshRequest(indexName);
+    private void refreshIndices(String jobId) {
+        RefreshRequest refreshRequest = new RefreshRequest(
+            AnomalyDetectorsIndex.jobStateIndexPattern(),
+            MlStatsIndex.indexPattern()
+        );
         refreshRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+
+        LOGGER.debug("[{}] Refreshing indices {}", jobId, Arrays.toString(refreshRequest.indices()));
+
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             client.admin().indices().refresh(refreshRequest).actionGet();
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
index eeb8924928ce4..e553ac1810729 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
@@ -47,7 +47,7 @@ public void persistWithRetry(ToXContentObject result, Function<String, String> d
                 MlStatsIndex.writeAlias(),
                 result,
                 new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
-                WriteRequest.RefreshPolicy.IMMEDIATE,
+                WriteRequest.RefreshPolicy.NONE,
                 docIdSupplier.apply(jobId),
                 () -> isCancelled == false,
                 errorMsg -> auditor.error(jobId,

From 3ef69ad29b051dbaa3a66849d09451aafdaf4675 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Tue, 24 Mar 2020 12:33:28 +0200
Subject: [PATCH 272/285] [ML] Unmute DFA bwc tests after backport of #53998
 (#54063)

Relates #53998
---
 .../core/ml/action/GetDataFrameAnalyticsStatsAction.java     | 4 ++--
 .../test/mixed_cluster/90_ml_data_frame_analytics_crud.yml   | 5 -----
 .../test/old_cluster/90_ml_data_frame_analytics_crud.yml     | 3 ---
 .../upgraded_cluster/90_ml_data_frame_analytics_crud.yml     | 5 -----
 4 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
index 3f65a917a7427..e37ccbbefc5b1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
@@ -203,7 +203,7 @@ public Stats(StreamInput in) throws IOException {
                 } else {
                     progress = in.readList(PhaseProgress::new);
                 }
-                if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
                     dataCounts = in.readOptionalWriteable(DataCounts::new);
                 } else {
                     dataCounts = null;
@@ -349,7 +349,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 } else {
                     out.writeList(progress);
                 }
-                if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                     out.writeOptionalWriteable(dataCounts);
                 }
                 if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
index 38c78803b6f61..5049f3360a42e 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53998"
-
 ---
 "Get old outlier_detection job":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
index 3f5c68ab93d28..923a56395e8a4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,7 +1,4 @@
 setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53998"
 
   - do:
       index:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
index 4a98e6440c662..3af10eb33e286 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml
@@ -1,8 +1,3 @@
-setup:
-  - skip:
-      version: "all"
-      reason:  "Until backport of https://github.com/elastic/elasticsearch/pull/53998"
-
 ---
 "Get old cluster outlier_detection job":
 

From aed8ce7cda125c10e7b9baa9d86f8f32b57141d7 Mon Sep 17 00:00:00 2001
From: Dan Hermann <danhermann@users.noreply.github.com>
Date: Tue, 24 Mar 2020 06:08:48 -0500
Subject: [PATCH 273/285] Cluster state and CRUD operations for data streams
 (#53877)

---
 .../test/indices.data_stream/10_basic.yml     |  25 ++-
 .../datastream/CreateDataStreamAction.java    |  57 +++++-
 .../datastream/DeleteDataStreamAction.java    |  66 +++++-
 .../datastream/GetDataStreamsAction.java      |  32 ++-
 .../elasticsearch/cluster/ClusterModule.java  |   4 +
 .../cluster/metadata/DataStreamMetadata.java  | 188 ++++++++++++++++++
 .../cluster/metadata/MetaData.java            |  32 +++
 .../CreateDataStreamRequestTests.java         |  47 +++++
 .../DeleteDataStreamRequestTests.java         |  46 +++++
 .../GetDataStreamsRequestTests.java           |  36 ++++
 .../metadata/DataStreamMetadataTests.java     |  59 ++++++
 .../cluster/metadata/DataStreamTests.java     |  17 +-
 .../cluster/metadata/MetaDataTests.java       |   1 +
 .../metadata/ToAndFromJsonMetaDataTests.java  |  12 ++
 14 files changed, 598 insertions(+), 24 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
index 035c20c1600bf..d21abfc11c754 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -1,26 +1,33 @@
 ---
-"Test stubs":
+"Create data stream":
   - skip:
-      version: " - 7.6.99"
-      reason: only available in 7.7+
+      version: "all"
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/54022"
 
   - do:
       indices.create_data_stream:
-        name: data-stream2
+        name: simple-data-stream1
         body:
           timestamp_field: "@timestamp"
   - is_true: acknowledged
 
+  - do:
+      indices.create_data_stream:
+        name: simple-data-stream2
+        body:
+          timestamp_field: "@timestamp2"
+  - is_true: acknowledged
+
   - do:
       indices.get_data_streams: {}
-  - match: { 0.name: my_data_stream1 }
+  - match: { 0.name: simple-data-stream1 }
   - match: { 0.timestamp_field: '@timestamp' }
-  - match: { 0.indices: ['my_data_stream1-000000'] }
-  - match: { 1.name: my_data_stream2 }
-  - match: { 1.timestamp_field: '@timestamp' }
+  - match: { 0.indices: [] }
+  - match: { 1.name: simple-data-stream2 }
+  - match: { 1.timestamp_field: '@timestamp2' }
   - match: { 1.indices: [] }
 
   - do:
       indices.delete_data_stream:
-        name: data-stream2
+        name: simple-data-stream2
   - is_true: acknowledged
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
index df6e829a28af4..37f190691f33a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
@@ -18,30 +18,42 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Objects;
 
 public class CreateDataStreamAction extends ActionType<AcknowledgedResponse> {
 
+    private static final Logger logger = LogManager.getLogger(CreateDataStreamAction.class);
+
     public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction();
     public static final String NAME = "indices:admin/data_stream/create";
 
@@ -64,7 +76,14 @@ public void setTimestampFieldName(String timestampFieldName) {
 
         @Override
         public ActionRequestValidationException validate() {
-            return null;
+            ActionRequestValidationException validationException = null;
+            if (Strings.hasText(name) == false) {
+                validationException = ValidateActions.addValidationError("name is missing", validationException);
+            }
+            if (Strings.hasText(timestampFieldName) == false) {
+                validationException = ValidateActions.addValidationError("timestamp field name is missing", validationException);
+            }
+            return validationException;
         }
 
         public Request(StreamInput in) throws IOException {
@@ -116,7 +135,41 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state,
                                        ActionListener<AcknowledgedResponse> listener) throws Exception {
-            listener.onResponse(new AcknowledgedResponse(true));
+            clusterService.submitStateUpdateTask("create-data-stream [" + request.name + "]",
+                new ClusterStateUpdateTask(Priority.HIGH) {
+
+                    @Override
+                    public TimeValue timeout() {
+                        return request.masterNodeTimeout();
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        listener.onFailure(e);
+                    }
+
+                    @Override
+                    public ClusterState execute(ClusterState currentState) throws Exception {
+                        return createDataStream(currentState, request);
+                    }
+
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        listener.onResponse(new AcknowledgedResponse(true));
+                    }
+                });
+        }
+
+        static ClusterState createDataStream(ClusterState currentState, Request request) {
+            if (currentState.metaData().dataStreams().containsKey(request.name)) {
+                throw new IllegalArgumentException("data_stream [" + request.name + "] already exists");
+            }
+
+            MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(
+                new DataStream(request.name, request.timestampFieldName, Collections.emptyList()));
+
+            logger.info("adding data stream [{}]", request.name);
+            return ClusterState.builder(currentState).metaData(builder).build();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
index 20a2ba4aa2cd6..91444ef64320b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
@@ -18,30 +18,44 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 
 public class DeleteDataStreamAction extends ActionType<AcknowledgedResponse> {
 
+    private static final Logger logger = LogManager.getLogger(DeleteDataStreamAction.class);
+
     public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction();
     public static final String NAME = "indices:admin/data_stream/delete";
 
@@ -59,7 +73,11 @@ public Request(String name) {
 
         @Override
         public ActionRequestValidationException validate() {
-            return null;
+            ActionRequestValidationException validationException = null;
+            if (Strings.hasText(name) == false) {
+                validationException = ValidateActions.addValidationError("name is missing", validationException);
+            }
+            return validationException;
         }
 
         public Request(StreamInput in) throws IOException {
@@ -108,7 +126,51 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state,
                                        ActionListener<AcknowledgedResponse> listener) throws Exception {
-            listener.onResponse(new AcknowledgedResponse(true));
+            clusterService.submitStateUpdateTask("remove-data-stream [" + request.name + "]", new ClusterStateUpdateTask(Priority.HIGH) {
+
+                @Override
+                public TimeValue timeout() {
+                    return request.masterNodeTimeout();
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return removeDataStream(currentState, request);
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    listener.onResponse(new AcknowledgedResponse(true));
+                }
+            });
+        }
+
+        static ClusterState removeDataStream(ClusterState currentState, Request request) {
+            Set<String> dataStreams = new HashSet<>();
+            for (String dataStreamName : currentState.metaData().dataStreams().keySet()) {
+                if (Regex.simpleMatch(request.name, dataStreamName)) {
+                    dataStreams.add(dataStreamName);
+                }
+            }
+            if (dataStreams.isEmpty()) {
+                // if a match-all pattern was specified and no data streams were found because none exist, do not
+                // fail with data stream missing exception
+                if (Regex.isMatchAllPattern(request.name)) {
+                    return currentState;
+                }
+                throw new ResourceNotFoundException("data_streams matching [" + request.name + "] not found");
+            }
+            MetaData.Builder metaData = MetaData.builder(currentState.metaData());
+            for (String dataStreamName : dataStreams) {
+                logger.info("removing data stream [{}]", dataStreamName);
+                metaData.removeDataStream(dataStreamName);
+            }
+            return ClusterState.builder(currentState).metaData(metaData).build();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
index 1549f056e811f..ed2411401aeb6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
@@ -41,8 +42,10 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class GetDataStreamsAction extends ActionType<GetDataStreamsAction.Response> {
@@ -154,11 +157,30 @@ protected Response read(StreamInput in) throws IOException {
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state,
                                        ActionListener<Response> listener) throws Exception {
-            List<DataStream> dataStreams = List.of(
-                new DataStream("my_data_stream1", "@timestamp", List.of("my_data_stream1-000000")),
-                new DataStream("my_data_stream2", "@timestamp", List.of())
-            );
-            listener.onResponse(new Response(dataStreams));
+            listener.onResponse(new Response(getDataStreams(state, request)));
+        }
+
+        static List<DataStream> getDataStreams(ClusterState clusterState, Request request) {
+            Map<String, DataStream> dataStreams = clusterState.metaData().dataStreams();
+
+            // return all data streams if no name was specified
+            if (request.names.length == 0) {
+                return new ArrayList<>(dataStreams.values());
+            }
+
+            final List<DataStream> results = new ArrayList<>();
+            for (String name : request.names) {
+                if (Regex.isSimpleMatchPattern(name)) {
+                    for (Map.Entry<String, DataStream> entry : dataStreams.entrySet()) {
+                        if (Regex.simpleMatch(name, entry.getKey())) {
+                            results.add(entry.getValue());
+                        }
+                    }
+                } else if (dataStreams.containsKey(name)) {
+                    results.add(dataStreams.get(name));
+                }
+            }
+            return results;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index f52b32f3ccf67..6dfe07e5c20aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
+import org.elasticsearch.cluster.metadata.DataStreamMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateV2Metadata;
@@ -133,6 +134,7 @@ public static List<Entry> getNamedWriteables() {
             ComponentTemplateMetadata::readDiffFrom);
         registerMetaDataCustom(entries, IndexTemplateV2Metadata.TYPE, IndexTemplateV2Metadata::new,
             IndexTemplateV2Metadata::readDiffFrom);
+        registerMetaDataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
@@ -155,6 +157,8 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
             ComponentTemplateMetadata::fromXContent));
         entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexTemplateV2Metadata.TYPE),
             IndexTemplateV2Metadata::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(DataStreamMetadata.TYPE),
+            DataStreamMetadata::fromXContent));
         return entries;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
new file mode 100644
index 0000000000000..5731c1353f989
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Custom {@link MetaData} implementation for storing a map of {@link DataStream}s and their names.
+ */
+public class DataStreamMetadata implements MetaData.Custom {
+
+    public static final String TYPE = "data_stream";
+    private static final ParseField DATA_STREAM = new ParseField("data_stream");
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<DataStreamMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
+        a -> new DataStreamMetadata((Map<String, DataStream>) a[0]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
+            Map<String, DataStream> dataStreams = new HashMap<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                String name = p.currentName();
+                dataStreams.put(name, DataStream.fromXContent(p));
+            }
+            return dataStreams;
+        }, DATA_STREAM);
+    }
+
+    private final Map<String, DataStream> dataStreams;
+
+    public DataStreamMetadata(Map<String, DataStream> dataStreams) {
+        this.dataStreams = dataStreams;
+    }
+
+    public DataStreamMetadata(StreamInput in) throws IOException {
+        this.dataStreams = in.readMap(StreamInput::readString, DataStream::new);
+    }
+
+    public Map<String, DataStream> dataStreams() {
+        return this.dataStreams;
+    }
+
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new DataStreamMetadata.DataStreamMetadataDiff((DataStreamMetadata) before, this);
+    }
+
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new DataStreamMetadata.DataStreamMetadataDiff(in);
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_8_0_0;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(this.dataStreams, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
+    }
+
+    public static DataStreamMetadata fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(DATA_STREAM.getPreferredName());
+        for (Map.Entry<String, DataStream> dataStream : dataStreams.entrySet()) {
+            builder.field(dataStream.getKey(), dataStream.getValue());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.dataStreams);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        DataStreamMetadata other = (DataStreamMetadata) obj;
+        return Objects.equals(this.dataStreams, other.dataStreams);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private final Map<String, DataStream> dataStreams = new HashMap<>();
+
+        public Builder putDataStream(DataStream dataStream) {
+            dataStreams.put(dataStream.getName(), dataStream);
+            return this;
+        }
+
+        public DataStreamMetadata build() {
+            return new DataStreamMetadata(dataStreams);
+        }
+    }
+
+    static class DataStreamMetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, DataStream>> dataStreamDiff;
+
+        DataStreamMetadataDiff(DataStreamMetadata before, DataStreamMetadata after) {
+            this.dataStreamDiff = DiffableUtils.diff(before.dataStreams, after.dataStreams,
+                DiffableUtils.getStringKeySerializer());
+        }
+
+        DataStreamMetadataDiff(StreamInput in) throws IOException {
+            this.dataStreamDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                DataStream::new, DataStream::readDiffFrom);
+        }
+
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            return new DataStreamMetadata(dataStreamDiff.apply(((DataStreamMetadata) part).dataStreams));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            dataStreamDiff.writeTo(out);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 8471364b07dea..73ab3ccab7176 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -674,6 +674,12 @@ public Map<String, IndexTemplateV2> templatesV2() {
             .orElse(Collections.emptyMap());
     }
 
+    public Map<String, DataStream> dataStreams() {
+        return Optional.ofNullable((DataStreamMetadata) this.custom(DataStreamMetadata.TYPE))
+            .map(DataStreamMetadata::dataStreams)
+            .orElse(Collections.emptyMap());
+    }
+
     public ImmutableOpenMap<String, Custom> customs() {
         return this.customs;
     }
@@ -1121,6 +1127,32 @@ public Builder removeIndexTemplate(String name) {
             return this;
         }
 
+        public Builder dataStreams(Map<String, DataStream> dataStreams) {
+            this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams));
+            return this;
+        }
+
+        public Builder put(DataStream dataStream) {
+            Objects.requireNonNull(dataStream, "it is invalid to add a null data stream");
+            Map<String, DataStream> existingDataStreams =
+                Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
+                    .map(dsmd -> new HashMap<>(dsmd.dataStreams()))
+                    .orElse(new HashMap<>());
+            existingDataStreams.put(dataStream.getName(), dataStream);
+            this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams));
+            return this;
+        }
+
+        public Builder removeDataStream(String name) {
+            Map<String, DataStream> existingDataStreams =
+                Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
+                    .map(dsmd -> new HashMap<>(dsmd.dataStreams()))
+                    .orElse(new HashMap<>());
+            existingDataStreams.remove(name);
+            this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams));
+            return this;
+        }
+
         public Custom getCustom(String type) {
             return customs.get(type);
         }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
index d6a846c205fb3..6945811c8df23 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
@@ -18,10 +18,21 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction.Request;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 public class CreateDataStreamRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -35,4 +46,40 @@ protected Request createTestInstance() {
         request.setTimestampFieldName(randomAlphaOfLength(8));
         return request;
     }
+
+    public void testValidateRequest() {
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream");
+        req.setTimestampFieldName("my-timestamp-field");
+        ActionRequestValidationException e = req.validate();
+        assertNull(e);
+    }
+
+    public void testValidateRequestWithoutTimestampField() {
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream");
+        ActionRequestValidationException e = req.validate();
+        assertNotNull(e);
+        assertThat(e.validationErrors().size(), equalTo(1));
+        assertThat(e.validationErrors().get(0), containsString("timestamp field name is missing"));
+    }
+
+    public void testCreateDataStream() {
+        final String dataStreamName = "my-data-stream";
+        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName);
+        ClusterState newState = CreateDataStreamAction.TransportAction.createDataStream(cs, req);
+        assertThat(newState.metaData().dataStreams().size(), equalTo(1));
+        assertThat(newState.metaData().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName));
+    }
+
+    public void testCreateDuplicateDataStream() {
+        final String dataStreamName = "my-data-stream";
+        DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
+        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
+            .metaData(MetaData.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CreateDataStreamAction.TransportAction.createDataStream(cs, req));
+        assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
index f460065699795..42a961d828f4e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
@@ -18,10 +18,22 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction.Request;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -33,4 +45,38 @@ protected Writeable.Reader<Request> instanceReader() {
     protected Request createTestInstance() {
         return new Request(randomAlphaOfLength(8));
     }
+
+    public void testValidateRequest() {
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request("my-data-stream");
+        ActionRequestValidationException e = req.validate();
+        assertNull(e);
+    }
+
+    public void testValidateRequestWithoutName() {
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request("");
+        ActionRequestValidationException e = req.validate();
+        assertNotNull(e);
+        assertThat(e.validationErrors().size(), equalTo(1));
+        assertThat(e.validationErrors().get(0), containsString("name is missing"));
+    }
+
+    public void testDeleteDataStream() {
+        final String dataStreamName = "my-data-stream";
+        DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
+        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
+            .metaData(MetaData.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(dataStreamName);
+
+        ClusterState newState = DeleteDataStreamAction.TransportAction.removeDataStream(cs, req);
+        assertThat(newState.metaData().dataStreams().size(), equalTo(0));
+    }
+
+    public void testDeleteNonexistentDataStream() {
+        final String dataStreamName = "my-data-stream";
+        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(dataStreamName);
+        ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class,
+            () -> DeleteDataStreamAction.TransportAction.removeDataStream(cs, req));
+        assertThat(e.getMessage(), containsString("data_streams matching [" + dataStreamName + "] not found"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
index 062bdef629cc9..1caa8ce568883 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
@@ -18,10 +18,21 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction.Request;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
 public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -33,4 +44,29 @@ protected Writeable.Reader<Request> instanceReader() {
     protected Request createTestInstance() {
         return new Request(generateRandomStringArray(8, 8, false));
     }
+
+    public void testValidateRequest() {
+        GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(new String[]{});
+        ActionRequestValidationException e = req.validate();
+        assertNull(e);
+    }
+
+    public void testGetDataStreams() {
+        final String dataStreamName = "my-data-stream";
+        DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
+        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
+            .metaData(MetaData.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
+        GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(new String[]{dataStreamName});
+        List<DataStream> dataStreams = GetDataStreamsAction.TransportAction.getDataStreams(cs, req);
+        assertThat(dataStreams.size(), equalTo(1));
+        assertThat(dataStreams.get(0).getName(), equalTo(dataStreamName));
+    }
+
+    public void testGetNonexistentDataStream() {
+        final String dataStreamName = "my-data-stream";
+        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
+        GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(new String[]{dataStreamName});
+        List<DataStream> dataStreams = GetDataStreamsAction.TransportAction.getDataStreams(cs, req);
+        assertThat(dataStreams.size(), equalTo(0));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
new file mode 100644
index 0000000000000..7ef4aad1a1802
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataStreamMetadataTests extends AbstractNamedWriteableTestCase<DataStreamMetadata> {
+
+    @Override
+    protected DataStreamMetadata createTestInstance() {
+        if (randomBoolean()) {
+            return new DataStreamMetadata(Collections.emptyMap());
+        }
+        Map<String, DataStream> dataStreams = new HashMap<>();
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            dataStreams.put(randomAlphaOfLength(5), DataStreamTests.randomInstance());
+        }
+        return new DataStreamMetadata(dataStreams);
+    }
+
+    @Override
+    protected DataStreamMetadata mutateInstance(DataStreamMetadata instance) throws IOException {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.singletonList(new NamedWriteableRegistry.Entry(DataStreamMetadata.class,
+            DataStreamMetadata.TYPE, DataStreamMetadata::new)));
+    }
+
+    @Override
+    protected Class<DataStreamMetadata> categoryClass() {
+        return DataStreamMetadata.class;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 072165ab098ef..8e2cae06290ce 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -28,6 +28,15 @@
 
 public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
 
+    public static DataStream randomInstance() {
+        int numIndices = randomIntBetween(0, 128);
+        List<String> indices = new ArrayList<>(numIndices);
+        for (int i = 0; i < numIndices; i++) {
+            indices.add(randomAlphaOfLength(10));
+        }
+        return new DataStream(randomAlphaOfLength(10), randomAlphaOfLength(10), indices);
+    }
+
     @Override
     protected DataStream doParseInstance(XContentParser parser) throws IOException {
         return DataStream.fromXContent(parser);
@@ -40,11 +49,7 @@ protected Writeable.Reader<DataStream> instanceReader() {
 
     @Override
     protected DataStream createTestInstance() {
-        int numIndices = randomIntBetween(0, 128);
-        List<String> indices = new ArrayList<>(numIndices);
-        for (int i = 0; i < numIndices; i++) {
-            indices.add(randomAlphaOfLength(10));
-        }
-        return new DataStream(randomAlphaOfLength(10), randomAlphaOfLength(10), indices);
+        return randomInstance();
     }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index bd2e48a20c2e4..e3099ab8f9de3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -939,6 +939,7 @@ public static MetaData randomMetaData() {
             .version(randomNonNegativeLong())
             .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
             .put("index_template_v2_" + randomAlphaOfLength(3), IndexTemplateV2Tests.randomInstance())
+            .put(DataStreamTests.randomInstance())
             .build();
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index d5e76c97e1ad4..d2a95cb4fc905 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -151,6 +151,8 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         .putAlias(newAliasMetaDataBuilder("alias-bar1"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put(new DataStream("data-stream1", "@timestamp", Collections.emptyList()))
+                .put(new DataStream("data-stream2", "@timestamp2", Collections.emptyList()))
                 .build();
 
         String metaDataSource = MetaData.Builder.toXContent(metaData);
@@ -323,6 +325,16 @@ public void testSimpleJsonFromAndTo() throws IOException {
             equalTo(new Template(Settings.builder().put("setting", "value").build(),
                 new CompressedXContent("{\"baz\":\"eggplant\"}"),
                 Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
+
+        // data streams
+        assertNotNull(parsedMetaData.dataStreams().get("data-stream1"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream1").getName(), is("data-stream1"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream1").getTimeStampField(), is("@timestamp"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream1").getIndices(), is(Collections.emptyList()));
+        assertNotNull(parsedMetaData.dataStreams().get("data-stream2"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream2").getName(), is("data-stream2"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream2").getTimeStampField(), is("@timestamp2"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream2").getIndices(), is(Collections.emptyList()));
     }
 
     private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";

From 68f42979f74d8653d3006078100ed55aa84c51ae Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 24 Mar 2020 12:31:07 +0100
Subject: [PATCH 274/285] Improve async search's tasks cancellation (#53799)

This commit adds an explicit cancellation of the search task if
the initial async search submit task is cancelled (connection closed by the user).
This was previously done through the cancellation of the parent task but we don't
handle grand-children cancellation yet so we have to manually cancel the search task
in order to ensure that shard actions are cancelled too.
This change can be considered as a workaround until #50990 is fixed.
---
 .../xpack/search/AsyncSearchTask.java         |  35 ++--
 .../TransportSubmitAsyncSearchAction.java     |  15 +-
 .../xpack/search/AsyncSearchActionTests.java  |  24 +++
 .../search/AsyncSearchIntegTestCase.java      | 170 +-----------------
 .../xpack/search/AsyncSearchTaskTests.java    |  25 ++-
 .../xpack/search/BlockingQueryBuilder.java    | 118 ++++++++++++
 .../search/CancellingAggregationBuilder.java  | 102 +++++++++++
 .../xpack/search/SearchTestPlugin.java        |  42 +++++
 .../xpack/search/ShardIdLatch.java            |  29 +++
 .../action/SubmitAsyncSearchRequest.java      |   3 +-
 10 files changed, 377 insertions(+), 186 deletions(-)
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java

diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
index 479daffc832b5..daac142a768ef 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
@@ -35,6 +35,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -42,6 +43,7 @@
  * Task that tracks the progress of a currently running {@link SearchRequest}.
  */
 final class AsyncSearchTask extends SearchTask {
+    private final BooleanSupplier checkSubmitCancellation;
     private final AsyncSearchId searchId;
     private final Client client;
     private final ThreadPool threadPool;
@@ -68,6 +70,7 @@ final class AsyncSearchTask extends SearchTask {
      * @param type The type of the task.
      * @param action The action name.
      * @param parentTaskId The parent task id.
+     * @param checkSubmitCancellation A boolean supplier that checks if the submit task has been cancelled.
      * @param originHeaders All the request context headers.
      * @param taskHeaders The filtered request headers for the task.
      * @param searchId The {@link AsyncSearchId} of the task.
@@ -78,6 +81,7 @@ final class AsyncSearchTask extends SearchTask {
                     String type,
                     String action,
                     TaskId parentTaskId,
+                    BooleanSupplier checkSubmitCancellation,
                     TimeValue keepAlive,
                     Map<String, String> originHeaders,
                     Map<String, String> taskHeaders,
@@ -86,6 +90,7 @@ final class AsyncSearchTask extends SearchTask {
                     ThreadPool threadPool,
                     Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier) {
         super(id, type, action, "async_search", parentTaskId, taskHeaders);
+        this.checkSubmitCancellation = checkSubmitCancellation;
         this.expirationTimeMillis = getStartTime() + keepAlive.getMillis();
         this.originHeaders = originHeaders;
         this.searchId = searchId;
@@ -212,13 +217,13 @@ private void internalAddCompletionListener(ActionListener<AsyncSearchResponse> l
 
                 final Cancellable cancellable;
                 try {
-                    cancellable = threadPool.schedule(() -> {
+                    cancellable = threadPool.schedule(threadPool.preserveContext(() -> {
                         if (hasRun.compareAndSet(false, true)) {
                             // timeout occurred before completion
                             removeCompletionListener(id);
                             listener.onResponse(getResponse());
                         }
-                    }, waitForCompletion, "generic");
+                    }), waitForCompletion, "generic");
                 } catch (EsRejectedExecutionException exc) {
                     listener.onFailure(exc);
                     return;
@@ -291,10 +296,13 @@ private AsyncSearchResponse getResponse() {
         return searchResponse.get().toAsyncSearchResponse(this, expirationTimeMillis);
     }
 
-    // cancels the task if it expired
-    private void checkExpiration() {
+    // checks if the search task should be cancelled
+    private void checkCancellation() {
         long now = System.currentTimeMillis();
-        if (expirationTimeMillis < now) {
+        if (expirationTimeMillis < now || checkSubmitCancellation.getAsBoolean()) {
+            // we cancel the search task if the initial submit task was cancelled,
+            // this is needed because the task cancellation mechanism doesn't
+            // handle the cancellation of grand-children.
             cancelTask(() -> {});
         }
     }
@@ -302,30 +310,31 @@ private void checkExpiration() {
     class Listener extends SearchProgressActionListener {
         @Override
         protected void onQueryResult(int shardIndex) {
-            checkExpiration();
+            checkCancellation();
         }
 
         @Override
         protected void onFetchResult(int shardIndex) {
-            checkExpiration();
+            checkCancellation();
         }
 
         @Override
         protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
             // best effort to cancel expired tasks
-            checkExpiration();
-            searchResponse.get().addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget));
+            checkCancellation();
+            searchResponse.get().addShardFailure(shardIndex,
+                new ShardSearchFailure(exc, shardTarget.getNodeId() != null ? shardTarget : null));
         }
 
         @Override
         protected void onFetchFailure(int shardIndex, Exception exc) {
-            checkExpiration();
+            checkCancellation();
         }
 
         @Override
         protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped, Clusters clusters, boolean fetchPhase) {
             // best effort to cancel expired tasks
-            checkExpiration();
+            checkCancellation();
             searchResponse.compareAndSet(null,
                 new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, aggReduceContextSupplier));
             executeInitListeners();
@@ -334,7 +343,7 @@ protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped,
         @Override
         public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
             // best effort to cancel expired tasks
-            checkExpiration();
+            checkCancellation();
             searchResponse.get().updatePartialResponse(shards.size(),
                 new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
                     null, null, false, null, reducePhase), aggs == null);
@@ -343,7 +352,7 @@ public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, Inter
         @Override
         public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
             // best effort to cancel expired tasks
-            checkExpiration();
+            checkCancellation();
             searchResponse.get().updatePartialResponse(shards.size(),
                 new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
                     null, null, false, null, reducePhase), true);
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
index 032c6114a1a44..7f002bd4e8f56 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
@@ -66,7 +66,7 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService,
     @Override
     protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener<AsyncSearchResponse> submitListener) {
         CancellableTask submitTask = (CancellableTask) task;
-        final SearchRequest searchRequest = createSearchRequest(request, submitTask.getId(), request.getKeepAlive());
+        final SearchRequest searchRequest = createSearchRequest(request, submitTask, request.getKeepAlive());
         AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest);
         searchAction.execute(searchTask, searchRequest, searchTask.getSearchProgressActionListener());
         searchTask.addCompletionListener(
@@ -81,7 +81,7 @@ public void onResponse(AsyncSearchResponse searchResponse) {
                                 // the user cancelled the submit so we don't store anything
                                 // and propagate the failure
                                 Exception cause = new TaskCancelledException(submitTask.getReasonCancelled());
-                                onFatalFailure(searchTask, cause, false, submitListener);
+                                onFatalFailure(searchTask, cause, searchResponse.isRunning(), submitListener);
                             } else {
                                 final String docId = searchTask.getSearchId().getDocId();
                                 // creates the fallback response if the node crashes/restarts in the middle of the request
@@ -129,7 +129,7 @@ public void onFailure(Exception exc) {
             }, request.getWaitForCompletion());
     }
 
-    private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long parentTaskId, TimeValue keepAlive) {
+    private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, CancellableTask submitTask, TimeValue keepAlive) {
         String docID = UUIDs.randomBase64UUID();
         Map<String, String> originHeaders = nodeClient.threadPool().getThreadContext().getHeaders();
         SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) {
@@ -138,16 +138,17 @@ public AsyncSearchTask createTask(long id, String type, String action, TaskId pa
                 AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id));
                 Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier =
                         () -> requestToAggReduceContextBuilder.apply(request.getSearchRequest());
-                return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId,
-                    store.getClient(), nodeClient.threadPool(), aggReduceContextSupplier);
+                return new AsyncSearchTask(id, type, action, parentTaskId,
+                    () -> submitTask.isCancelled(), keepAlive, originHeaders, taskHeaders, searchId, store.getClient(),
+                    nodeClient.threadPool(), aggReduceContextSupplier);
             }
         };
-        searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), parentTaskId));
+        searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), submitTask.getId()));
         return searchRequest;
     }
 
     private void onFatalFailure(AsyncSearchTask task, Exception error, boolean shouldCancel, ActionListener<AsyncSearchResponse> listener) {
-        if (shouldCancel) {
+        if (shouldCancel && task.isCancelled() == false) {
             task.cancelTask(() -> {
                 try {
                     task.addCompletionListener(finalResponse -> taskManager.unregister(task));
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
index 373030fe12efa..ffacf2398e107 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
@@ -253,4 +253,28 @@ public void testNoIndex() throws Exception {
         ElasticsearchException exc = response.getFailure();
         assertThat(exc.getMessage(), containsString("no such index"));
     }
+
+    public void testCancellation() throws Exception {
+        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(indexName);
+        request.getSearchRequest().source(
+            new SearchSourceBuilder().aggregation(new CancellingAggregationBuilder("test"))
+        );
+        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        AsyncSearchResponse response = submitAsyncSearch(request);
+        assertNotNull(response.getSearchResponse());
+        assertTrue(response.isRunning());
+        assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards));
+        assertThat(response.getSearchResponse().getSuccessfulShards(), equalTo(0));
+        assertThat(response.getSearchResponse().getFailedShards(), equalTo(0));
+
+        response = getAsyncSearch(response.getId());
+        assertNotNull(response.getSearchResponse());
+        assertTrue(response.isRunning());
+        assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards));
+        assertThat(response.getSearchResponse().getSuccessfulShards(), equalTo(0));
+        assertThat(response.getSearchResponse().getFailedShards(), equalTo(0));
+
+        deleteAsyncSearch(response.getId());
+        ensureTaskRemoval(response.getId());
+    }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
index e5e65a832db8e..9c4c6757d67cd 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
@@ -5,11 +5,7 @@
  */
 package org.elasticsearch.xpack.search;
 
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.TotalHits;
-import org.apache.lucene.search.Weight;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
@@ -18,22 +14,12 @@
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.reindex.ReindexPlugin;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.TaskId;
@@ -48,15 +34,11 @@
 import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.io.Closeable;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -73,7 +55,7 @@ interface SearchResponseIterator extends Iterator<AsyncSearchResponse>, Closeabl
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class,
-            QueryBlockPlugin.class, ReindexPlugin.class);
+            SearchTestPlugin.class, ReindexPlugin.class);
     }
 
     /**
@@ -152,14 +134,14 @@ protected SearchResponseIterator assertBlockingIterator(String indexName,
             .collect(
                 Collectors.toMap(
                     Function.identity(),
-                    id -> new ShardIdLatch(id, new CountDownLatch(1), failures.decrementAndGet() >= 0)
+                    id -> new ShardIdLatch(id, failures.decrementAndGet() >= 0)
                 )
             );
         ShardIdLatch[] shardLatchArray = shardLatchMap.values().stream()
-            .sorted(Comparator.comparing(ShardIdLatch::shard))
+            .sorted(Comparator.comparing(ShardIdLatch::shardId))
             .toArray(ShardIdLatch[]::new);
         resetPluginsLatch(shardLatchMap);
-        request.getSearchRequest().source().query(new BlockQueryBuilder(shardLatchMap));
+        request.getSearchRequest().source().query(new BlockingQueryBuilder(shardLatchMap));
 
         final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get();
 
@@ -197,7 +179,7 @@ private AsyncSearchResponse doNext() throws Exception {
                 int step = shardIndex == 0 ? progressStep+1 : progressStep-1;
                 int index = 0;
                 while (index < step && shardIndex < shardLatchArray.length) {
-                    if (shardLatchArray[shardIndex].shouldFail == false) {
+                    if (shardLatchArray[shardIndex].shouldFail() == false) {
                         ++index;
                     }
                     shardLatchArray[shardIndex++].countDown();
@@ -242,8 +224,8 @@ private AsyncSearchResponse doNext() throws Exception {
             @Override
             public void close() {
                 Arrays.stream(shardLatchArray).forEach(shard -> {
-                    if (shard.latch.getCount() == 1) {
-                        shard.latch.countDown();
+                    if (shard.getCount() == 1) {
+                        shard.countDown();
                     }
                 });
             }
@@ -252,143 +234,7 @@ public void close() {
 
     private void resetPluginsLatch(Map<ShardId, ShardIdLatch> newLatch) {
         for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) {
-            pluginsService.filterPlugins(QueryBlockPlugin.class).forEach(p -> p.reset(newLatch));
-        }
-    }
-
-    public static class QueryBlockPlugin extends Plugin implements SearchPlugin {
-        private Map<ShardId, ShardIdLatch> shardsLatch;
-
-        public QueryBlockPlugin() {
-            this.shardsLatch = null;
-        }
-
-        public void reset(Map<ShardId, ShardIdLatch> newLatch) {
-            shardsLatch = newLatch;
-        }
-
-        @Override
-        public List<QuerySpec<?>> getQueries() {
-            return Collections.singletonList(
-                new QuerySpec<>("block_match_all",
-                    in -> new BlockQueryBuilder(in, shardsLatch),
-                    p -> BlockQueryBuilder.fromXContent(p, shardsLatch))
-            );
-        }
-    }
-
-    private static class BlockQueryBuilder extends AbstractQueryBuilder<BlockQueryBuilder> {
-        public static final String NAME = "block_match_all";
-        private final Map<ShardId, ShardIdLatch> shardsLatch;
-
-        private BlockQueryBuilder(Map<ShardId, ShardIdLatch> shardsLatch) {
-            super();
-            this.shardsLatch = shardsLatch;
-        }
-
-        BlockQueryBuilder(StreamInput in, Map<ShardId, ShardIdLatch> shardsLatch) throws IOException {
-            super(in);
-            this.shardsLatch = shardsLatch;
-        }
-
-        private BlockQueryBuilder() {
-            this.shardsLatch = null;
-        }
-
-        @Override
-        protected void doWriteTo(StreamOutput out) {}
-
-        @Override
-        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME);
-            builder.endObject();
-        }
-
-        private static final ObjectParser<BlockQueryBuilder, Void> PARSER = new ObjectParser<>(NAME, BlockQueryBuilder::new);
-
-        public static BlockQueryBuilder fromXContent(XContentParser parser, Map<ShardId, ShardIdLatch> shardsLatch) {
-            try {
-                PARSER.apply(parser, null);
-                return new BlockQueryBuilder(shardsLatch);
-            } catch (IllegalArgumentException e) {
-                throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
-            }
-        }
-
-        @Override
-        protected Query doToQuery(QueryShardContext context) {
-            final Query delegate = Queries.newMatchAllQuery();
-            return new Query() {
-                @Override
-                public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
-                    if (shardsLatch != null) {
-                        try {
-                            final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId()));
-                            latch.await();
-                            if (latch.shouldFail) {
-                                throw new IOException("boum");
-                            }
-                        } catch (InterruptedException e) {
-                            throw new RuntimeException(e);
-                        }
-                    }
-                    return delegate.createWeight(searcher, scoreMode, boost);
-                }
-
-                @Override
-                public String toString(String field) {
-                    return delegate.toString(field);
-                }
-
-                @Override
-                public boolean equals(Object obj) {
-                    return false;
-                }
-
-                @Override
-                public int hashCode() {
-                    return 0;
-                }
-            };
-        }
-
-        @Override
-        protected boolean doEquals(BlockQueryBuilder other) {
-            return false;
-        }
-
-        @Override
-        protected int doHashCode() {
-            return 0;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-    }
-
-    private static class ShardIdLatch {
-        private final ShardId shard;
-        private final CountDownLatch latch;
-        private final boolean shouldFail;
-
-        private ShardIdLatch(ShardId shard, CountDownLatch latch, boolean shouldFail) {
-            this.shard = shard;
-            this.latch = latch;
-            this.shouldFail = shouldFail;
-        }
-
-        ShardId shard() {
-            return shard;
-        }
-
-        void countDown() {
-            latch.countDown();
-        }
-
-        void await() throws InterruptedException {
-            latch.await();
+            pluginsService.filterPlugins(SearchTestPlugin.class).forEach(p -> p.resetQueryLatch(newLatch));
         }
     }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
index 8c3f57883ec14..0d8a0930c4601 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
@@ -27,6 +28,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -35,6 +37,23 @@
 public class AsyncSearchTaskTests extends ESTestCase {
     private ThreadPool threadPool;
 
+    private static class TestTask extends CancellableTask {
+        private TestTask(long id, String type, String action, String description, TaskId parentTaskId, Map<String, String> headers) {
+            super(id, type, action, description, parentTaskId, headers);
+        }
+
+        @Override
+        public boolean shouldCancelChildrenOnCancellation() {
+            return false;
+        }
+    }
+
+    private static TestTask createSubmitTask() {
+        return new TestTask(0L, "", "", "test", new TaskId("node1", 0), Collections.emptyMap());
+    }
+
+
+
     @Before
     public void beforeTest() {
         threadPool = new TestThreadPool(getTestName());
@@ -46,7 +65,7 @@ public void afterTest() {
     }
 
     public void testWaitForInit() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), () -> false, TimeValue.timeValueHours(1),
             Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
             new NoOpClient(threadPool), threadPool, null);
         int numShards = randomIntBetween(0, 10);
@@ -86,7 +105,7 @@ public void onFailure(Exception e) {
     }
 
     public void testWithFailure() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), () -> false, TimeValue.timeValueHours(1),
             Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
             new NoOpClient(threadPool), threadPool, null);
         int numThreads = randomIntBetween(1, 10);
@@ -114,7 +133,7 @@ public void onFailure(Exception e) {
     }
 
     public void testWaitForCompletion() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), () -> false, TimeValue.timeValueHours(1),
             Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
             new NoOpClient(threadPool), threadPool, null);
         int numShards = randomIntBetween(0, 10);
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
new file mode 100644
index 0000000000000..5939cc10458c3
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A query builder that blocks shard execution based on the provided {@link ShardIdLatch}.
+ */
+class BlockingQueryBuilder extends AbstractQueryBuilder<BlockingQueryBuilder> {
+    public static final String NAME = "block";
+    private final Map<ShardId, ShardIdLatch> shardsLatch;
+
+    BlockingQueryBuilder(Map<ShardId, ShardIdLatch> shardsLatch) {
+        super();
+        this.shardsLatch = shardsLatch;
+    }
+
+    BlockingQueryBuilder(StreamInput in, Map<ShardId, ShardIdLatch> shardsLatch) throws IOException {
+        super(in);
+        this.shardsLatch = shardsLatch;
+    }
+
+    BlockingQueryBuilder() {
+        this.shardsLatch = null;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) {}
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.endObject();
+    }
+
+    private static final ObjectParser<BlockingQueryBuilder, Void> PARSER = new ObjectParser<>(NAME, BlockingQueryBuilder::new);
+
+    public static BlockingQueryBuilder fromXContent(XContentParser parser, Map<ShardId, ShardIdLatch> shardsLatch) {
+        try {
+            PARSER.apply(parser, null);
+            return new BlockingQueryBuilder(shardsLatch);
+        } catch (IllegalArgumentException e) {
+            throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
+        }
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) {
+        final Query delegate = Queries.newMatchAllQuery();
+        return new Query() {
+            @Override
+            public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+                if (shardsLatch != null) {
+                    try {
+                        final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId()));
+                        latch.await();
+                        if (latch.shouldFail()) {
+                            throw new IOException("boum");
+                        }
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                return delegate.createWeight(searcher, scoreMode, boost);
+            }
+
+            @Override
+            public String toString(String field) {
+                return delegate.toString(field);
+            }
+
+            @Override
+            public boolean equals(Object obj) {
+                return false;
+            }
+
+            @Override
+            public int hashCode() {
+                return 0;
+            }
+        };
+    }
+
+    @Override
+    protected boolean doEquals(BlockingQueryBuilder other) {
+        return false;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return 0;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
new file mode 100644
index 0000000000000..354c3c4ec340a
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An aggregation builder that blocks shard search action until the task is cancelled.
+ */
+public class CancellingAggregationBuilder extends AbstractAggregationBuilder<CancellingAggregationBuilder> {
+    static final String NAME = "cancel";
+    static final int SLEEP_TIME = 10;
+
+    public CancellingAggregationBuilder(String name) {
+        super(name);
+    }
+
+    public CancellingAggregationBuilder(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metaData) {
+        return new CancellingAggregationBuilder(name);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    static final ConstructingObjectParser<CancellingAggregationBuilder, String> PARSER =
+        new ConstructingObjectParser<>(NAME, false, (args, name) -> new CancellingAggregationBuilder(name));
+
+
+    static CancellingAggregationBuilder fromXContent(String aggName, XContentParser parser) {
+        try {
+            return PARSER.apply(parser, aggName);
+        } catch (IllegalArgumentException e) {
+            throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected AggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent,
+                                        AggregatorFactories.Builder subfactoriesBuilder) throws IOException {
+        final FilterAggregationBuilder filterAgg = new FilterAggregationBuilder(name, QueryBuilders.matchAllQuery());
+        filterAgg.subAggregations(subfactoriesBuilder);
+        final AggregatorFactory factory = filterAgg.build(queryShardContext, parent);
+        return new AggregatorFactory(name, queryShardContext, parent, subfactoriesBuilder, metaData) {
+            @Override
+            protected Aggregator createInternal(SearchContext searchContext,
+                                                Aggregator parent,
+                                                boolean collectsFromSingleBucket,
+                                                List<PipelineAggregator> pipelineAggregators,
+                                                Map<String, Object> metaData) throws IOException {
+                while (searchContext.isCancelled() == false) {
+                    try {
+                        Thread.sleep(SLEEP_TIME);
+                    } catch (InterruptedException e) {
+                        throw new IOException(e);
+                    }
+                }
+                return factory.create(searchContext, parent, collectsFromSingleBucket);
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java
new file mode 100644
index 0000000000000..88daac239fad1
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class SearchTestPlugin extends Plugin implements SearchPlugin {
+    private Map<ShardId, ShardIdLatch> shardsLatch;
+
+    public SearchTestPlugin() {
+        this.shardsLatch = null;
+    }
+
+    public void resetQueryLatch(Map<ShardId, ShardIdLatch> newLatch) {
+        shardsLatch = newLatch;
+    }
+
+    @Override
+    public List<QuerySpec<?>> getQueries() {
+        return Collections.singletonList(
+            new QuerySpec<>(BlockingQueryBuilder.NAME,
+                in -> new BlockingQueryBuilder(in, shardsLatch),
+                p -> BlockingQueryBuilder.fromXContent(p, shardsLatch))
+        );
+    }
+
+    @Override
+    public List<AggregationSpec> getAggregations() {
+        return Collections.singletonList(new AggregationSpec(CancellingAggregationBuilder.NAME, CancellingAggregationBuilder::new,
+            CancellingAggregationBuilder.PARSER).addResultReader(InternalFilter::new));
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java
new file mode 100644
index 0000000000000..dd171b3102894
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.index.shard.ShardId;
+
+import java.util.concurrent.CountDownLatch;
+
+class ShardIdLatch extends CountDownLatch {
+    private final ShardId shard;
+    private final boolean shouldFail;
+
+    ShardIdLatch(ShardId shard, boolean shouldFail) {
+        super(1);
+        this.shard = shard;
+        this.shouldFail = shouldFail;
+    }
+
+    ShardId shardId() {
+        return shard;
+    }
+
+    boolean shouldFail() {
+        return shouldFail;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
index a6397cbf08c3b..6717c766f03e8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
@@ -150,7 +150,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
         return new CancellableTask(id, type, action, toString(), parentTaskId, headers) {
             @Override
             public boolean shouldCancelChildrenOnCancellation() {
-                return true;
+                // we cancel the underlying search action explicitly in the submit action
+                return false;
             }
         };
     }

From 36377d75876823d93b2cf589872bfb9ea4feb326 Mon Sep 17 00:00:00 2001
From: Przemko Robakowski <przemko.robakowski@elastic.co>
Date: Tue, 24 Mar 2020 13:09:59 +0100
Subject: [PATCH 275/285] Add validation for component templates (#54023)

* Add validation for component templates

This change adds validation to make sure that settings and mappings are correct in
component template. It's done the same way as in index templates - code is reused.

Reletes to #53101

* Fix checkstyle violation

* Update server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

Co-authored-by: Lee Hinman <dakrone@users.noreply.github.com>
---
 .../TransportPutComponentTemplateAction.java  | 12 ++-
 .../MetaDataIndexTemplateService.java         | 51 +++++++------
 .../metadata/ComponentTemplateTests.java      |  2 +-
 .../MetaDataIndexTemplateServiceTests.java    | 73 +++++++++++++------
 4 files changed, 88 insertions(+), 50 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
index c915a56231472..bc2816c5614b5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -45,14 +46,17 @@ public class TransportPutComponentTemplateAction
     extends TransportMasterNodeAction<PutComponentTemplateAction.Request, AcknowledgedResponse> {
 
     private final MetaDataIndexTemplateService indexTemplateService;
+    private final IndexScopedSettings indexScopedSettings;
 
     @Inject
     public TransportPutComponentTemplateAction(TransportService transportService, ClusterService clusterService,
                                                ThreadPool threadPool, MetaDataIndexTemplateService indexTemplateService,
-                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                               IndexScopedSettings indexScopedSettings) {
         super(PutComponentTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters,
             PutComponentTemplateAction.Request::new, indexNameExpressionResolver);
         this.indexTemplateService = indexTemplateService;
+        this.indexScopedSettings = indexScopedSettings;
     }
 
     @Override
@@ -78,8 +82,10 @@ protected void masterOperation(Task task, final PutComponentTemplateAction.Reque
         Template template = componentTemplate.template();
         // Normalize the index settings if necessary
         if (template.settings() != null) {
-            Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
-            template = new Template(settings.build(), template.mappings(), template.aliases());
+            Settings.Builder builder = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
+            Settings settings = builder.build();
+            indexScopedSettings.validate(settings, true);
+            template = new Template(settings, template.mappings(), template.aliases());
             componentTemplate = new ComponentTemplate(template, componentTemplate.version(), componentTemplate.metadata());
         }
         indexTemplateService.putComponentTemplate(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index ecce58e5a7154..21b7333eeace7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -154,7 +155,7 @@ public void onFailure(String source, Exception e) {
                 }
 
                 @Override
-                public ClusterState execute(ClusterState currentState) {
+                public ClusterState execute(ClusterState currentState) throws Exception {
                     return addComponentTemplate(currentState, create, name, template);
                 }
 
@@ -166,14 +167,15 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
     }
 
     // Package visible for testing
-    static ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
-                                             final String name, final ComponentTemplate template) {
+    ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
+                                             final String name, final ComponentTemplate template) throws Exception {
         if (create && currentState.metaData().componentTemplates().containsKey(name)) {
             throw new IllegalArgumentException("component template [" + name + "] already exists");
         }
 
-        // TODO: validation of component template
-        // validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
+        CompressedXContent mappings = template.template().mappings();
+        String stringMappings = mappings == null ? null : mappings.string();
+        validateTemplate(template.template().settings(), stringMappings, indicesService, xContentRegistry);
 
         logger.info("adding component template [{}]", name);
         return ClusterState.builder(currentState)
@@ -274,7 +276,20 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
                 }
 
-                validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
+                templateBuilder.order(request.order);
+                templateBuilder.version(request.version);
+                templateBuilder.patterns(request.indexPatterns);
+                templateBuilder.settings(request.settings);
+
+                if (request.mappings != null) {
+                    try {
+                        templateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, request.mappings);
+                    } catch (Exception e) {
+                        throw new MapperParsingException("Failed to parse mapping: {}", e, request.mappings);
+                    }
+                }
+
+                validateTemplate(request.settings, request.mappings, indicesService, xContentRegistry);
 
                 for (Alias alias : request.aliases) {
                     AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
@@ -355,20 +370,20 @@ public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, Strin
         return matchedTemplates;
     }
 
-    private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder,
-            IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
+    private static void validateTemplate(Settings settings, String mappings,
+                                         IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
         Index createdIndex = null;
         final String temporaryIndexName = UUIDs.randomBase64UUID();
         try {
             // use the provided values, otherwise just pick valid dummy values
-            int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(request.settings);
-            int dummyShards = request.settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+            int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(settings);
+            int dummyShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                     dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
 
             //create index service for parsing and validating "mappings"
             Settings dummySettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(request.settings)
+                .put(settings)
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyShards)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
@@ -378,19 +393,9 @@ private static void validateAndAddTemplate(final PutRequest request, IndexTempla
             IndexService dummyIndexService = indicesService.createIndex(tmpIndexMetadata, Collections.emptyList(), false);
             createdIndex = dummyIndexService.index();
 
-            templateBuilder.order(request.order);
-            templateBuilder.version(request.version);
-            templateBuilder.patterns(request.indexPatterns);
-            templateBuilder.settings(request.settings);
-
-            if (request.mappings != null) {
-                try {
-                    templateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, request.mappings);
-                } catch (Exception e) {
-                    throw new MapperParsingException("Failed to parse mapping: {}", e, request.mappings);
-                }
+            if (mappings != null) {
                 dummyIndexService.mapperService().merge(MapperService.SINGLE_MAPPING_NAME,
-                    MapperService.parseMapping(xContentRegistry, request.mappings), MergeReason.MAPPING_UPDATE);
+                    MapperService.parseMapping(xContentRegistry, mappings), MergeReason.MAPPING_UPDATE);
             }
 
         } finally {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
index 470d573bb5212..82976d348f4e3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -85,7 +85,7 @@ public static ComponentTemplate randomInstance() {
         return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta);
     }
 
-    private static Map<String, AliasMetaData> randomAliases() {
+    public static Map<String, AliasMetaData> randomAliases() {
         String aliasName = randomAlphaOfLength(5);
         AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
             .filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
index 94d37a52b89be..4e9afabaa0266 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import com.fasterxml.jackson.core.JsonParseException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -37,6 +39,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -197,21 +200,41 @@ public void testPutGlobalTemplateWithIndexHiddenSetting() throws Exception {
         assertThat(errors.get(0).getMessage(), containsString("global templates may not specify the setting index.hidden"));
     }
 
-    public void testAddComponentTemplate() {
+    public void testAddComponentTemplate() throws Exception{
+        MetaDataIndexTemplateService metaDataIndexTemplateService = getMetaDataIndexTemplateService();
         ClusterState state = ClusterState.EMPTY_STATE;
-        ComponentTemplate template = ComponentTemplateTests.randomInstance();
-        state = MetaDataIndexTemplateService.addComponentTemplate(state, false, "foo", template);
+        Template template = new Template(Settings.builder().build(), null, ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
+        state = metaDataIndexTemplateService.addComponentTemplate(state, false, "foo", componentTemplate);
 
         assertNotNull(state.metaData().componentTemplates().get("foo"));
-        assertThat(state.metaData().componentTemplates().get("foo"), equalTo(template));
+        assertThat(state.metaData().componentTemplates().get("foo"), equalTo(componentTemplate));
 
         final ClusterState throwState = ClusterState.builder(state).build();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> MetaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", template));
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", componentTemplate));
         assertThat(e.getMessage(), containsString("component template [foo] already exists"));
 
-        state = MetaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", template);
+        state = metaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", componentTemplate);
         assertNotNull(state.metaData().componentTemplates().get("bar"));
+
+        template = new Template(Settings.builder().build(), new CompressedXContent("{\"invalid\"}"),
+            ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate2 = new ComponentTemplate(template, 1L, new HashMap<>());
+        expectThrows(JsonParseException.class,
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate2));
+
+        template = new Template(Settings.builder().build(), new CompressedXContent("{\"invalid\":\"invalid\"}"),
+            ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate3 = new ComponentTemplate(template, 1L, new HashMap<>());
+        expectThrows(MapperParsingException.class,
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate3));
+
+        template = new Template(Settings.builder().put("invalid", "invalid").build(), new CompressedXContent("{}"),
+            ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate4 = new ComponentTemplate(template, 1L, new HashMap<>());
+        expectThrows(IllegalArgumentException.class,
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4));
     }
 
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
@@ -247,23 +270,7 @@ public void onFailure(Exception e) {
     }
 
     private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
-        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
-                Settings.EMPTY,
-                clusterService,
-                indicesService,
-                null,
-                null,
-                new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
-                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-                null,
-                xContentRegistry(),
-                Collections.emptyList(),
-                true);
-        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
-                clusterService, createIndexService, new AliasValidator(), indicesService,
-                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
+        MetaDataIndexTemplateService service = getMetaDataIndexTemplateService();
 
         final List<Throwable> throwables = new ArrayList<>();
         final CountDownLatch latch = new CountDownLatch(1);
@@ -282,4 +289,24 @@ public void onFailure(Exception e) {
         latch.await();
         return throwables;
     }
+
+    private MetaDataIndexTemplateService getMetaDataIndexTemplateService() {
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
+                Settings.EMPTY,
+                clusterService,
+                indicesService,
+                null,
+                null,
+                new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                null,
+                xContentRegistry(),
+                Collections.emptyList(),
+                true);
+        return new MetaDataIndexTemplateService(
+                clusterService, createIndexService, new AliasValidator(), indicesService,
+                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
+    }
 }

From 04c21affcf6b2215f22a690223ba1e520d3c6f9b Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Tue, 24 Mar 2020 12:50:32 +0000
Subject: [PATCH 276/285] Clean up IndexTemplateMetaData xcontent serialization
 (#54003)

IndexTemplateMetaData x-content serialization is a bit confused and trappy at the
moment, with multiple static methods on the Builder object which may or may not
emit the template name and/or mapping types. This commit consolidates everything
into a single toXContent() method on the base class, with different options passed
through as ToXContent.Params. Deserialization is converted to use an ObjectParser,
taking the template name as a context.
---
 .../get/GetIndexTemplatesResponse.java        |   7 +-
 .../elasticsearch/cluster/ClusterState.java   |   5 +-
 .../metadata/IndexTemplateMetaData.java       | 253 +++++++-----------
 .../cluster/metadata/MetaData.java            |   6 +-
 .../metadata/TemplateUpgradeService.java      |  10 +-
 .../metadata/IndexTemplateMetaDataTests.java  |  21 +-
 .../xpack/core/template/TemplateUtils.java    |   2 +-
 .../exporter/http/TemplateHttpResource.java   |   6 +-
 .../exporter/http/HttpExporterIT.java         |   6 +-
 9 files changed, 125 insertions(+), 191 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index a726f4e108d7c..c1fa7d7ffafc2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -31,8 +30,6 @@
 import java.util.List;
 import java.util.Objects;
 
-import static java.util.Collections.singletonMap;
-
 public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {
 
     private final List<IndexTemplateMetaData> indexTemplates;
@@ -77,11 +74,9 @@ public int hashCode() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params);
-
         builder.startObject();
         for (IndexTemplateMetaData indexTemplateMetaData : getIndexTemplates()) {
-            IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params);
+            indexTemplateMetaData.toXContent(builder, params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index f7ea8a68283bb..f831207e2f0dd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -426,10 +427,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             coordinationMetaData().toXContent(builder, params);
             builder.endObject();
 
+            ToXContent.Params templateParams = new DelegatingMapParams(Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true"), params);
             builder.startObject("templates");
             for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
-                IndexTemplateMetaData templateMetaData = cursor.value;
-                IndexTemplateMetaData.Builder.toXContentWithTypes(templateMetaData, builder, params);
+                cursor.value.toXContent(builder, templateParams);
             }
             builder.endObject();
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index 4d2db27a4a840..935ab92f4d056 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -20,36 +20,37 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
-public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {
+public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> implements ToXContentFragment {
 
     private final String name;
 
@@ -228,9 +229,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public String toString() {
         try {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            builder.startObject();
-            IndexTemplateMetaData.Builder.toXContentWithTypes(this, builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
+            this.toXContent(builder, ToXContent.EMPTY_PARAMS);
             return Strings.toString(builder);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -319,185 +318,113 @@ public Builder putAlias(AliasMetaData.Builder aliasMetaData) {
             return this;
         }
 
+        public void putAliases(List<AliasMetaData> aliases) {
+            for (AliasMetaData alias : aliases) {
+                putAlias(alias);
+            }
+        }
+
         public IndexTemplateMetaData build() {
             return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings.build(), aliases.build());
         }
 
-        /**
-         * Serializes the template to xContent, using the legacy format where the mappings are
-         * nested under the type name.
-         *
-         * This method is used for serializing templates before storing them in the cluster metadata,
-         * and also in the REST layer when returning a deprecated typed response.
-         */
-        public static void toXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
-                                               XContentBuilder builder,
-                                               ToXContent.Params params) throws IOException {
-            builder.startObject(indexTemplateMetaData.name());
-            toInnerXContent(indexTemplateMetaData, builder, params, true);
-            builder.endObject();
-        }
+    }
 
-        /**
-         * Removes the nested type in the xContent representation of {@link IndexTemplateMetaData}.
-         *
-         * This method is useful to help bridge the gap between an the internal representation which still uses (the legacy format) a
-         * nested type in the mapping, and the external representation which does not use a nested type in the mapping.
-         */
-        public static void removeType(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder) throws IOException {
-            builder.startObject();
-            toInnerXContent(indexTemplateMetaData, builder,
-                new ToXContent.MapParams(Collections.singletonMap("reduce_mappings", "true")), false);
-            builder.endObject();
-        }
+    public static final String INCLUDE_TYPE_NAME = "include_type_name";
+    public static final String INCLUDE_TEMPLATE_NAME = "include_template_name";
 
-        /**
-         * Serializes the template to xContent, making sure not to nest mappings under the
-         * type name.
-         *
-         * Note that this method should currently only be used for creating REST responses,
-         * and not when directly updating stored templates. Index templates are still stored
-         * in the old, typed format, and have yet to be migrated to be typeless.
-         */
-        public static void toXContent(IndexTemplateMetaData indexTemplateMetaData,
-                                      XContentBuilder builder,
-                                      ToXContent.Params params) throws IOException {
-            builder.startObject(indexTemplateMetaData.name());
-            toInnerXContent(indexTemplateMetaData, builder, params, false);
-            builder.endObject();
-        }
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
 
+        boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME, false);
+        boolean includeTemplateName = params.paramAsBoolean(INCLUDE_TEMPLATE_NAME, true);
 
-        static void toInnerXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
-                                             XContentBuilder builder,
-                                             ToXContent.Params params) throws IOException {
-            toInnerXContent(indexTemplateMetaData, builder, params, true);
+        if (includeTemplateName) {
+            builder.startObject(this.name());
         }
 
-        private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
-                                            XContentBuilder builder,
-                                            ToXContent.Params params,
-                                            boolean includeTypeName) throws IOException {
-
-            builder.field("order", indexTemplateMetaData.order());
-            if (indexTemplateMetaData.version() != null) {
-                builder.field("version", indexTemplateMetaData.version());
-            }
-            builder.field("index_patterns", indexTemplateMetaData.patterns());
-
-            builder.startObject("settings");
-            indexTemplateMetaData.settings().toXContent(builder, params);
-            builder.endObject();
+        builder.field("order", this.order());
+        if (this.version() != null) {
+            builder.field("version", this.version());
+        }
+        builder.field("index_patterns", this.patterns());
 
-            includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false);
+        builder.startObject("settings");
+        this.settings().toXContent(builder, params);
+        builder.endObject();
 
-            CompressedXContent m = indexTemplateMetaData.mappings();
-            if (m != null) {
-                Map<String, Object> documentMapping = XContentHelper.convertToMap(new BytesArray(m.uncompressed()), true).v2();
-                if (includeTypeName == false) {
-                    documentMapping = reduceMapping(documentMapping);
-                }
-                builder.field("mappings");
-                builder.map(documentMapping);
-            } else {
-                builder.startObject("mappings").endObject();
+        CompressedXContent m = this.mappings();
+        if (m != null) {
+            Map<String, Object> documentMapping = XContentHelper.convertToMap(new BytesArray(m.uncompressed()), true).v2();
+            if (includeTypeName == false) {
+                documentMapping = reduceMapping(documentMapping);
             }
+            builder.field("mappings");
+            builder.map(documentMapping);
+        } else {
+            builder.startObject("mappings").endObject();
+        }
 
-            builder.startObject("aliases");
-            for (ObjectCursor<AliasMetaData> cursor : indexTemplateMetaData.aliases().values()) {
-                AliasMetaData.Builder.toXContent(cursor.value, builder, params);
-            }
-            builder.endObject();
+        builder.startObject("aliases");
+        for (ObjectCursor<AliasMetaData> cursor : this.aliases().values()) {
+            AliasMetaData.Builder.toXContent(cursor.value, builder, params);
         }
+        builder.endObject();
 
-        @SuppressWarnings("unchecked")
-        private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
-            assert mapping.keySet().size() == 1;
-            return (Map<String, Object>) mapping.values().iterator().next();
+        if (includeTemplateName) {
+            builder.endObject();
         }
 
-        public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
-            Builder builder = new Builder(templateName);
-
-            String currentFieldName = skipTemplateName(parser);
-            XContentParser.Token token;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    if ("settings".equals(currentFieldName)) {
-                        Settings.Builder templateSettingsBuilder = Settings.builder();
-                        templateSettingsBuilder.put(Settings.fromXContent(parser));
-                        templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
-                        builder.settings(templateSettingsBuilder.build());
-                    } else if ("mappings".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            if (token == XContentParser.Token.FIELD_NAME) {
-                                currentFieldName = parser.currentName();
-                            } else if (token == XContentParser.Token.START_OBJECT) {
-                                String mappingType = currentFieldName;
-                                Map<String, Object> mappingSource =
-                                    MapBuilder.<String, Object>newMapBuilder().put(mappingType, parser.mapOrdered()).map();
-                                builder.putMapping(mappingType, Strings.toString(XContentFactory.jsonBuilder().map(mappingSource)));
-                            }
-                        }
-                    } else if ("aliases".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
-                        }
-                    } else {
-                        throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName);
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    if ("mappings".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            Map<String, Object> mapping = parser.mapOrdered();
-                            if (mapping.size() == 1) {
-                                String mappingType = mapping.keySet().iterator().next();
-                                String mappingSource = Strings.toString(XContentFactory.jsonBuilder().map(mapping));
-
-                                if (mappingSource == null) {
-                                    // crap, no mapping source, warn?
-                                } else {
-                                    builder.putMapping(mappingType, mappingSource);
-                                }
-                            }
-                        }
-                    } else if ("index_patterns".equals(currentFieldName)) {
-                        List<String> index_patterns = new ArrayList<>();
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            index_patterns.add(parser.text());
-                        }
-                        builder.patterns(index_patterns);
-                    }
-                } else if (token.isValue()) {
-                    if ("order".equals(currentFieldName)) {
-                        builder.order(parser.intValue());
-                    } else if ("version".equals(currentFieldName)) {
-                        builder.version(parser.intValue());
-                    }
-                }
+        return builder;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
+        assert mapping.keySet().size() == 1;
+        return (Map<String, Object>) mapping.values().iterator().next();
+    }
+
+    private static final ObjectParser<Builder, String> PARSER = ObjectParser.fromBuilder("index_template", Builder::new);
+
+    static {
+        PARSER.declareObject(Builder::settings,
+            (p, c) -> Settings.builder().put(Settings.fromXContent(p)).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(),
+            new ParseField("settings"));
+        PARSER.declareObject((builder, map) -> {
+            if (map.isEmpty() == false) {
+                String type = map.keySet().iterator().next();
+                builder.putMapping(type, map.get(type));
             }
-            return builder.build();
-        }
+        }, (p, c) -> parseMappings(p), new ParseField("mappings"));
+        PARSER.declareNamedObjects(Builder::putAliases, (p, c, n) -> AliasMetaData.Builder.fromXContent(p), new ParseField("aliases"));
+        PARSER.declareStringArray(Builder::patterns, new ParseField("index_patterns"));
+        PARSER.declareInt(Builder::order, new ParseField("order"));
+        PARSER.declareInt(Builder::version, new ParseField("version"));
+    }
 
-        private static String skipTemplateName(XContentParser parser) throws IOException {
-            XContentParser.Token token = parser.nextToken();
-            if (token == XContentParser.Token.START_OBJECT) {
-                token = parser.nextToken();
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    String currentFieldName = parser.currentName();
-                    if (VALID_FIELDS.contains(currentFieldName)) {
-                        return currentFieldName;
-                    } else {
-                        // we just hit the template name, which should be ignored and we move on
-                        parser.nextToken();
-                    }
+    // TODO it would be really nice if we could just copy bytes here, there's no need to parse and then
+    // re-stream the mappings
+    private static Map<String, CompressedXContent> parseMappings(XContentParser parser) throws IOException {
+        String mappingType = null;
+        String mappings = null;
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                mappingType = parser.currentName();
+            } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                if (mappingType == null) {
+                    throw new XContentParseException(parser.getTokenLocation(), "Expected a mapping type");
                 }
+                mappings = Strings.toString(XContentFactory.jsonBuilder().map(Map.of(mappingType, parser.mapOrdered())));
             }
-
-            return null;
         }
+        if (mappings == null) {
+            return Collections.emptyMap();
+        }
+        return Map.of(mappingType, new CompressedXContent(mappings));
+    }
+
+    public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
+        return PARSER.parse(parser, templateName).build();
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 73ab3ccab7176..e8b607b65b20e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -1404,9 +1404,11 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
                 builder.endObject();
             }
 
+            ToXContent.Params typedParams
+                = new ToXContent.DelegatingMapParams(Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true"), params);
             builder.startObject("templates");
             for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
-                IndexTemplateMetaData.Builder.toXContentWithTypes(cursor.value, builder, params);
+                cursor.value.toXContent(builder, typedParams);
             }
             builder.endObject();
 
@@ -1471,7 +1473,7 @@ public static MetaData fromXContent(XContentParser parser, boolean preserveUnkno
                         builder.hashesOfConsistentSettings(parser.mapStrings());
                     } else if ("templates".equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
+                            builder.put(IndexTemplateMetaData.fromXContent(parser, parser.currentName()));
                         }
                     } else {
                         try {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
index deb04067acb9a..4b595ad14fc2f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
@@ -57,8 +57,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.UnaryOperator;
 
-import static java.util.Collections.singletonMap;
-
 /**
  * Upgrades Templates on behalf of installed {@link Plugin}s when a node joins the cluster
  */
@@ -248,14 +246,12 @@ Optional<Tuple<Map<String, BytesReference>, Set<String>>> calculateTemplateChang
         return Optional.empty();
     }
 
-    private static final ToXContent.Params PARAMS = new ToXContent.MapParams(singletonMap("reduce_mappings", "true"));
+    private static final ToXContent.Params PARAMS
+        = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
 
     private BytesReference toBytesReference(IndexTemplateMetaData templateMetaData) {
         try {
-            return XContentHelper.toXContent((builder, params) -> {
-                IndexTemplateMetaData.Builder.toInnerXContentWithTypes(templateMetaData, builder, params);
-                return builder;
-            }, XContentType.JSON, PARAMS, false);
+            return XContentHelper.toXContent(templateMetaData, XContentType.JSON, PARAMS, false);
         } catch (IOException ex) {
             throw new IllegalStateException("Cannot serialize template [" + templateMetaData.getName() + "]", ex);
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
index 49e886c1d4745..b4410550912e3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
@@ -34,6 +34,7 @@
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.contains;
@@ -54,13 +55,15 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
         final IndexTemplateMetaData indexTemplateMetaData;
         try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
             DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytes, XContentType.JSON)) {
-            indexTemplateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
+            indexTemplateMetaData = IndexTemplateMetaData.fromXContent(parser, "test");
         }
 
         final BytesReference templateBytesRoundTrip;
+        final ToXContent.Params params = new ToXContent.MapParams(
+            Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true", IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
         try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) {
             builder.startObject();
-            IndexTemplateMetaData.Builder.toXContentWithTypes(indexTemplateMetaData, builder, ToXContent.EMPTY_PARAMS);
+            indexTemplateMetaData.toXContent(builder, params);
             builder.endObject();
             templateBytesRoundTrip = BytesReference.bytes(builder);
         }
@@ -68,7 +71,7 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
         final IndexTemplateMetaData indexTemplateMetaDataRoundTrip;
         try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
             DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytesRoundTrip, XContentType.JSON)) {
-            indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
+            indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.fromXContent(parser, "test");
         }
         assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip));
     }
@@ -97,7 +100,7 @@ public void testValidateInvalidIndexPatterns() throws Exception {
                  XContentHelper.createParser(NamedXContentRegistry.EMPTY,
                      DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmptyPattern), XContentType.JSON)) {
             final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
+                () -> IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
             assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got []"));
         }
 
@@ -112,7 +115,7 @@ DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmpty
                  XContentHelper.createParser(NamedXContentRegistry.EMPTY,
                      DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithoutPattern), XContentType.JSON)) {
             final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
+                () -> IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
             assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got null"));
         }
     }
@@ -122,7 +125,7 @@ public void testParseTemplateWithAliases() throws Exception {
         try (XContentParser parser =
                  XContentHelper.createParser(NamedXContentRegistry.EMPTY,
                      DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateInJSON), XContentType.JSON)) {
-            IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100));
+            IndexTemplateMetaData template = IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100));
             assertThat(template.aliases().containsKey("log"), equalTo(true));
             assertThat(template.patterns(), contains("pattern-1"));
         }
@@ -157,12 +160,14 @@ public void testFromToXContent() throws Exception {
             templateBuilder.putMapping("doc", "{\"doc\":{\"properties\":{\"type\":\"text\"}}}");
         }
         IndexTemplateMetaData template = templateBuilder.build();
+        final ToXContent.Params params = new ToXContent.MapParams(
+            Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true", IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
         XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.JSON.xContent()));
         builder.startObject();
-        IndexTemplateMetaData.Builder.toXContentWithTypes(template, builder, ToXContent.EMPTY_PARAMS);
+        template.toXContent(builder, params);
         builder.endObject();
         try (XContentParser parser = createParser(shuffleXContent(builder))) {
-            IndexTemplateMetaData parsed = IndexTemplateMetaData.Builder.fromXContent(parser, templateName);
+            IndexTemplateMetaData parsed = IndexTemplateMetaData.fromXContent(parser, templateName);
             assertThat(parsed, equalTo(template));
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
index 19140d9c011c4..cccad052c16b3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
@@ -47,7 +47,7 @@ public static void loadTemplateIntoMap(String resource, Map<String, IndexTemplat
         final String template = loadTemplate(resource, version, versionProperty);
         try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template)) {
-            map.put(templateName, IndexTemplateMetaData.Builder.fromXContent(parser, templateName));
+            map.put(templateName, IndexTemplateMetaData.fromXContent(parser, templateName));
         } catch (IOException e) {
             // TODO: should we handle this with a thrown exception?
             logger.error("Error loading template [{}] as part of metadata upgrading", templateName);
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
index d523864c6715b..f4801a64f2fcd 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -107,10 +108,13 @@ protected void doPublish(final RestClient client, final ActionListener<Boolean>
      HttpEntity templateToHttpEntity() {
         // the internal representation of a template has type nested under mappings.
         // this uses xContent to help remove the type before sending to the remote cluster
+        ToXContent.Params params = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
         try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
             .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template.get())) {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, templateName), builder);
+            builder.startObject();
+            IndexTemplateMetaData.fromXContent(parser, templateName).toXContent(builder, params);
+            builder.endObject();
             return new StringEntity(BytesReference.bytes(builder).utf8ToString(), ContentType.APPLICATION_JSON);
         } catch (IOException ex) {
             throw new IllegalStateException("Cannot serialize template [" + templateName + "] for monitoring export", ex);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
index f1a3501487c06..d21dc2ac62995 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -971,7 +972,10 @@ private String getExternalTemplateRepresentation(String internalRepresentation)
         try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
             .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, internalRepresentation)) {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, ""), builder);
+            ToXContent.Params params = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
+            builder.startObject();
+            IndexTemplateMetaData.fromXContent(parser, "").toXContent(builder, params);
+            builder.endObject();
             return BytesReference.bytes(builder).utf8ToString();
         }
     }

From b5db9657d1eadce9902041e5b128bf32c02d302a Mon Sep 17 00:00:00 2001
From: Andrei Stefan <astefan@users.noreply.github.com>
Date: Tue, 24 Mar 2020 14:54:32 +0200
Subject: [PATCH 277/285] SQL: jdbc debugging enhancement (#53880)

* add flush always output option that will flush the output printer
after each debug message when enabled (disabled by default)
* at debug output initializationtime, log debug output
information about OS, JVM and default JVM timezone
---
 .../elasticsearch/xpack/sql/jdbc/Debug.java   | 15 ++++++----
 .../xpack/sql/jdbc/DebugLog.java              | 30 +++++++++++++++++--
 .../xpack/sql/jdbc/JdbcConfiguration.java     | 13 +++++++-
 .../sql/jdbc/JdbcConfigurationTests.java      | 17 +++++++++++
 4 files changed, 67 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java
index cb10395e3925c..7f4385fe01518 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java
@@ -113,10 +113,9 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
             synchronized (Debug.class) {
                 log = OUTPUT_MANAGED.get(managedPrinter);
                 if (log == null) {
-                    log = new DebugLog(managedPrinter);
+                    log = createLog(managedPrinter, info.flushAlways());
                     OUTPUT_MANAGED.put(managedPrinter, log);
                 }
-                return log;
             }
         }
 
@@ -135,7 +134,7 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
                 ERR = null;
             }
             if (ERR == null) {
-                ERR = new DebugLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)));
+                ERR = createLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)), info.flushAlways());
             }
             return ERR;
         }
@@ -154,7 +153,7 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
             }
 
             if (OUT == null) {
-                OUT = new DebugLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)));
+                OUT = createLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)), info.flushAlways());
             }
             return OUT;
         }
@@ -165,7 +164,7 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
                 // must be local file
                 try {
                     PrintWriter print = new PrintWriter(Files.newBufferedWriter(Paths.get("").resolve(out), StandardCharsets.UTF_8));
-                    log = new DebugLog(print);
+                    log = createLog(print, info.flushAlways());
                     OUTPUT_CACHE.put(out, log);
                     OUTPUT_REFS.put(out, Integer.valueOf(0));
                 } catch (Exception ex) {
@@ -178,6 +177,12 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
         return log;
     }
 
+    private static DebugLog createLog(PrintWriter print, boolean flushAlways) {
+        DebugLog log = new DebugLog(print, flushAlways);
+        log.logSystemInfo();
+        return log;
+    }
+
     static void release(JdbcConfiguration info) {
         if (!info.debug()) {
             return;
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java
index e439502e971fa..c3f5598d16261 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java
@@ -17,9 +17,11 @@ final class DebugLog {
     private static final String HEADER = "%tF/%tT.%tL - ";
 
     final PrintWriter print;
+    private boolean flushAlways;
 
-    DebugLog(PrintWriter print) {
+    DebugLog(PrintWriter print, boolean flushAlways) {
         this.print = print;
+        this.flushAlways = flushAlways;
     }
 
     void logMethod(Method m, Object[] args) {
@@ -31,9 +33,11 @@ void logMethod(Method m, Object[] args) {
                 m.getName(),
                 //array(m.getParameterTypes()),
                 array(args));
+        if (flushAlways) {
+            print.flush();
+        }
     }
 
-
     void logResult(Method m, Object[] args, Object r) {
         long time = System.currentTimeMillis();
         print.printf(Locale.ROOT, HEADER + "%s#%s(%s) returned %s%n",
@@ -44,6 +48,9 @@ void logResult(Method m, Object[] args, Object r) {
                 //array(m.getParameterTypes()),
                 array(args),
                 r);
+        if (flushAlways) {
+            print.flush();
+        }
     }
 
     void logException(Method m, Object[] args, Throwable t) {
@@ -57,6 +64,25 @@ void logException(Method m, Object[] args, Throwable t) {
         print.flush();
     }
 
+    void logSystemInfo() {
+        long time = System.currentTimeMillis();
+        print.printf(Locale.ROOT, HEADER + "OS[%s/%s/%s], JVM[%s/%s/%s/%s]",
+                time, time, time,
+                System.getProperty("os.name"),
+                System.getProperty("os.version"),
+                System.getProperty("os.arch"),
+                System.getProperty("java.vm.vendor"),
+                System.getProperty("java.vm.name"),
+                System.getProperty("java.version"),
+                System.getProperty("java.vm.version"));
+        print.println();
+        time = System.currentTimeMillis();
+        print.printf(Locale.ROOT, HEADER + "JVM default timezone: %s",
+                time, time, time,
+                java.util.TimeZone.getDefault().toString());
+        print.println();
+        print.flush();
+    }
 
     private static String array(Object[] a) {
         if (a == null || a.length == 0) {
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
index 8fd4c6119b434..04ba6046229b1 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
@@ -47,6 +47,10 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     // can be out/err/url
     static final String DEBUG_OUTPUT_DEFAULT = "err";
 
+    static final String DEBUG_FLUSH_ALWAYS = "debug.flushAlways";
+    // can be buffered/immediate
+    static final String DEBUG_FLUSH_ALWAYS_DEFAULT = "false";
+
     public static final String TIME_ZONE = "timezone";
     // follow the JDBC spec and use the JVM default...
     // to avoid inconsistency, the default is picked up once at startup and reused across connections
@@ -63,7 +67,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
 
     // options that don't change at runtime
     private static final Set<String> OPTION_NAMES = new LinkedHashSet<>(
-            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT));
+            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT, DEBUG_FLUSH_ALWAYS));
 
     static {
         // trigger version initialization
@@ -76,6 +80,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     // immutable properties
     private final boolean debug;
     private final String debugOut;
+    private final boolean flushAlways;
 
     // mutable ones
     private ZoneId zoneId;
@@ -158,6 +163,8 @@ private JdbcConfiguration(URI baseURI, String u, Properties props) throws JdbcSQ
 
         this.debug = parseValue(DEBUG, props.getProperty(DEBUG, DEBUG_DEFAULT), Boolean::parseBoolean);
         this.debugOut = props.getProperty(DEBUG_OUTPUT, DEBUG_OUTPUT_DEFAULT);
+        this.flushAlways = parseValue(DEBUG_FLUSH_ALWAYS, props.getProperty(DEBUG_FLUSH_ALWAYS, DEBUG_FLUSH_ALWAYS_DEFAULT),
+                Boolean::parseBoolean);
 
         this.zoneId = parseValue(TIME_ZONE, props.getProperty(TIME_ZONE, TIME_ZONE_DEFAULT),
                 s -> TimeZone.getTimeZone(s).toZoneId().normalized());
@@ -184,6 +191,10 @@ public String debugOut() {
         return debugOut;
     }
 
+    public boolean flushAlways() {
+        return flushAlways;
+    }
+
     public TimeZone timeZone() {
         return zoneId != null ? TimeZone.getTimeZone(zoneId) : null;
     }
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java
index 9b7eb9262aac1..680b1eebe4883 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java
@@ -75,6 +75,23 @@ public void testDebugOut() throws Exception {
         assertThat(ci.debugOut(), is("jdbc.out"));
     }
 
+    public void testDebugFlushAlways() throws Exception {
+        JdbcConfiguration ci = ci("jdbc:es://a:1/?debug=true&debug.flushAlways=false");
+        assertThat(ci.baseUri().toString(), is("http://a:1/"));
+        assertThat(ci.debug(), is(true));
+        assertThat(ci.flushAlways(), is(false));
+
+        ci = ci("jdbc:es://a:1/?debug=true&debug.flushAlways=true");
+        assertThat(ci.baseUri().toString(), is("http://a:1/"));
+        assertThat(ci.debug(), is(true));
+        assertThat(ci.flushAlways(), is(true));
+
+        ci = ci("jdbc:es://a:1/?debug=true");
+        assertThat(ci.baseUri().toString(), is("http://a:1/"));
+        assertThat(ci.debug(), is(true));
+        assertThat(ci.flushAlways(), is(false));
+    }
+
     public void testTypeInParam() throws Exception {
         Exception e = expectThrows(JdbcSQLException.class, () -> ci("jdbc:es://a:1/foo/bar/tar?debug=true&debug.out=jdbc.out"));
         assertEquals("Unknown parameter [debug.out]; did you mean [debug.output]", e.getMessage());

From 99278644a6cefb55b770869106e6fc67183a53f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= <przemyslaw.witek@elastic.co>
Date: Tue, 24 Mar 2020 14:19:58 +0100
Subject: [PATCH 278/285] Use the new ML state index name (.ml-state-000001)
 instead of the legacy one (.ml-state) (#54070)

---
 .../org/elasticsearch/client/MachineLearningIT.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 17a9c0cd1cfca..a1204823d749b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -874,7 +874,7 @@ private  String createExpiredData(String jobId) throws Exception {
         {
             // Index a randomly named unused state document
             String docId = "non_existing_job_" + randomFrom("model_state_1234567#1", "quantiles", "categorizer_state#1");
-            IndexRequest indexRequest = new IndexRequest(".ml-state").id(docId);
+            IndexRequest indexRequest = new IndexRequest(".ml-state-000001").id(docId);
             indexRequest.source(Collections.emptyMap(), XContentType.JSON);
             indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
             highLevelClient().index(indexRequest, RequestOptions.DEFAULT);
@@ -944,8 +944,8 @@ public void testDeleteExpiredData() throws Exception {
         assertTrue(forecastExists(jobId, forecastId));
 
         {
-            // Verify .ml-state contains the expected unused state document
-            Iterable<SearchHit> hits = searchAll(".ml-state");
+            // Verify .ml-state* contains the expected unused state document
+            Iterable<SearchHit> hits = searchAll(".ml-state*");
             List<SearchHit> target = new ArrayList<>();
             hits.forEach(target::add);
             long numMatches = target.stream()
@@ -974,8 +974,8 @@ public void testDeleteExpiredData() throws Exception {
         assertFalse(forecastExists(jobId, forecastId));
 
         {
-            // Verify .ml-state doesn't contain unused state documents
-            Iterable<SearchHit> hits = searchAll(".ml-state");
+            // Verify .ml-state* doesn't contain unused state documents
+            Iterable<SearchHit> hits = searchAll(".ml-state*");
             List<SearchHit> hitList = new ArrayList<>();
             hits.forEach(hitList::add);
             long numMatches = hitList.stream()

From 609b06411be4298802516979f62b7e052031092a Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 24 Mar 2020 14:21:00 +0100
Subject: [PATCH 279/285] Wait for Active license before running CCR API tests
 (#53966)

DocsClientYamlTestSuiteIT sometimes fails for CCR
related tests because tests are started before the license
is fully applied and active within the cluster. The first
tests to be executed then fails with the error noticed
in #53430. This can be easily reproduced locally by
only running CCR docs tests.

This commit adds some @Before logic in
DocsClientYamlTestSuiteIT so that it waits for the
license to be active before running CCR tests.

Closes #53430
---
 .../smoketest/DocsClientYamlTestSuiteIT.java  | 14 +++++++-
 .../test/rest/ESRestTestCase.java             | 33 +++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index 249101cfc54a3..069945f1b7474 100644
--- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -22,7 +22,6 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
 import org.apache.http.HttpHost;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.BytesRef;
@@ -46,6 +45,7 @@
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -102,6 +102,13 @@ protected ClientYamlTestClient initClientYamlTestClient(
         return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
+    @Before
+    public void waitForRequirements() throws Exception {
+        if (isCcrTest()) {
+            ESRestTestCase.waitForActiveLicense(adminClient());
+        }
+    }
+
     @After
     public void cleanup() throws Exception {
         if (isMachineLearningTest() || isTransformTest()) {
@@ -163,6 +170,11 @@ protected boolean isTransformTest() {
         return testName != null && (testName.contains("/transform/") || testName.contains("\\transform\\"));
     }
 
+    protected boolean isCcrTest() {
+        String testName = getTestName();
+        return testName != null && testName.contains("/ccr/");
+    }
+
     /**
      * Compares the results of running two analyzers against many random
      * strings. The goal is to figure out if two anlayzers are "the same" by
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 445fa98af651f..8fe7464087408 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -22,6 +22,7 @@
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContexts;
@@ -99,6 +100,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.in;
+import static org.hamcrest.Matchers.notNullValue;
 
 /**
  * Superclass for tests that interact with an external test cluster using Elasticsearch's {@link RestClient}.
@@ -1297,4 +1299,35 @@ private void ensureGlobalCheckpointSynced(String index) throws Exception {
                 });
         }, 60, TimeUnit.SECONDS);
     }
+
+    /**
+     * Wait for the license to be applied and active. The specified admin client is used to check the license and this is done using
+     * {@link ESTestCase#assertBusy(CheckedRunnable)} to give some time to the License to be applied on nodes.
+     *
+     * @param restClient the client to use
+     * @throws Exception if an exception is thrown while checking the status of the license
+     */
+    protected static void waitForActiveLicense(final RestClient restClient) throws Exception {
+        assertBusy(() -> {
+            final Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder());
+
+            final Response response = restClient.performRequest(request);
+            assertOK(response);
+
+            try (InputStream is = response.getEntity().getContent()) {
+                XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue());
+                final Map<String, ?> map = XContentHelper.convertToMap(xContentType.xContent(), is, true);
+                assertThat(map, notNullValue());
+                assertThat("License must exist", map.containsKey("license"), equalTo(true));
+                @SuppressWarnings("unchecked")
+                final Map<String, ?> license = (Map<String, ?>) map.get("license");
+                assertThat("Expecting non-null license", license, notNullValue());
+                assertThat("License status must exist", license.containsKey("status"), equalTo(true));
+                final String status = (String) license.get("status");
+                assertThat("Expecting non-null license status", status, notNullValue());
+                assertThat("Expecting active license", status, equalTo("active"));
+            }
+        });
+    }
 }

From c1d8341e6e045ca8f7f39856c3a17b49056df873 Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim.vernum@elastic.co>
Date: Wed, 25 Mar 2020 00:35:39 +1100
Subject: [PATCH 280/285] Add SAML IdP plugin for internal use (#54046)

This change merges the "feature-internal-idp" branch into Elasticsearch.

This introduces a small identity-provider plugin as a child of the x-pack module.
This allows ES to act as a SAML IdP, for users who are authenticated against the
Elasticsearch cluster.

This feature is intended for internal use within Elastic Cloud environments
and is not supported for any other use case. It falls under an enterprise license tier.

The IdP is disabled by default.

Co-authored-by: Ioannis Kakavas <ioannis@elastic.co>
---
 .../xpack/core/ClientHelper.java              |   1 +
 .../xpack/core/security/SecurityContext.java  |  16 +
 .../support/SecondaryAuthentication.java      |   5 +
 .../support/RestorableContextClassLoader.java |   2 +-
 .../plugin-metadata/plugin-security.policy    |   4 +
 .../idp/saml-service-provider-template.json   |  99 +++
 x-pack/plugin/identity-provider/build.gradle  | 368 +++++++++
 .../forbidden/xml-signatures.txt              |   8 +
 .../licenses/cryptacular-1.2.4.jar.sha1       |   1 +
 .../licenses/cryptacular-LICENSE.txt          | 202 +++++
 .../licenses/cryptacular-NOTICE.txt           | 710 ++++++++++++++++++
 .../licenses/guava-19.0.jar.sha1              |   1 +
 .../licenses/guava-LICENSE.txt                | 202 +++++
 .../licenses/guava-NOTICE.txt                 |   0
 .../licenses/httpclient-LICENSE.txt           | 558 ++++++++++++++
 .../licenses/httpclient-NOTICE.txt            |   6 +
 .../licenses/httpclient-cache-4.5.10.jar.sha1 |   1 +
 .../licenses/java-support-7.5.1.jar.sha1      |   1 +
 .../licenses/log4j-slf4j-impl-2.11.1.jar.sha1 |   1 +
 .../licenses/log4j-slf4j-impl-LICENSE.txt     | 202 +++++
 .../licenses/log4j-slf4j-impl-NOTICE.txt      |   8 +
 .../licenses/metrics-core-3.2.2.jar.sha1      |   1 +
 .../licenses/metrics-core-LICENSE.txt         | 202 +++++
 .../licenses/metrics-core-NOTICE.txt          |   4 +
 .../licenses/opensaml-core-3.4.5.jar.sha1     |   1 +
 .../opensaml-messaging-api-3.4.5.jar.sha1     |   1 +
 .../opensaml-messaging-impl-3.4.5.jar.sha1    |   1 +
 .../opensaml-profile-api-3.4.5.jar.sha1       |   1 +
 .../opensaml-profile-impl-3.4.5.jar.sha1      |   1 +
 .../licenses/opensaml-saml-api-3.4.5.jar.sha1 |   1 +
 .../opensaml-saml-impl-3.4.5.jar.sha1         |   1 +
 .../opensaml-security-api-3.4.5.jar.sha1      |   1 +
 .../opensaml-security-impl-3.4.5.jar.sha1     |   1 +
 .../licenses/opensaml-soap-api-3.4.5.jar.sha1 |   1 +
 .../opensaml-soap-impl-3.4.5.jar.sha1         |   1 +
 .../opensaml-storage-api-3.4.5.jar.sha1       |   1 +
 .../opensaml-storage-impl-3.4.5.jar.sha1      |   1 +
 .../opensaml-xmlsec-api-3.4.5.jar.sha1        |   1 +
 .../opensaml-xmlsec-impl-3.4.5.jar.sha1       |   1 +
 .../licenses/shibboleth-LICENSE.txt           | 202 +++++
 .../licenses/shibboleth-NOTICE.txt            |   0
 .../licenses/slf4j-api-1.6.2.jar.sha1         |   1 +
 .../licenses/slf4j-api-LICENSE.txt            |  21 +
 .../licenses/slf4j-api-NOTICE.txt             |   0
 .../licenses/xmlsec-2.1.4.jar.sha1            |   1 +
 .../licenses/xmlsec-LICENSE.txt               | 202 +++++
 .../licenses/xmlsec-NOTICE.txt                |   9 +
 .../plugin/identity-provider/qa/build.gradle  |   0
 .../qa/idp-rest-tests/build.gradle            |  33 +
 .../xpack/idp/IdpRestTestCase.java            |  32 +
 .../idp/ManageServiceProviderRestIT.java      | 138 ++++
 .../src/test/resources/idp-sign.crt           |  22 +
 .../src/test/resources/idp-sign.key           |  27 +
 .../src/test/resources/roles.yml              |   6 +
 .../xpack/idp/IdentityProviderPlugin.java     | 160 ++++
 .../DeleteSamlServiceProviderAction.java      |  22 +
 .../DeleteSamlServiceProviderRequest.java     |  79 ++
 .../DeleteSamlServiceProviderResponse.java    |  99 +++
 .../action/PutSamlServiceProviderAction.java  |  19 +
 .../action/PutSamlServiceProviderRequest.java | 138 ++++
 .../PutSamlServiceProviderResponse.java       |  97 +++
 .../SamlInitiateSingleSignOnAction.java       |  21 +
 .../SamlInitiateSingleSignOnRequest.java      |  78 ++
 .../SamlInitiateSingleSignOnResponse.java     |  51 ++
 .../xpack/idp/action/SamlMetadataAction.java  |  18 +
 .../xpack/idp/action/SamlMetadataRequest.java |  50 ++
 .../idp/action/SamlMetadataResponse.java      |  36 +
 .../SamlValidateAuthnRequestAction.java       |  18 +
 .../SamlValidateAuthnRequestRequest.java      |  57 ++
 .../SamlValidateAuthnRequestResponse.java     |  60 ++
 ...nsportDeleteSamlServiceProviderAction.java |  62 ++
 ...TransportPutSamlServiceProviderAction.java | 120 +++
 ...ansportSamlInitiateSingleSignOnAction.java | 146 ++++
 .../action/TransportSamlMetadataAction.java   |  37 +
 ...ansportSamlValidateAuthnRequestAction.java |  42 ++
 .../xpack/idp/authc/AuthenticationMethod.java |  17 +
 .../xpack/idp/authc/NetworkControl.java       |  15 +
 .../privileges/ServiceProviderPrivileges.java |  55 ++
 .../idp/privileges/UserPrivilegeResolver.java | 127 ++++
 ...dAuthenticationResponseMessageBuilder.java | 108 +++
 .../saml/authn/SamlAuthnRequestValidator.java | 354 +++++++++
 ...lAuthenticationResponseMessageBuilder.java | 274 +++++++
 .../saml/authn/UserServiceAuthentication.java |  85 +++
 .../idp/saml/idp/SamlIdPMetadataBuilder.java  | 264 +++++++
 .../idp/saml/idp/SamlIdentityProvider.java    | 208 +++++
 .../saml/idp/SamlIdentityProviderBuilder.java | 356 +++++++++
 .../idp/saml/idp/SamlMetadataGenerator.java   | 104 +++
 .../saml/rest/action/IdpBaseRestHandler.java  |  50 ++
 .../RestDeleteSamlServiceProviderAction.java  |  60 ++
 .../RestPutSamlServiceProviderAction.java     |  63 ++
 .../RestSamlInitiateSingleSignOnAction.java   |  77 ++
 .../rest/action/RestSamlMetadataAction.java   |  58 ++
 ...mlValidateAuthenticationRequestAction.java |  72 ++
 .../idp/saml/sp/CloudServiceProvider.java     | 116 +++
 .../idp/saml/sp/SamlServiceProvider.java      |  56 ++
 .../saml/sp/SamlServiceProviderDocument.java  | 539 +++++++++++++
 .../idp/saml/sp/SamlServiceProviderIndex.java | 337 +++++++++
 .../saml/sp/SamlServiceProviderResolver.java  | 151 ++++
 .../idp/saml/sp/ServiceProviderDefaults.java  |  59 ++
 .../idp/saml/sp/ServiceProviderException.java |  24 +
 .../saml/support/SamlAuthenticationState.java | 151 ++++
 .../xpack/idp/saml/support/SamlFactory.java   | 359 +++++++++
 .../xpack/idp/saml/support/SamlInit.java      |  54 ++
 .../idp/saml/support/SamlObjectSigner.java    |  58 ++
 .../xpack/idp/saml/support/XmlValidator.java  |  95 +++
 .../plugin-metadata/plugin-security.policy    |  29 +
 .../support/saml-schema-assertion-2.0.xsd     | 283 +++++++
 .../saml/support/saml-schema-metadata-2.0.xsd | 337 +++++++++
 .../saml/support/saml-schema-protocol-2.0.xsd | 302 ++++++++
 .../xpack/idp/saml/support/xenc-schema.xsd    | 135 ++++
 .../xpack/idp/saml/support/xml.xsd            | 286 +++++++
 .../idp/saml/support/xmldsig-core-schema.xsd  | 308 ++++++++
 .../idp/LocalStateIdentityProviderPlugin.java |  41 +
 ...DeleteSamlServiceProviderRequestTests.java |  33 +
 ...eleteSamlServiceProviderResponseTests.java |  38 +
 .../PutSamlServiceProviderRequestTests.java   | 154 ++++
 .../idp/action/SamlIdentityProviderTests.java | 349 +++++++++
 .../SamlInitiateSingleSignOnRequestTests.java |  37 +
 .../SamlValidateAuthnRequestRequestTests.java |  36 +
 ...portPutSamlServiceProviderActionTests.java | 174 +++++
 ...tSamlInitiateSingleSignOnRequestTests.java | 175 +++++
 .../UserPrivilegeResolverTests.java           | 168 +++++
 ...ledAuthenticationResponseBuilderTests.java |  63 ++
 .../authn/SamlAuthnRequestValidatorTests.java | 312 ++++++++
 ...enticationResponseMessageBuilderTests.java |  79 ++
 .../idp/SamlIdentityProviderBuilderTests.java | 536 +++++++++++++
 .../saml/idp/SamlIdpMetadataBuilderTests.java | 177 +++++
 .../saml/idp/SamlMetadataGeneratorTests.java  | 132 ++++
 .../rest/action/IdpBaseRestHandlerTests.java  |  58 ++
 .../sp/SamlServiceProviderDocumentTests.java  | 162 ++++
 .../sp/SamlServiceProviderIndexTests.java     | 299 ++++++++
 .../sp/SamlServiceProviderResolverTests.java  | 169 +++++
 .../support/SamlAuthenticationStateTests.java | 110 +++
 .../saml/support/SamlObjectSignerTests.java   | 166 ++++
 .../test/IdentityProviderIntegTestCase.java   | 297 ++++++++
 .../xpack/idp/saml/test/IdpSamlTestCase.java  | 186 +++++
 .../src/test/resources/keypair/README.txt     |   6 +
 .../resources/keypair/keypair_RSA2_4096.crt   |  29 +
 .../resources/keypair/keypair_RSA2_4096.key   |  52 ++
 .../resources/keypair/keypair_RSA_1024.crt    |  15 +
 .../resources/keypair/keypair_RSA_1024.key    |  15 +
 .../resources/keypair/keypair_RSA_2048.crt    |  21 +
 .../resources/keypair/keypair_RSA_2048.key    |  27 +
 .../resources/keypair/keypair_RSA_4096.crt    |  29 +
 .../resources/keypair/keypair_RSA_4096.key    |  52 ++
 .../xpack/idp/saml/idp/README.txt             |  24 +
 .../xpack/idp/saml/idp/multi_signing.jks      | Bin 0 -> 6640 bytes
 .../xpack/idp/saml/idp/multi_signing.p12      | Bin 0 -> 7462 bytes
 .../xpack/idp/saml/idp/signing.p12            | Bin 0 -> 2419 bytes
 .../xpack/idp/saml/idp/signing1.crt           |  17 +
 .../xpack/idp/saml/idp/signing1.key           |  28 +
 .../xpack/idp/saml/idp/signing2.crt           |  17 +
 .../xpack/idp/saml/idp/signing2.key           |  28 +
 .../xpack/idp/saml/idp/signing3.crt           |  17 +
 .../xpack/idp/saml/idp/signing3.key           |  28 +
 .../xpack/idp/saml/idp/signing4.crt           |   9 +
 .../xpack/idp/saml/idp/signing4.key           |   4 +
 .../authc/saml/SamlAuthenticator.java         |   4 +-
 .../xpack/security/authc/saml/SamlRealm.java  |   6 +-
 .../authc/saml/SamlRequestHandler.java        |  24 +-
 .../xpack/security/authc/saml/SamlUtils.java  |   8 +-
 .../security/authz/AuthorizationUtils.java    |   2 +
 .../plugin-metadata/plugin-security.policy    |   3 +-
 163 files changed, 15145 insertions(+), 19 deletions(-)
 rename x-pack/plugin/{security/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/security/support/RestorableContextClassLoader.java (96%)
 create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json
 create mode 100644 x-pack/plugin/identity-provider/build.gradle
 create mode 100644 x-pack/plugin/identity-provider/forbidden/xml-signatures.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/guava-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/shibboleth-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/slf4j-api-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/qa/build.gradle
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.jks
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.p12
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing.p12
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index ed03ab9ffc058..49a457f0a7f55 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -53,6 +53,7 @@ public final class ClientHelper {
     public static final String ENRICH_ORIGIN = "enrich";
     public static final String TRANSFORM_ORIGIN = "transform";
     public static final String ASYNC_SEARCH_ORIGIN = "async_search";
+    public static final String IDP_ORIGIN = "idp";
 
     private ClientHelper() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
index a12f12ae53017..53704147438a6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
@@ -7,7 +7,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
@@ -41,13 +43,27 @@ public SecurityContext(Settings settings, ThreadContext threadContext) {
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
     }
 
+    /**
+     * Returns the current user information, or throws {@link org.elasticsearch.ElasticsearchSecurityException}
+     * if the current request has no authentication information.
+     */
+    public User requireUser() {
+        User user = getUser();
+        if (user == null) {
+            throw new ElasticsearchSecurityException("there is no user available in the current context");
+        }
+        return user;
+    }
+
     /** Returns the current user information, or null if the current request has no authentication info. */
+    @Nullable
     public User getUser() {
         Authentication authentication = getAuthentication();
         return authentication == null ? null : authentication.getUser();
     }
 
     /** Returns the authentication information, or null if the current request has no authentication info. */
+    @Nullable
     public Authentication getAuthentication() {
         try {
             return authenticationSerializer.readFromContext(threadContext);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java
index 8f032f480e07f..213c94e7dad3f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.user.User;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -55,6 +56,10 @@ public Authentication getAuthentication() {
         return authentication;
     }
 
+    public User getUser() {
+        return authentication.getUser();
+    }
+
     public <T> T execute(Function<ThreadContext.StoredContext, T> body) {
         return this.securityContext.executeWithAuthentication(this.authentication, body);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RestorableContextClassLoader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
similarity index 96%
rename from x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RestorableContextClassLoader.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
index 95c68eab9c28a..f3c7ec44a5249 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RestorableContextClassLoader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.support;
+package org.elasticsearch.xpack.core.security.support;
 
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
index 70abb67a76240..7a848345ec96b 100644
--- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
@@ -2,6 +2,10 @@ grant {
   // bouncy castle
   permission java.security.SecurityPermission "putProviderProperty.BC";
 
+  // needed in (cf. o.e.x.c.s.s.RestorableContextClassLoader)
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "setContextClassLoader";
+
   // needed for x-pack security extension
   permission java.security.SecurityPermission "createPolicy.JavaPolicy";
   permission java.security.SecurityPermission "getPolicy";
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json
new file mode 100644
index 0000000000000..8dfbcd246615b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json
@@ -0,0 +1,99 @@
+{
+  "index_patterns": [
+    "saml-service-provider-*"
+  ],
+  "aliases": {
+    "saml-service-provider": {}
+  },
+  "order": 100,
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "auto_expand_replicas": "0-1",
+    "index.priority": 10,
+    "index.refresh_interval": "1s",
+    "index.format": 1
+  },
+  "mappings": {
+    "_doc": {
+      "_meta": {
+        "idp-version": "${idp.template.version}"
+      },
+      "dynamic": "strict",
+      "properties": {
+        "name": {
+          "type": "text"
+        },
+        "entity_id": {
+          "type": "keyword"
+        },
+        "acs": {
+          "type": "keyword"
+        },
+        "enabled": {
+          "type": "boolean"
+        },
+        "created": {
+          "type": "date",
+          "format": "epoch_millis"
+        },
+        "last_modified": {
+          "type": "date",
+          "format": "epoch_millis"
+        },
+        "name_id_format": {
+          "type": "keyword"
+        },
+        "sign_messages": {
+          "type": "keyword"
+        },
+        "authn_expiry_ms": {
+          "type": "long"
+        },
+        "privileges": {
+          "type": "object",
+          "properties": {
+            "resource": {
+              "type": "keyword"
+            },
+            "roles": {
+              "type": "object",
+              "dynamic": false
+            }
+          }
+        },
+        "attributes": {
+          "type": "object",
+          "properties": {
+            "principal": {
+              "type": "keyword"
+            },
+            "email": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            },
+            "roles": {
+              "type": "keyword"
+            }
+          }
+        },
+        "certificates": {
+          "type": "object",
+          "properties": {
+            "sp_signing": {
+              "type": "text"
+            },
+            "idp_signing": {
+              "type": "text"
+            },
+            "idp_metadata": {
+              "type": "text"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle
new file mode 100644
index 0000000000000..4dab2c984b529
--- /dev/null
+++ b/x-pack/plugin/identity-provider/build.gradle
@@ -0,0 +1,368 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'nebula.maven-scm'
+esplugin {
+  name 'x-pack-identity-provider'
+  description 'Elasticsearch Expanded Pack Plugin - Identity Provider'
+  classname 'org.elasticsearch.xpack.idp.IdentityProviderPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+archivesBaseName = 'x-pack-identity-provider'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+  // So that we can extend LocalStateCompositeXPackPlugin
+  testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+
+  // the following are all SAML dependencies - might as well download the whole internet
+  compile "org.opensaml:opensaml-core:3.4.5"
+  compile "org.opensaml:opensaml-saml-api:3.4.5"
+  compile "org.opensaml:opensaml-saml-impl:3.4.5"
+  compile "org.opensaml:opensaml-messaging-api:3.4.5"
+  compile "org.opensaml:opensaml-messaging-impl:3.4.5"
+  compile "org.opensaml:opensaml-security-api:3.4.5"
+  compile "org.opensaml:opensaml-security-impl:3.4.5"
+  compile "org.opensaml:opensaml-profile-api:3.4.5"
+  compile "org.opensaml:opensaml-profile-impl:3.4.5"
+  compile "org.opensaml:opensaml-xmlsec-api:3.4.5"
+  compile "org.opensaml:opensaml-xmlsec-impl:3.4.5"
+  compile "org.opensaml:opensaml-soap-api:3.4.5"
+  compile "org.opensaml:opensaml-soap-impl:3.4.5"
+  compile "org.opensaml:opensaml-storage-api:3.4.5"
+  compile "org.opensaml:opensaml-storage-impl:3.4.5"
+  compile "net.shibboleth.utilities:java-support:7.5.1"
+  compile "org.apache.santuario:xmlsec:2.1.4"
+  compile "io.dropwizard.metrics:metrics-core:3.2.2"
+  compile ("org.cryptacular:cryptacular:1.2.4") {
+      exclude group: 'org.bouncycastle'
+  }
+  compile "org.slf4j:slf4j-api:${versions.slf4j}"
+  compile "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+  compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
+  compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
+  compile "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
+  compile 'com.google.guava:guava:19.0'
+
+  testCompile 'org.elasticsearch:securemock:1.2'
+  testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
+}
+
+compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
+compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
+
+dependencyLicenses {
+  mapping from: /java-support|opensaml-.*/, to: 'shibboleth'
+  mapping from: /http.*/, to: 'httpclient'
+}
+
+forbiddenPatterns {
+  exclude '**/*.key'
+  exclude '**/*.p12'
+  exclude '**/*.der'
+  exclude '**/*.zip'
+}
+
+forbiddenApisMain {
+  signaturesFiles += files('forbidden/xml-signatures.txt')
+}
+
+// classes are missing, e.g. com.ibm.icu.lang.UCharacter
+thirdPartyAudit {
+    ignoreMissingClasses (
+        // SAML dependencies
+        // [missing classes] Some cli utilities that we don't use depend on these missing JCommander classes
+        'com.beust.jcommander.JCommander',
+        'com.beust.jcommander.converters.BaseConverter',
+        // [missing classes] Shibboleth + OpenSAML have servlet support that we don't use
+        'javax.servlet.AsyncContext',
+        'javax.servlet.DispatcherType',
+        'javax.servlet.Filter',
+        'javax.servlet.FilterChain',
+        'javax.servlet.FilterConfig',
+        'javax.servlet.RequestDispatcher',
+        'javax.servlet.ServletContext',
+        'javax.servlet.ServletException',
+        'javax.servlet.ServletInputStream',
+        'javax.servlet.ServletOutputStream',
+        'javax.servlet.ServletRequest',
+        'javax.servlet.ServletResponse',
+        'javax.servlet.http.Cookie',
+        'javax.servlet.http.HttpServletRequest',
+        'javax.servlet.http.HttpServletResponse',
+        'javax.servlet.http.HttpServletResponseWrapper',
+        'javax.servlet.http.HttpSession',
+        'javax.servlet.http.Part',
+        // [missing classes] Shibboleth + OpenSAML have velocity support that we don't use
+        'org.apache.velocity.VelocityContext',
+        'org.apache.velocity.app.VelocityEngine',
+        'org.apache.velocity.context.Context',
+        'org.apache.velocity.exception.VelocityException',
+        'org.apache.velocity.runtime.RuntimeServices',
+        'org.apache.velocity.runtime.log.LogChute',
+        'org.apache.velocity.runtime.resource.loader.StringResourceLoader',
+        'org.apache.velocity.runtime.resource.util.StringResourceRepository',
+        // [missing classes] OpenSAML depends on Apache XML security which depends on Xalan, but only for functionality that OpenSAML doesn't use
+        'org.apache.xml.dtm.DTM',
+        'org.apache.xml.utils.PrefixResolver',
+        'org.apache.xml.utils.PrefixResolverDefault',
+        'org.apache.xpath.Expression',
+        'org.apache.xpath.NodeSetDTM',
+        'org.apache.xpath.XPath',
+        'org.apache.xpath.XPathContext',
+        'org.apache.xpath.compiler.FunctionTable',
+        'org.apache.xpath.functions.Function',
+        'org.apache.xpath.objects.XNodeSet',
+        'org.apache.xpath.objects.XObject',
+        // [missing classes] OpenSAML storage has an optional LDAP storage impl
+        'org.ldaptive.AttributeModification',
+        'org.ldaptive.AttributeModificationType',
+        'org.ldaptive.Connection',
+        'org.ldaptive.DeleteOperation',
+        'org.ldaptive.DeleteRequest',
+        'org.ldaptive.LdapAttribute',
+        'org.ldaptive.LdapEntry',
+        'org.ldaptive.LdapException',
+        'org.ldaptive.ModifyOperation',
+        'org.ldaptive.ModifyRequest',
+        'org.ldaptive.Response',
+        'org.ldaptive.ResultCode',
+        'org.ldaptive.SearchOperation',
+        'org.ldaptive.SearchRequest',
+        'org.ldaptive.SearchResult',
+        'org.ldaptive.ext.MergeOperation',
+        'org.ldaptive.ext.MergeRequest',
+        'org.ldaptive.pool.ConnectionPool',
+        'org.ldaptive.pool.PooledConnectionFactory',
+        // [missing classes] OpenSAML storage has an optional JSON-backed storage impl
+        'javax.json.Json',
+        'javax.json.JsonException',
+        'javax.json.JsonNumber',
+        'javax.json.JsonObject',
+        'javax.json.JsonReader',
+        'javax.json.JsonValue$ValueType',
+        'javax.json.JsonValue',
+        'javax.json.stream.JsonGenerator',
+        // [missing classes] OpenSAML storage has an optional JPA storage impl
+        'javax.persistence.EntityManager',
+        'javax.persistence.EntityManagerFactory',
+        'javax.persistence.EntityTransaction',
+        'javax.persistence.LockModeType',
+        'javax.persistence.Query',
+        // [missing classes] OpenSAML storage and HttpClient cache have optional memcache support
+        'net.spy.memcached.CASResponse',
+        'net.spy.memcached.CASValue',
+        'net.spy.memcached.MemcachedClient',
+        'net.spy.memcached.MemcachedClientIF',
+        'net.spy.memcached.CachedData',
+        'net.spy.memcached.internal.OperationFuture',
+        'net.spy.memcached.transcoders.Transcoder',
+        // [missing classes] Http Client cache has optional ehcache support
+        'net.sf.ehcache.Ehcache',
+        'net.sf.ehcache.Element',
+        // [missing classes] SLF4j includes an optional class that depends on an extension class (!)
+        'org.slf4j.ext.EventData',
+        // Bouncycastle is an optional dependency for apache directory, cryptacular and opensaml packages. We
+        // acknowledge them here instead of adding bouncy castle as a compileOnly dependency
+        'org.bouncycastle.asn1.ASN1Encodable',
+        'org.bouncycastle.asn1.ASN1InputStream',
+        'org.bouncycastle.asn1.ASN1Integer',
+        'org.bouncycastle.asn1.ASN1ObjectIdentifier',
+        'org.bouncycastle.asn1.ASN1OctetString',
+        'org.bouncycastle.asn1.ASN1Primitive',
+        'org.bouncycastle.asn1.ASN1Sequence',
+        'org.bouncycastle.asn1.ASN1TaggedObject',
+        'org.bouncycastle.asn1.DEROctetString',
+        'org.bouncycastle.asn1.DERSequence',
+        'org.bouncycastle.asn1.pkcs.EncryptedPrivateKeyInfo',
+        'org.bouncycastle.asn1.pkcs.EncryptionScheme',
+        'org.bouncycastle.asn1.pkcs.KeyDerivationFunc',
+        'org.bouncycastle.asn1.pkcs.PBEParameter',
+        'org.bouncycastle.asn1.pkcs.PBES2Parameters',
+        'org.bouncycastle.asn1.pkcs.PBKDF2Params',
+        'org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers',
+        'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+        'org.bouncycastle.asn1.x500.AttributeTypeAndValue',
+        'org.bouncycastle.asn1.x500.RDN',
+        'org.bouncycastle.asn1.x500.X500Name',
+        'org.bouncycastle.asn1.x509.AccessDescription',
+        'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+        'org.bouncycastle.asn1.x509.AuthorityKeyIdentifier',
+        'org.bouncycastle.asn1.x509.BasicConstraints',
+        'org.bouncycastle.asn1.x509.DistributionPoint',
+        'org.bouncycastle.asn1.x509.Extension',
+        'org.bouncycastle.asn1.x509.GeneralName',
+        'org.bouncycastle.asn1.x509.GeneralNames',
+        'org.bouncycastle.asn1.x509.GeneralNamesBuilder',
+        'org.bouncycastle.asn1.x509.KeyPurposeId',
+        'org.bouncycastle.asn1.x509.KeyUsage',
+        'org.bouncycastle.asn1.x509.PolicyInformation',
+        'org.bouncycastle.asn1.x509.SubjectKeyIdentifier',
+        'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+        'org.bouncycastle.asn1.x9.X9ECParameters',
+        'org.bouncycastle.cert.X509v3CertificateBuilder',
+        'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
+        'org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils',
+        'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+        'org.bouncycastle.crypto.BlockCipher',
+        'org.bouncycastle.crypto.BufferedBlockCipher',
+        'org.bouncycastle.crypto.CipherParameters',
+        'org.bouncycastle.crypto.Digest',
+        'org.bouncycastle.crypto.PBEParametersGenerator',
+        'org.bouncycastle.crypto.StreamCipher',
+        'org.bouncycastle.crypto.digests.GOST3411Digest',
+        'org.bouncycastle.crypto.digests.MD2Digest',
+        'org.bouncycastle.crypto.digests.MD4Digest',
+        'org.bouncycastle.crypto.digests.MD5Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD128Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD160Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD256Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD320Digest',
+        'org.bouncycastle.crypto.digests.SHA1Digest',
+        'org.bouncycastle.crypto.digests.SHA224Digest',
+        'org.bouncycastle.crypto.digests.SHA256Digest',
+        'org.bouncycastle.crypto.digests.SHA384Digest',
+        'org.bouncycastle.crypto.digests.SHA3Digest',
+        'org.bouncycastle.crypto.digests.SHA512Digest',
+        'org.bouncycastle.crypto.digests.TigerDigest',
+        'org.bouncycastle.crypto.digests.WhirlpoolDigest',
+        'org.bouncycastle.crypto.engines.AESEngine',
+        'org.bouncycastle.crypto.engines.BlowfishEngine',
+        'org.bouncycastle.crypto.engines.CAST5Engine',
+        'org.bouncycastle.crypto.engines.CAST6Engine',
+        'org.bouncycastle.crypto.engines.CamelliaEngine',
+        'org.bouncycastle.crypto.engines.DESEngine',
+        'org.bouncycastle.crypto.engines.DESedeEngine',
+        'org.bouncycastle.crypto.engines.GOST28147Engine',
+        'org.bouncycastle.crypto.engines.Grain128Engine',
+        'org.bouncycastle.crypto.engines.HC128Engine',
+        'org.bouncycastle.crypto.engines.HC256Engine',
+        'org.bouncycastle.crypto.engines.ISAACEngine',
+        'org.bouncycastle.crypto.engines.NoekeonEngine',
+        'org.bouncycastle.crypto.engines.RC2Engine',
+        'org.bouncycastle.crypto.engines.RC4Engine',
+        'org.bouncycastle.crypto.engines.RC532Engine',
+        'org.bouncycastle.crypto.engines.RC564Engine',
+        'org.bouncycastle.crypto.engines.RC6Engine',
+        'org.bouncycastle.crypto.engines.SEEDEngine',
+        'org.bouncycastle.crypto.engines.Salsa20Engine',
+        'org.bouncycastle.crypto.engines.SerpentEngine',
+        'org.bouncycastle.crypto.engines.SkipjackEngine',
+        'org.bouncycastle.crypto.engines.TEAEngine',
+        'org.bouncycastle.crypto.engines.TwofishEngine',
+        'org.bouncycastle.crypto.engines.VMPCEngine',
+        'org.bouncycastle.crypto.engines.XTEAEngine',
+        'org.bouncycastle.crypto.generators.BCrypt',
+        'org.bouncycastle.crypto.generators.OpenSSLPBEParametersGenerator',
+        'org.bouncycastle.crypto.generators.PKCS5S1ParametersGenerator',
+        'org.bouncycastle.crypto.generators.PKCS5S2ParametersGenerator',
+        'org.bouncycastle.crypto.io.CipherInputStream',
+        'org.bouncycastle.crypto.io.CipherOutputStream',
+        'org.bouncycastle.crypto.macs.HMac',
+        'org.bouncycastle.crypto.modes.AEADBlockCipher',
+        'org.bouncycastle.crypto.modes.CBCBlockCipher',
+        'org.bouncycastle.crypto.modes.CCMBlockCipher',
+        'org.bouncycastle.crypto.modes.CFBBlockCipher',
+        'org.bouncycastle.crypto.modes.EAXBlockCipher',
+        'org.bouncycastle.crypto.modes.GCMBlockCipher',
+        'org.bouncycastle.crypto.modes.OCBBlockCipher',
+        'org.bouncycastle.crypto.modes.OFBBlockCipher',
+        'org.bouncycastle.crypto.paddings.BlockCipherPadding',
+        'org.bouncycastle.crypto.paddings.ISO10126d2Padding',
+        'org.bouncycastle.crypto.paddings.ISO7816d4Padding',
+        'org.bouncycastle.crypto.paddings.PKCS7Padding',
+        'org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher',
+        'org.bouncycastle.crypto.paddings.TBCPadding',
+        'org.bouncycastle.crypto.paddings.X923Padding',
+        'org.bouncycastle.crypto.paddings.ZeroBytePadding',
+        'org.bouncycastle.crypto.params.AEADParameters',
+        'org.bouncycastle.crypto.params.AsymmetricKeyParameter',
+        'org.bouncycastle.crypto.params.DSAKeyParameters',
+        'org.bouncycastle.crypto.params.DSAParameters',
+        'org.bouncycastle.crypto.params.DSAPrivateKeyParameters',
+        'org.bouncycastle.crypto.params.DSAPublicKeyParameters',
+        'org.bouncycastle.crypto.params.ECDomainParameters',
+        'org.bouncycastle.crypto.params.ECKeyParameters',
+        'org.bouncycastle.crypto.params.ECPrivateKeyParameters',
+        'org.bouncycastle.crypto.params.ECPublicKeyParameters',
+        'org.bouncycastle.crypto.params.KeyParameter',
+        'org.bouncycastle.crypto.params.ParametersWithIV',
+        'org.bouncycastle.crypto.params.RC2Parameters',
+        'org.bouncycastle.crypto.params.RC5Parameters',
+        'org.bouncycastle.crypto.params.RSAKeyParameters',
+        'org.bouncycastle.crypto.params.RSAPrivateCrtKeyParameters',
+        'org.bouncycastle.crypto.prng.EntropySource',
+        'org.bouncycastle.crypto.prng.SP800SecureRandom',
+        'org.bouncycastle.crypto.prng.SP800SecureRandomBuilder',
+        'org.bouncycastle.crypto.prng.drbg.HashSP800DRBG',
+        'org.bouncycastle.crypto.prng.drbg.SP80090DRBG',
+        'org.bouncycastle.crypto.signers.DSASigner',
+        'org.bouncycastle.crypto.signers.ECDSASigner',
+        'org.bouncycastle.crypto.signers.RSADigestSigner',
+        'org.bouncycastle.crypto.util.PrivateKeyFactory',
+        'org.bouncycastle.crypto.util.PrivateKeyInfoFactory',
+        'org.bouncycastle.crypto.util.PublicKeyFactory',
+        'org.bouncycastle.crypto.util.SubjectPublicKeyInfoFactory',
+        'org.bouncycastle.jcajce.provider.asymmetric.dsa.KeyPairGeneratorSpi',
+        'org.bouncycastle.jcajce.provider.asymmetric.ec.KeyPairGeneratorSpi$EC',
+        'org.bouncycastle.jcajce.provider.asymmetric.rsa.KeyPairGeneratorSpi',
+        'org.bouncycastle.jcajce.provider.asymmetric.util.EC5Util',
+        'org.bouncycastle.jcajce.provider.asymmetric.util.ECUtil',
+        'org.bouncycastle.jce.spec.ECNamedCurveGenParameterSpec',
+        'org.bouncycastle.math.ec.ECFieldElement',
+        'org.bouncycastle.math.ec.ECPoint',
+        'org.bouncycastle.openssl.jcajce.JcaPEMWriter',
+        'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+        'org.bouncycastle.util.Arrays',
+        'org.bouncycastle.util.Strings',
+        'org.bouncycastle.util.io.Streams',
+        'org.bouncycastle.x509.extension.X509ExtensionUtil'
+    )
+
+  ignoreViolations(
+    // Guava uses internal java api: sun.misc.Unsafe
+    'com.google.common.cache.Striped64',
+    'com.google.common.cache.Striped64$1',
+    'com.google.common.cache.Striped64$Cell',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+  )
+}
+
+thirdPartyAudit.ignoreMissingClasses(
+  'javax.xml.bind.JAXBContext',
+  'javax.xml.bind.JAXBElement',
+  'javax.xml.bind.JAXBException',
+  'javax.xml.bind.Unmarshaller',
+  'javax.xml.bind.UnmarshallerHandler',
+)
+
+
+test {
+  /*
+   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+   * other if we allow them to set the number of available processors as it's set-once in Netty.
+   */
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+// xpack modules are installed in real clusters as the meta plugin, so
+// installing them as individual plugins for integ tests doesn't make sense,
+// so we disable integ tests
+integTest.enabled = false
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+  project.subprojects
+    .find { it.path == project.path + ":qa" }
+    .subprojects
+    .findAll { it.path.startsWith(project.path + ":qa") }
+    .each { check.dependsOn it.check }
+}
+
diff --git a/x-pack/plugin/identity-provider/forbidden/xml-signatures.txt b/x-pack/plugin/identity-provider/forbidden/xml-signatures.txt
new file mode 100644
index 0000000000000..0acb0db2da1a7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/forbidden/xml-signatures.txt
@@ -0,0 +1,8 @@
+@defaultMessage DocumentBuilderFactory should not be used directly. (See x-pack-security SamlUtils#getHardenedDocumentBuilder if you ever need one)
+javax.xml.parsers.DocumentBuilderFactory#newInstance()
+javax.xml.parsers.DocumentBuilderFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+
+@defaultMessage TransformerFactory should not be used directly. Use IdPSamlTestCase#getHardenedXMLTransformer() instead.
+javax.xml.transform.TransformerFactory#newInstance()
+javax.xml.transform.TransformerFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+
diff --git a/x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1 b/x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1
new file mode 100644
index 0000000000000..19095bb5dff64
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1
@@ -0,0 +1 @@
+4994c015d87886212683245d13e87f6fb903a760
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt
new file mode 100644
index 0000000000000..f17da0d736217
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt
@@ -0,0 +1,710 @@
+
+
+
+
+
+
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+  <link rel="dns-prefetch" href="https://assets-cdn.github.com">
+  <link rel="dns-prefetch" href="https://avatars0.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://avatars1.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://avatars2.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://avatars3.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
+  <link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
+
+
+
+  <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/frameworks-2d2d4c150f7000385741c6b992b302689ecd172246c6428904e0813be9bceca6.css" media="all" rel="stylesheet" />
+  <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/github-0522ae8d3b3bdc841d2f91f90efd5f1fd9040d910905674cd134ced43a6dfea6.css" media="all" rel="stylesheet" />
+  
+  
+  <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/site-cfab053e93f0e27f4c63d4ff6b7957bd25f711667fe678e747f8a4d88c47b38d.css" media="all" rel="stylesheet" />
+  
+
+  <meta name="viewport" content="width=device-width">
+  
+  <title>cryptacular/NOTICE at master · vt-middleware/cryptacular · GitHub</title>
+  <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">
+  <link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
+  <meta property="fb:app_id" content="1401488693436528">
+
+    
+    <meta content="https://avatars7.githubusercontent.com/u/6122907?v=4&amp;s=400" property="og:image" /><meta content="GitHub" property="og:site_name" /><meta content="object" property="og:type" /><meta content="vt-middleware/cryptacular" property="og:title" /><meta content="https://github.com/vt-middleware/cryptacular" property="og:url" /><meta content="cryptacular - The friendly complement to the BouncyCastle crypto API for Java." property="og:description" />
+
+  <link rel="assets" href="https://assets-cdn.github.com/">
+  
+  <meta name="pjax-timeout" content="1000">
+  
+  <meta name="request-id" content="E0E4:26F16:12A5AE6:1D11801:596C7978" data-pjax-transient>
+  
+
+  <meta name="selected-link" value="repo_source" data-pjax-transient>
+
+  <meta name="google-site-verification" content="KT5gs8h0wvaagLKAVWq8bbeNwnZZK1r1XQysX3xurLU">
+<meta name="google-site-verification" content="ZzhVyEFwb7w3e0-uOTltm8Jsck2F5StVihD0exw2fsA">
+    <meta name="google-analytics" content="UA-3769691-2">
+
+<meta content="collector.githubapp.com" name="octolytics-host" /><meta content="github" name="octolytics-app-id" /><meta content="https://collector.githubapp.com/github-external/browser_event" name="octolytics-event-url" /><meta content="E0E4:26F16:12A5AE6:1D11801:596C7978" name="octolytics-dimension-request_id" /><meta content="sea" name="octolytics-dimension-region_edge" /><meta content="iad" name="octolytics-dimension-region_render" />
+<meta content="/&lt;user-name&gt;/&lt;repo-name&gt;/blob/show" data-pjax-transient="true" name="analytics-location" />
+
+
+
+
+  <meta class="js-ga-set" name="dimension1" content="Logged Out">
+
+
+  
+
+      <meta name="hostname" content="github.com">
+  <meta name="user-login" content="">
+
+      <meta name="expected-hostname" content="github.com">
+    <meta name="js-proxy-site-detection-payload" content="N2ZhMjk0NTA4MjI1NmZhYTVlNzM5NzVjZmFkOWY2NGFkNmMxYzcyMGViNzAzZGQxMGMzZmJhZDQ3YWZiZTI0OHx7InJlbW90ZV9hZGRyZXNzIjoiMTEwLjIwLjIyMC4xMzUiLCJyZXF1ZXN0X2lkIjoiRTBFNDoyNkYxNjoxMkE1QUU2OjFEMTE4MDE6NTk2Qzc5NzgiLCJ0aW1lc3RhbXAiOjE1MDAyODEyMDgsImhvc3QiOiJnaXRodWIuY29tIn0=">
+
+
+  <meta name="html-safe-nonce" content="5aa226b80a18dc40894e1d405e4eb31cfca7d616">
+
+  <meta http-equiv="x-pjax-version" content="f682644ce1bb9629b9d9d9bedf64801b">
+  
+
+      <link href="https://github.com/vt-middleware/cryptacular/commits/master.atom" rel="alternate" title="Recent Commits to cryptacular:master" type="application/atom+xml">
+
+  <meta name="description" content="cryptacular - The friendly complement to the BouncyCastle crypto API for Java.">
+  <meta name="go-import" content="github.com/vt-middleware/cryptacular git https://github.com/vt-middleware/cryptacular.git">
+
+  <meta content="6122907" name="octolytics-dimension-user_id" /><meta content="vt-middleware" name="octolytics-dimension-user_login" /><meta content="15714989" name="octolytics-dimension-repository_id" /><meta content="vt-middleware/cryptacular" name="octolytics-dimension-repository_nwo" /><meta content="true" name="octolytics-dimension-repository_public" /><meta content="false" name="octolytics-dimension-repository_is_fork" /><meta content="15714989" name="octolytics-dimension-repository_network_root_id" /><meta content="vt-middleware/cryptacular" name="octolytics-dimension-repository_network_root_nwo" /><meta content="false" name="octolytics-dimension-repository_explore_github_marketplace_ci_cta_shown" />
+
+
+    <link rel="canonical" href="https://github.com/vt-middleware/cryptacular/blob/master/NOTICE" data-pjax-transient>
+
+
+  <meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">
+
+  <meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">
+
+  <link rel="mask-icon" href="https://assets-cdn.github.com/pinned-octocat.svg" color="#000000">
+  <link rel="icon" type="image/x-icon" href="https://assets-cdn.github.com/favicon.ico">
+
+<meta name="theme-color" content="#1e2327">
+
+
+
+  </head>
+
+  <body class="logged-out env-production page-blob">
+    
+
+
+
+  <div class="position-relative js-header-wrapper ">
+    <a href="#start-of-content" tabindex="1" class="px-2 py-4 show-on-focus js-skip-to-content">Skip to content</a>
+    <div id="js-pjax-loader-bar" class="pjax-loader-bar"><div class="progress"></div></div>
+
+    
+    
+    
+
+
+
+          <header class="site-header js-details-container Details" role="banner">
+  <div class="site-nav-container">
+    <a class="header-logo-invertocat" href="https://github.com/" aria-label="Homepage" data-ga-click="(Logged out) Header, go to homepage, icon:logo-wordmark">
+      <svg aria-hidden="true" class="octicon octicon-mark-github" height="32" version="1.1" viewBox="0 0 16 16" width="32"><path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg>
+    </a>
+
+    <button class="btn-link float-right site-header-toggle js-details-target" type="button" aria-label="Toggle navigation" aria-expanded="false">
+      <svg aria-hidden="true" class="octicon octicon-three-bars" height="24" version="1.1" viewBox="0 0 12 16" width="18"><path fill-rule="evenodd" d="M11.41 9H.59C0 9 0 8.59 0 8c0-.59 0-1 .59-1H11.4c.59 0 .59.41.59 1 0 .59 0 1-.59 1h.01zm0-4H.59C0 5 0 4.59 0 4c0-.59 0-1 .59-1H11.4c.59 0 .59.41.59 1 0 .59 0 1-.59 1h.01zM.59 11H11.4c.59 0 .59.41.59 1 0 .59 0 1-.59 1H.59C0 13 0 12.59 0 12c0-.59 0-1 .59-1z"/></svg>
+    </button>
+
+    <div class="site-header-menu">
+      <nav class="site-header-nav">
+        <a href="/features" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:features" data-selected-links="/features /features/code-review /features/project-management /features/integrations /features">
+          Features
+</a>        <a href="/business" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:business" data-selected-links="/business /business/security /business/customers /business">
+          Business
+</a>        <a href="/explore" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:explore" data-selected-links="/explore /trending /trending/developers /stars /integrations /integrations/feature/code /integrations/feature/collaborate /integrations/feature/ship showcases showcases_search showcases_landing /explore">
+          Explore
+</a>            <a href="/marketplace" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:marketplace" data-selected-links=" /marketplace">
+              Marketplace
+</a>        <a href="/pricing" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:pricing" data-selected-links="/pricing /pricing/developer /pricing/team /pricing/business-hosted /pricing/business-enterprise /pricing">
+          Pricing
+</a>      </nav>
+
+      <div class="site-header-actions">
+          <div class="header-search scoped-search site-scoped-search js-site-search" role="search">
+  <!-- '"` --><!-- </textarea></xmp> --></option></form><form accept-charset="UTF-8" action="/vt-middleware/cryptacular/search" class="js-site-search-form" data-scoped-search-url="/vt-middleware/cryptacular/search" data-unscoped-search-url="/search" method="get"><div style="margin:0;padding:0;display:inline"><input name="utf8" type="hidden" value="&#x2713;" /></div>
+    <label class="form-control header-search-wrapper js-chromeless-input-container">
+        <a href="/vt-middleware/cryptacular/blob/master/NOTICE" class="header-search-scope no-underline">This repository</a>
+      <input type="text"
+        class="form-control header-search-input js-site-search-focus js-site-search-field is-clearable"
+        data-hotkey="s"
+        name="q"
+        value=""
+        placeholder="Search"
+        aria-label="Search this repository"
+        data-unscoped-placeholder="Search GitHub"
+        data-scoped-placeholder="Search"
+        autocapitalize="off">
+        <input type="hidden" class="js-site-search-type-field" name="type" >
+    </label>
+</form></div>
+
+
+          <a class="text-bold site-header-link" href="/login?return_to=%2Fvt-middleware%2Fcryptacular%2Fblob%2Fmaster%2FNOTICE" data-ga-click="(Logged out) Header, clicked Sign in, text:sign-in">Sign in</a>
+            <span class="text-gray">or</span>
+            <a class="text-bold site-header-link" href="/join?source=header-repo" data-ga-click="(Logged out) Header, clicked Sign up, text:sign-up">Sign up</a>
+      </div>
+    </div>
+  </div>
+</header>
+
+
+  </div>
+
+  <div id="start-of-content" class="show-on-focus"></div>
+
+    <div id="js-flash-container">
+</div>
+
+
+
+  <div role="main">
+        <div itemscope itemtype="http://schema.org/SoftwareSourceCode">
+    <div id="js-repo-pjax-container" data-pjax-container>
+      
+
+
+
+  
+
+
+    <div class="pagehead repohead instapaper_ignore readability-menu experiment-repo-nav">
+      <div class="container repohead-details-container">
+
+        <ul class="pagehead-actions">
+  <li>
+      <a href="/login?return_to=%2Fvt-middleware%2Fcryptacular"
+    class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+    aria-label="You must be signed in to watch a repository" rel="nofollow">
+    <svg aria-hidden="true" class="octicon octicon-eye" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M8.06 2C3 2 0 8 0 8s3 6 8.06 6C13 14 16 8 16 8s-3-6-7.94-6zM8 12c-2.2 0-4-1.78-4-4 0-2.2 1.8-4 4-4 2.22 0 4 1.8 4 4 0 2.22-1.78 4-4 4zm2-4c0 1.11-.89 2-2 2-1.11 0-2-.89-2-2 0-1.11.89-2 2-2 1.11 0 2 .89 2 2z"/></svg>
+    Watch
+  </a>
+  <a class="social-count" href="/vt-middleware/cryptacular/watchers"
+     aria-label="9 users are watching this repository">
+    9
+  </a>
+
+  </li>
+
+  <li>
+      <a href="/login?return_to=%2Fvt-middleware%2Fcryptacular"
+    class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+    aria-label="You must be signed in to star a repository" rel="nofollow">
+    <svg aria-hidden="true" class="octicon octicon-star" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M14 6l-4.9-.64L7 1 4.9 5.36 0 6l3.6 3.26L2.67 14 7 11.67 11.33 14l-.93-4.74z"/></svg>
+    Star
+  </a>
+
+    <a class="social-count js-social-count" href="/vt-middleware/cryptacular/stargazers"
+      aria-label="15 users starred this repository">
+      15
+    </a>
+
+  </li>
+
+  <li>
+      <a href="/login?return_to=%2Fvt-middleware%2Fcryptacular"
+        class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+        aria-label="You must be signed in to fork a repository" rel="nofollow">
+        <svg aria-hidden="true" class="octicon octicon-repo-forked" height="16" version="1.1" viewBox="0 0 10 16" width="10"><path fill-rule="evenodd" d="M8 1a1.993 1.993 0 0 0-1 3.72V6L5 8 3 6V4.72A1.993 1.993 0 0 0 2 1a1.993 1.993 0 0 0-1 3.72V6.5l3 3v1.78A1.993 1.993 0 0 0 5 15a1.993 1.993 0 0 0 1-3.72V9.5l3-3V4.72A1.993 1.993 0 0 0 8 1zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zm3 10c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zm3-10c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"/></svg>
+        Fork
+      </a>
+
+    <a href="/vt-middleware/cryptacular/network" class="social-count"
+       aria-label="1 user forked this repository">
+      1
+    </a>
+  </li>
+</ul>
+
+        <h1 class="public ">
+  <svg aria-hidden="true" class="octicon octicon-repo" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M4 9H3V8h1v1zm0-3H3v1h1V6zm0-2H3v1h1V4zm0-2H3v1h1V2zm8-1v12c0 .55-.45 1-1 1H6v2l-1.5-1.5L3 16v-2H1c-.55 0-1-.45-1-1V1c0-.55.45-1 1-1h10c.55 0 1 .45 1 1zm-1 10H1v2h2v-1h3v1h5v-2zm0-10H2v9h9V1z"/></svg>
+  <span class="author" itemprop="author"><a href="/vt-middleware" class="url fn" rel="author">vt-middleware</a></span><!--
+--><span class="path-divider">/</span><!--
+--><strong itemprop="name"><a href="/vt-middleware/cryptacular" data-pjax="#js-repo-pjax-container">cryptacular</a></strong>
+
+</h1>
+
+      </div>
+      <div class="container">
+        
+<nav class="reponav js-repo-nav js-sidenav-container-pjax"
+     itemscope
+     itemtype="http://schema.org/BreadcrumbList"
+     role="navigation"
+     data-pjax="#js-repo-pjax-container">
+
+  <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+    <a href="/vt-middleware/cryptacular" class="js-selected-navigation-item selected reponav-item" data-hotkey="g c" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches /vt-middleware/cryptacular" itemprop="url">
+      <svg aria-hidden="true" class="octicon octicon-code" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M9.5 3L8 4.5 11.5 8 8 11.5 9.5 13 14 8 9.5 3zm-5 0L0 8l4.5 5L6 11.5 2.5 8 6 4.5 4.5 3z"/></svg>
+      <span itemprop="name">Code</span>
+      <meta itemprop="position" content="1">
+</a>  </span>
+
+    <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+      <a href="/vt-middleware/cryptacular/issues" class="js-selected-navigation-item reponav-item" data-hotkey="g i" data-selected-links="repo_issues repo_labels repo_milestones /vt-middleware/cryptacular/issues" itemprop="url">
+        <svg aria-hidden="true" class="octicon octicon-issue-opened" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"/></svg>
+        <span itemprop="name">Issues</span>
+        <span class="Counter">4</span>
+        <meta itemprop="position" content="2">
+</a>    </span>
+
+  <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+    <a href="/vt-middleware/cryptacular/pulls" class="js-selected-navigation-item reponav-item" data-hotkey="g p" data-selected-links="repo_pulls /vt-middleware/cryptacular/pulls" itemprop="url">
+      <svg aria-hidden="true" class="octicon octicon-git-pull-request" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M11 11.28V5c-.03-.78-.34-1.47-.94-2.06C9.46 2.35 8.78 2.03 8 2H7V0L4 3l3 3V4h1c.27.02.48.11.69.31.21.2.3.42.31.69v6.28A1.993 1.993 0 0 0 10 15a1.993 1.993 0 0 0 1-3.72zm-1 2.92c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zM4 3c0-1.11-.89-2-2-2a1.993 1.993 0 0 0-1 3.72v6.56A1.993 1.993 0 0 0 2 15a1.993 1.993 0 0 0 1-3.72V4.72c.59-.34 1-.98 1-1.72zm-.8 10c0 .66-.55 1.2-1.2 1.2-.65 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"/></svg>
+      <span itemprop="name">Pull requests</span>
+      <span class="Counter">1</span>
+      <meta itemprop="position" content="3">
+</a>  </span>
+
+    <a href="/vt-middleware/cryptacular/projects" class="js-selected-navigation-item reponav-item" data-selected-links="repo_projects new_repo_project repo_project /vt-middleware/cryptacular/projects">
+      <svg aria-hidden="true" class="octicon octicon-project" height="16" version="1.1" viewBox="0 0 15 16" width="15"><path fill-rule="evenodd" d="M10 12h3V2h-3v10zm-4-2h3V2H6v8zm-4 4h3V2H2v12zm-1 1h13V1H1v14zM14 0H1a1 1 0 0 0-1 1v14a1 1 0 0 0 1 1h13a1 1 0 0 0 1-1V1a1 1 0 0 0-1-1z"/></svg>
+      Projects
+      <span class="Counter" >0</span>
+</a>
+
+
+    <div class="reponav-dropdown js-menu-container">
+      <button type="button" class="btn-link reponav-item reponav-dropdown js-menu-target " data-no-toggle aria-expanded="false" aria-haspopup="true">
+        Insights
+        <svg aria-hidden="true" class="octicon octicon-triangle-down v-align-middle text-gray" height="11" version="1.1" viewBox="0 0 12 16" width="8"><path fill-rule="evenodd" d="M0 5l6 6 6-6z"/></svg>
+      </button>
+      <div class="dropdown-menu-content js-menu-content">
+        <div class="dropdown-menu dropdown-menu-sw">
+          <a class="dropdown-item" href="/vt-middleware/cryptacular/pulse" data-skip-pjax>
+            <svg aria-hidden="true" class="octicon octicon-pulse" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M11.5 8L8.8 5.4 6.6 8.5 5.5 1.6 2.38 8H0v2h3.6l.9-1.8.9 5.4L9 8.5l1.6 1.5H14V8z"/></svg>
+            Pulse
+          </a>
+          <a class="dropdown-item" href="/vt-middleware/cryptacular/graphs" data-skip-pjax>
+            <svg aria-hidden="true" class="octicon octicon-graph" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M16 14v1H0V0h1v14h15zM5 13H3V8h2v5zm4 0H7V3h2v10zm4 0h-2V6h2v7z"/></svg>
+            Graphs
+          </a>
+        </div>
+      </div>
+    </div>
+</nav>
+
+      </div>
+    </div>
+
+<div class="container new-discussion-timeline experiment-repo-nav">
+  <div class="repository-content">
+
+    
+  <a href="/vt-middleware/cryptacular/blob/c26911e3cd28497ce9daa3ce682e09cb2d1d8688/NOTICE" class="d-none js-permalink-shortcut" data-hotkey="y">Permalink</a>
+
+  <!-- blob contrib key: blob_contributors:v21:f5054b9e46039e0ad937c69e6151b7d4 -->
+
+  <div class="file-navigation js-zeroclipboard-container">
+    
+<div class="select-menu branch-select-menu js-menu-container js-select-menu float-left">
+  <button class=" btn btn-sm select-menu-button js-menu-target css-truncate" data-hotkey="w"
+    
+    type="button" aria-label="Switch branches or tags" aria-expanded="false" aria-haspopup="true">
+      <i>Branch:</i>
+      <span class="js-select-button css-truncate-target">master</span>
+  </button>
+
+  <div class="select-menu-modal-holder js-menu-content js-navigation-container" data-pjax>
+
+    <div class="select-menu-modal">
+      <div class="select-menu-header">
+        <svg aria-label="Close" class="octicon octicon-x js-menu-close" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M7.48 8l3.75 3.75-1.48 1.48L6 9.48l-3.75 3.75-1.48-1.48L4.52 8 .77 4.25l1.48-1.48L6 6.52l3.75-3.75 1.48 1.48z"/></svg>
+        <span class="select-menu-title">Switch branches/tags</span>
+      </div>
+
+      <div class="select-menu-filters">
+        <div class="select-menu-text-filter">
+          <input type="text" aria-label="Filter branches/tags" id="context-commitish-filter-field" class="form-control js-filterable-field js-navigation-enable" placeholder="Filter branches/tags">
+        </div>
+        <div class="select-menu-tabs">
+          <ul>
+            <li class="select-menu-tab">
+              <a href="#" data-tab-filter="branches" data-filter-placeholder="Filter branches/tags" class="js-select-menu-tab" role="tab">Branches</a>
+            </li>
+            <li class="select-menu-tab">
+              <a href="#" data-tab-filter="tags" data-filter-placeholder="Find a tag…" class="js-select-menu-tab" role="tab">Tags</a>
+            </li>
+          </ul>
+        </div>
+      </div>
+
+      <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="branches" role="menu">
+
+        <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring">
+
+
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/vt-middleware/cryptacular/blob/ISSUE-31+32/NOTICE"
+               data-name="ISSUE-31+32"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                ISSUE-31+32
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/vt-middleware/cryptacular/blob/gh-pages/NOTICE"
+               data-name="gh-pages"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                gh-pages
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open selected"
+               href="/vt-middleware/cryptacular/blob/master/NOTICE"
+               data-name="master"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                master
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/vt-middleware/cryptacular/blob/v1.1/NOTICE"
+               data-name="v1.1"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                v1.1
+              </span>
+            </a>
+        </div>
+
+          <div class="select-menu-no-results">Nothing to show</div>
+      </div>
+
+      <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="tags">
+        <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring">
+
+
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.2.1/NOTICE"
+              data-name="v1.2.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.2.1">
+                v1.2.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.2.0/NOTICE"
+              data-name="v1.2.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.2.0">
+                v1.2.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.1.2/NOTICE"
+              data-name="v1.1.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.1.2">
+                v1.1.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.1.1/NOTICE"
+              data-name="v1.1.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.1.1">
+                v1.1.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.1.0/NOTICE"
+              data-name="v1.1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.1.0">
+                v1.1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0/NOTICE"
+              data-name="v1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0">
+                v1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC6/NOTICE"
+              data-name="v1.0-RC6"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC6">
+                v1.0-RC6
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC4/NOTICE"
+              data-name="v1.0-RC4"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC4">
+                v1.0-RC4
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC3/NOTICE"
+              data-name="v1.0-RC3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC3">
+                v1.0-RC3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC2/NOTICE"
+              data-name="v1.0-RC2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC2">
+                v1.0-RC2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC1/NOTICE"
+              data-name="v1.0-RC1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC1">
+                v1.0-RC1
+              </span>
+            </a>
+        </div>
+
+        <div class="select-menu-no-results">Nothing to show</div>
+      </div>
+
+    </div>
+  </div>
+</div>
+
+    <div class="BtnGroup float-right">
+      <a href="/vt-middleware/cryptacular/find/master"
+            class="js-pjax-capture-input btn btn-sm BtnGroup-item"
+            data-pjax
+            data-hotkey="t">
+        Find file
+      </a>
+      <button aria-label="Copy file path to clipboard" class="js-zeroclipboard btn btn-sm BtnGroup-item tooltipped tooltipped-s" data-copied-hint="Copied!" type="button">Copy path</button>
+    </div>
+    <div class="breadcrumb js-zeroclipboard-target">
+      <span class="repo-root js-repo-root"><span class="js-path-segment"><a href="/vt-middleware/cryptacular"><span>cryptacular</span></a></span></span><span class="separator">/</span><strong class="final-path">NOTICE</strong>
+    </div>
+  </div>
+
+
+  
+  <div class="commit-tease">
+      <span class="float-right">
+        <a class="commit-tease-sha" href="/vt-middleware/cryptacular/commit/6dd6f199ac3ecc3b4c5aef9e04be3bbe265a30a1" data-pjax>
+          6dd6f19
+        </a>
+        <relative-time datetime="2017-07-06T22:28:36Z">Jul 7, 2017</relative-time>
+      </span>
+      <div>
+        <img alt="@dfish3r" class="avatar" height="20" src="https://avatars6.githubusercontent.com/u/1051499?v=4&amp;s=40" width="20" />
+        <a href="/dfish3r" class="user-mention" rel="contributor">dfish3r</a>
+          <a href="/vt-middleware/cryptacular/commit/6dd6f199ac3ecc3b4c5aef9e04be3bbe265a30a1" class="message" data-pjax="true" title="Update year in notice.">Update year in notice.</a>
+      </div>
+
+    <div class="commit-tease-contributors">
+      <button type="button" class="btn-link muted-link contributors-toggle" data-facebox="#blob_contributors_box">
+        <strong>1</strong>
+         contributor
+      </button>
+      
+    </div>
+
+    <div id="blob_contributors_box" style="display:none">
+      <h2 class="facebox-header" data-facebox-id="facebox-header">Users who have contributed to this file</h2>
+      <ul class="facebox-user-list" data-facebox-id="facebox-description">
+          <li class="facebox-user-list-item">
+            <img alt="@dfish3r" height="24" src="https://avatars4.githubusercontent.com/u/1051499?v=4&amp;s=48" width="24" />
+            <a href="/dfish3r">dfish3r</a>
+          </li>
+      </ul>
+    </div>
+  </div>
+
+  <div class="file">
+    <div class="file-header">
+  <div class="file-actions">
+
+    <div class="BtnGroup">
+      <a href="/vt-middleware/cryptacular/raw/master/NOTICE" class="btn btn-sm BtnGroup-item" id="raw-url">Raw</a>
+        <a href="/vt-middleware/cryptacular/blame/master/NOTICE" class="btn btn-sm js-update-url-with-hash BtnGroup-item" data-hotkey="b">Blame</a>
+      <a href="/vt-middleware/cryptacular/commits/master/NOTICE" class="btn btn-sm BtnGroup-item" rel="nofollow">History</a>
+    </div>
+
+
+        <button type="button" class="btn-octicon disabled tooltipped tooltipped-nw"
+          aria-label="You must be signed in to make or propose changes">
+          <svg aria-hidden="true" class="octicon octicon-pencil" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M0 12v3h3l8-8-3-3-8 8zm3 2H1v-2h1v1h1v1zm10.3-9.3L12 6 9 3l1.3-1.3a.996.996 0 0 1 1.41 0l1.59 1.59c.39.39.39 1.02 0 1.41z"/></svg>
+        </button>
+        <button type="button" class="btn-octicon btn-octicon-danger disabled tooltipped tooltipped-nw"
+          aria-label="You must be signed in to make or propose changes">
+          <svg aria-hidden="true" class="octicon octicon-trashcan" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M11 2H9c0-.55-.45-1-1-1H5c-.55 0-1 .45-1 1H2c-.55 0-1 .45-1 1v1c0 .55.45 1 1 1v9c0 .55.45 1 1 1h7c.55 0 1-.45 1-1V5c.55 0 1-.45 1-1V3c0-.55-.45-1-1-1zm-1 12H3V5h1v8h1V5h1v8h1V5h1v8h1V5h1v9zm1-10H2V3h9v1z"/></svg>
+        </button>
+  </div>
+
+  <div class="file-info">
+      7 lines (5 sloc)
+      <span class="file-info-divider"></span>
+    165 Bytes
+  </div>
+</div>
+
+    
+
+  <div itemprop="text" class="blob-wrapper data type-text">
+      <table class="highlight tab-size js-file-line-container" data-tab-size="8">
+      <tr>
+        <td id="L1" class="blob-num js-line-number" data-line-number="1"></td>
+        <td id="LC1" class="blob-code blob-code-inner js-file-line">Cryptacular Java Library</td>
+      </tr>
+      <tr>
+        <td id="L2" class="blob-num js-line-number" data-line-number="2"></td>
+        <td id="LC2" class="blob-code blob-code-inner js-file-line">Copyright (C) 2003-2017 Virginia Tech.</td>
+      </tr>
+      <tr>
+        <td id="L3" class="blob-num js-line-number" data-line-number="3"></td>
+        <td id="LC3" class="blob-code blob-code-inner js-file-line">All rights reserved.</td>
+      </tr>
+      <tr>
+        <td id="L4" class="blob-num js-line-number" data-line-number="4"></td>
+        <td id="LC4" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L5" class="blob-num js-line-number" data-line-number="5"></td>
+        <td id="LC5" class="blob-code blob-code-inner js-file-line">This product includes software developed at</td>
+      </tr>
+      <tr>
+        <td id="L6" class="blob-num js-line-number" data-line-number="6"></td>
+        <td id="LC6" class="blob-code blob-code-inner js-file-line">Virginia Tech (http://www.vt.edu).</td>
+      </tr>
+</table>
+
+
+  </div>
+
+  </div>
+
+  <button type="button" data-facebox="#jump-to-line" data-facebox-class="linejump" data-hotkey="l" class="d-none">Jump to Line</button>
+  <div id="jump-to-line" style="display:none">
+    <!-- '"` --><!-- </textarea></xmp> --></option></form><form accept-charset="UTF-8" action="" class="js-jump-to-line-form" method="get"><div style="margin:0;padding:0;display:inline"><input name="utf8" type="hidden" value="&#x2713;" /></div>
+      <input class="form-control linejump-input js-jump-to-line-field" type="text" placeholder="Jump to line&hellip;" aria-label="Jump to line" autofocus>
+      <button type="submit" class="btn">Go</button>
+</form>  </div>
+
+  </div>
+  <div class="modal-backdrop js-touch-events"></div>
+</div>
+
+    </div>
+  </div>
+
+  </div>
+
+      
+<div class="container site-footer-container">
+  <div class="site-footer " role="contentinfo">
+    <ul class="site-footer-links float-right">
+        <li><a href="https://github.com/contact" data-ga-click="Footer, go to contact, text:contact">Contact GitHub</a></li>
+      <li><a href="https://developer.github.com" data-ga-click="Footer, go to api, text:api">API</a></li>
+      <li><a href="https://training.github.com" data-ga-click="Footer, go to training, text:training">Training</a></li>
+      <li><a href="https://shop.github.com" data-ga-click="Footer, go to shop, text:shop">Shop</a></li>
+        <li><a href="https://github.com/blog" data-ga-click="Footer, go to blog, text:blog">Blog</a></li>
+        <li><a href="https://github.com/about" data-ga-click="Footer, go to about, text:about">About</a></li>
+
+    </ul>
+
+    <a href="https://github.com" aria-label="Homepage" class="site-footer-mark" title="GitHub">
+      <svg aria-hidden="true" class="octicon octicon-mark-github" height="24" version="1.1" viewBox="0 0 16 16" width="24"><path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg>
+</a>
+    <ul class="site-footer-links">
+      <li>&copy; 2017 <span title="0.10938s from unicorn-2925809464-f2ltq">GitHub</span>, Inc.</li>
+        <li><a href="https://github.com/site/terms" data-ga-click="Footer, go to terms, text:terms">Terms</a></li>
+        <li><a href="https://github.com/site/privacy" data-ga-click="Footer, go to privacy, text:privacy">Privacy</a></li>
+        <li><a href="https://github.com/security" data-ga-click="Footer, go to security, text:security">Security</a></li>
+        <li><a href="https://status.github.com/" data-ga-click="Footer, go to status, text:status">Status</a></li>
+        <li><a href="https://help.github.com" data-ga-click="Footer, go to help, text:help">Help</a></li>
+    </ul>
+  </div>
+</div>
+
+
+
+  <div id="ajax-error-message" class="ajax-error-message flash flash-error">
+    <svg aria-hidden="true" class="octicon octicon-alert" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M8.865 1.52c-.18-.31-.51-.5-.87-.5s-.69.19-.87.5L.275 13.5c-.18.31-.18.69 0 1 .19.31.52.5.87.5h13.7c.36 0 .69-.19.86-.5.17-.31.18-.69.01-1L8.865 1.52zM8.995 13h-2v-2h2v2zm0-3h-2V6h2v4z"/></svg>
+    <button type="button" class="flash-close js-flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
+      <svg aria-hidden="true" class="octicon octicon-x" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M7.48 8l3.75 3.75-1.48 1.48L6 9.48l-3.75 3.75-1.48-1.48L4.52 8 .77 4.25l1.48-1.48L6 6.52l3.75-3.75 1.48 1.48z"/></svg>
+    </button>
+    You can't perform that action at this time.
+  </div>
+
+
+    <script crossorigin="anonymous" src="https://assets-cdn.github.com/assets/compat-91f98c37fc84eac24836eec2567e9912742094369a04c4eba6e3cd1fa18902d9.js"></script>
+    <script crossorigin="anonymous" src="https://assets-cdn.github.com/assets/frameworks-f84bb87b149685d1e6c6f057ee324f2cd496e677f5a359a8b5db853313bb83e6.js"></script>
+    
+    <script async="async" crossorigin="anonymous" src="https://assets-cdn.github.com/assets/github-13fa3aa50ac8f9fa9a7d198f0cd13b0905775d39446ad076d17d8f74a998438a.js"></script>
+    
+    
+    
+    
+  <div class="js-stale-session-flash stale-session-flash flash flash-warn flash-banner d-none">
+    <svg aria-hidden="true" class="octicon octicon-alert" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M8.865 1.52c-.18-.31-.51-.5-.87-.5s-.69.19-.87.5L.275 13.5c-.18.31-.18.69 0 1 .19.31.52.5.87.5h13.7c.36 0 .69-.19.86-.5.17-.31.18-.69.01-1L8.865 1.52zM8.995 13h-2v-2h2v2zm0-3h-2V6h2v4z"/></svg>
+    <span class="signed-in-tab-flash">You signed in with another tab or window. <a href="">Reload</a> to refresh your session.</span>
+    <span class="signed-out-tab-flash">You signed out in another tab or window. <a href="">Reload</a> to refresh your session.</span>
+  </div>
+  <div class="facebox" id="facebox" style="display:none;">
+  <div class="facebox-popup">
+    <div class="facebox-content" role="dialog" aria-labelledby="facebox-header" aria-describedby="facebox-description">
+    </div>
+    <button type="button" class="facebox-close js-facebox-close" aria-label="Close modal">
+      <svg aria-hidden="true" class="octicon octicon-x" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M7.48 8l3.75 3.75-1.48 1.48L6 9.48l-3.75 3.75-1.48-1.48L4.52 8 .77 4.25l1.48-1.48L6 6.52l3.75-3.75 1.48 1.48z"/></svg>
+    </button>
+  </div>
+</div>
+
+
+  </body>
+</html>
+
diff --git a/x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1 b/x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1
new file mode 100644
index 0000000000000..02f937f821f45
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1
@@ -0,0 +1 @@
+6ce200f6b23222af3d8abb6b6459e6c44f4bb0e9
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/guava-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/guava-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt
new file mode 100644
index 0000000000000..32f01eda18fe9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt
new file mode 100644
index 0000000000000..91e5c40c4c6d3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache HttpComponents Client
+Copyright 1999-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1 b/x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1
new file mode 100644
index 0000000000000..3b1d8db643114
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1
@@ -0,0 +1 @@
+b195778247a21e980cb9f80c41364dc0c38feaef
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1 b/x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1
new file mode 100644
index 0000000000000..2d138de40005a
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1
@@ -0,0 +1 @@
+c3fecaa141e8f0fff8a14e6800aefa8155c9b3e8
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1 b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..6178556b31848
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
@@ -0,0 +1 @@
+4b41b53a3a2d299ce381a69d165381ca19f62912
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt
new file mode 100644
index 0000000000000..ea99ef1d4726b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt
@@ -0,0 +1,8 @@
+
+Apache Log4j SLF4J Binding
+Copyright 1999-2017 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
diff --git a/x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1 b/x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1
new file mode 100644
index 0000000000000..dedc1dfa8d0e4
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1
@@ -0,0 +1 @@
+cd9886f498ee2ab2d994f0c779e5553b2c450416
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt
new file mode 100644
index 0000000000000..255f6a193ad0f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt
@@ -0,0 +1,4 @@
+Dropwizard
+Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2016 Dropwizard Team
+
+This product includes software developed by Coda Hale and Yammer, Inc.
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..be9ed5be9296b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1
@@ -0,0 +1 @@
+0958fae127de9e8b0296e6f089c7451b6d5f0846
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..c370c3bd6369c
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+e3ec93dfbf90c451e9f7fb34a3e33a6ac60edd31
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..9131bc2ac23c1
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+beaca9bd69ad861dbb55f1694853a02cb6988ae7
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..409c3b9b6db30
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+bb0a1f97d38342a5715bad628ee24000b08e821e
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..38ce75e4c2da3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+6cb4595c7a988d964f6a2d55dcac754b0c68904e
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..d575629f16277
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+bef43d21b2d878baceae291af4a0ad3449c7d7ec
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..f52b95e499521
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+ecf4a9552575d38cffd4dc56d95e7564b7dccfc1
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..7db3d5ea3558e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+15cbb232ae6665edc5df5f260e551e69fdb362e5
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..e5bbed6dae0ba
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+b2bc1aa5b0f400aa50499f3783b10e9f7c216a47
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..840170214e9f7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+c497df002980c6e482ce7b828924bb24f60f99f7
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..219b3060af86c
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+30ed8d37259e840df5b3fd8daf7b654129a9190c
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..2290462e05c65
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+a984671fd04e50da03f68003d2b062578e63ec86
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..c511027f9aff4
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+a4b828fe1a9d64953ecdd8a9e00ff31b63ad6ef0
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..44dea7d989bef
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+a1b10f97deca1e3405f95db5b39697c0d46f5e0d
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..dcdc61dc2e247
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+d46cb9854a1ff85bea34ece7077bc32dbc2f10da
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/shibboleth-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/shibboleth-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1 b/x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1
new file mode 100644
index 0000000000000..a2f93ea55802b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1
@@ -0,0 +1 @@
+8619e95939167fb37245b5670135e4feb0ec7d50
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt
new file mode 100644
index 0000000000000..c8e72c2cd4738
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt
@@ -0,0 +1,21 @@
+ Copyright (c) 2004-2017 QOS.ch
+ All rights reserved.
+
+ Permission is hereby granted, free  of charge, to any person obtaining
+ a  copy  of this  software  and  associated  documentation files  (the
+ "Software"), to  deal in  the Software without  restriction, including
+ without limitation  the rights to  use, copy, modify,  merge, publish,
+ distribute,  sublicense, and/or sell  copies of  the Software,  and to
+ permit persons to whom the Software  is furnished to do so, subject to
+ the following conditions:
+ 
+ The  above  copyright  notice  and  this permission  notice  shall  be
+ included in all copies or substantial portions of the Software.
+ 
+ THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
+ EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
+ MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/x-pack/plugin/identity-provider/licenses/slf4j-api-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/slf4j-api-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1 b/x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1
new file mode 100644
index 0000000000000..d85a4194f6a59
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1
@@ -0,0 +1 @@
+cb43326f02e3e77526c24269c8b5d3cc3f7f6653
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt
new file mode 100644
index 0000000000000..ab7c6276cd836
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt
@@ -0,0 +1,9 @@
+
+Apache XML Security for Java
+Copyright 2000-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+
diff --git a/x-pack/plugin/identity-provider/qa/build.gradle b/x-pack/plugin/identity-provider/qa/build.gradle
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
new file mode 100644
index 0000000000000..85884ef2d74bd
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
@@ -0,0 +1,33 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('identity-provider'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  setting 'xpack.idp.enabled', 'true'
+  setting 'xpack.idp.entity_id', 'https://idp.test.es.elasticsearch.org/'
+  setting 'xpack.idp.sso_endpoint.redirect', 'http://idp.test.es.elasticsearch.org/test/saml/redirect'
+  setting 'xpack.idp.signing.certificate', 'idp-sign.crt'
+  setting 'xpack.idp.signing.key', 'idp-sign.key'
+  setting 'xpack.idp.privileges.application', 'elastic-cloud'
+
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+
+  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+  extraConfigFile 'idp-sign.crt', file('src/test/resources/idp-sign.crt')
+  extraConfigFile 'idp-sign.key', file('src/test/resources/idp-sign.key')
+
+  user username: "admin_user", password: "admin-password"
+  user username: "idp_user", password: "idp-password", role: "idp_role"
+}
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
new file mode 100644
index 0000000000000..e833d644bf3a5
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public abstract class IdpRestTestCase extends ESRestTestCase {
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("idp_user", new SecureString("idp-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java
new file mode 100644
index 0000000000000..4cae5a29bf5d0
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp;
+
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class ManageServiceProviderRestIT extends IdpRestTestCase {
+
+    // From build.gradle
+    private final String IDP_ENTITY_ID = "https://idp.test.es.elasticsearch.org/";
+    // From SAMLConstants
+    private final String REDIRECT_BINDING = "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect";
+
+    public void testCreateAndDeleteServiceProvider() throws Exception {
+        final String entityId = "ec:" + randomAlphaOfLength(8) + ":" + randomAlphaOfLength(12);
+        final Map<String, Object> request = Map.ofEntries(
+            Map.entry("name", "Test SP"),
+            Map.entry("acs", "https://sp1.test.es.elasticsearch.org/saml/acs"),
+            Map.entry("privileges", Map.ofEntries(
+                Map.entry("resource", entityId),
+                Map.entry("roles", Map.of("superuser", "role:superuser", "viewer", "role:viewer"))
+            )),
+            Map.entry("attributes", Map.ofEntries(
+                Map.entry("principal", "https://idp.test.es.elasticsearch.org/attribute/principal"),
+                Map.entry("name", "https://idp.test.es.elasticsearch.org/attribute/name"),
+                Map.entry("email", "https://idp.test.es.elasticsearch.org/attribute/email"),
+                Map.entry("roles", "https://idp.test.es.elasticsearch.org/attribute/roles")
+            )));
+        final DocumentVersion docVersion = createServiceProvider(entityId, request);
+        checkIndexDoc(docVersion);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        getMetaData(entityId);
+        deleteServiceProvider(entityId, docVersion);
+        expectThrows(ResponseException.class, () -> getMetaData(entityId));
+        expectThrows(ResponseException.class, () -> deleteServiceProvider(entityId, docVersion));
+    }
+
+    private DocumentVersion createServiceProvider(String entityId, Map<String, Object> body) throws IOException {
+        final Request request = new Request("PUT", "/_idp/saml/sp/" + encode(entityId) + "?refresh=" + RefreshPolicy.IMMEDIATE.getValue());
+        final String entity = Strings.toString(JsonXContent.contentBuilder().map(body));
+        request.setJsonEntity(entity);
+        final Response response = client().performRequest(request);
+        final Map<String, Object> map = entityAsMap(response);
+        assertThat(ObjectPath.eval("service_provider.entity_id", map), equalTo(entityId));
+        assertThat(ObjectPath.eval("service_provider.enabled", map), equalTo(true));
+
+        final Object docId = ObjectPath.eval("document._id", map);
+        final Object seqNo = ObjectPath.eval("document._seq_no", map);
+        final Object primaryTerm = ObjectPath.eval("document._primary_term", map);
+        assertThat(docId, instanceOf(String.class));
+        assertThat(seqNo, instanceOf(Number.class));
+        assertThat(primaryTerm, instanceOf(Number.class));
+        return new DocumentVersion((String) docId, asLong(primaryTerm), asLong(seqNo));
+    }
+
+    private void checkIndexDoc(DocumentVersion docVersion) throws IOException {
+        final Request request = new Request("GET", SamlServiceProviderIndex.INDEX_NAME + "/_doc/" + docVersion.id);
+        final Response response = adminClient().performRequest(request);
+        final Map<String, Object>
+            map = entityAsMap(response);
+        assertThat(map.get("_index"), equalTo(SamlServiceProviderIndex.INDEX_NAME));
+        assertThat(map.get("_id"), equalTo(docVersion.id));
+        assertThat(asLong(map.get("_seq_no")), equalTo(docVersion.seqNo));
+        assertThat(asLong(map.get("_primary_term")), equalTo(docVersion.primaryTerm));
+    }
+
+    private void deleteServiceProvider(String entityId, DocumentVersion version) throws IOException {
+        final Response response = client().performRequest(new Request("DELETE",
+            "/_idp/saml/sp/" + encode(entityId) + "?refresh=" + RefreshPolicy.IMMEDIATE.getValue()));
+        final Map<String, Object> map = entityAsMap(response);
+
+        assertThat(ObjectPath.eval("document._id", map), equalTo(version.id));
+
+        Long seqNo = asLong(ObjectPath.eval("document._seq_no", map));
+        Long primaryTerm = asLong(ObjectPath.eval("document._primary_term", map));
+        if (primaryTerm == version.primaryTerm) {
+            assertThat(seqNo, greaterThanOrEqualTo(version.seqNo));
+        } else {
+            assertThat(primaryTerm, greaterThanOrEqualTo(version.primaryTerm));
+        }
+
+        assertThat(ObjectPath.eval("service_provider.entity_id", map), equalTo(entityId));
+    }
+
+    private void getMetaData(String entityId) throws IOException {
+        final Map<String, Object> map = getAsMap("/_idp/saml/metadata/" + encode(entityId));
+        assertThat(map, notNullValue());
+        assertThat(map.keySet(), containsInAnyOrder("metadata"));
+        final Object metadata = map.get("metadata");
+        assertThat(metadata, notNullValue());
+        assertThat(metadata, instanceOf(String.class));
+        assertThat((String) metadata, containsString(IDP_ENTITY_ID));
+        assertThat((String) metadata, containsString(REDIRECT_BINDING));
+    }
+
+    private String encode(String param) {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8);
+    }
+
+    private Long asLong(Object val) {
+        if (val == null) {
+            return null;
+        }
+        if (val instanceof Long) {
+            return (Long) val;
+        }
+        if (val instanceof Number) {
+            return ((Number) val).longValue();
+        }
+        if (val instanceof String) {
+            return Long.parseLong((String) val);
+        }
+        throw new IllegalArgumentException("Value [" + val + "] of type [" + val.getClass() + "] is not a Long");
+    }
+}
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt
new file mode 100644
index 0000000000000..151aec5b336bb
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDoDCCAoigAwIBAgIVAPVpXjny5MCPPsCCGvInDfZQu+4JMA0GCSqGSIb3DQEB
+CwUAMF8xEzARBgoJkiaJk/IsZAEZFgNvcmcxHTAbBgoJkiaJk/IsZAEZFg1lbGFz
+dGljc2VhcmNoMQswCQYDVQQLEwJlczENMAsGA1UECxMEdGVzdDENMAsGA1UEAxME
+c2lnbjAeFw0yMDAzMTMwNDAzNDFaFw00NzA3MjkwNDAzNDFaMF8xEzARBgoJkiaJ
+k/IsZAEZFgNvcmcxHTAbBgoJkiaJk/IsZAEZFg1lbGFzdGljc2VhcmNoMQswCQYD
+VQQLEwJlczENMAsGA1UECxMEdGVzdDENMAsGA1UEAxMEc2lnbjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAI/JK5IlwBixo2mqX8wDnH2F+tXDJKU8uNez
+oulYFVMp17iK41sOIwCInd6DGDJPkgIIkobnmp3gDgnLckzApC8Ck9xmnjmEQdg+
+gZO1gtgBVU6Lj8haCzPUMVszGodlraeaRMTIBrNG6GPMo/PN5hj2vyC383jiDlE9
+K3I+Y0rxhnPj6Ic3aTO/bXiMG4FiIvag/ViITnvkL1mREwF9JER40dgQoaJJJFTf
+jcWYnd1q9FDlIhylj2KJwXhOkSZ5znGM93b3GZ5VZUm8zzMlq/RzyywDet2jyOSr
+Bn9X3lrp0fPjfhTW+O/+2G/k6bVP29BYJ5Im+JVjjaYbYFI5pQ0CAwEAAaNTMFEw
+HQYDVR0OBBYEFEghxQTreebAJEak+zywt5B/LpaLMB8GA1UdIwQYMBaAFEghxQTr
+eebAJEak+zywt5B/LpaLMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQAD
+ggEBACtGNnaUK4UtKPOSg7fOW3/tIJoPhriuSlAdqQxNRYjkdmPJsY7Gra/kikCm
+0dRUbttMTfOqwZwu5KdKJUo3duXKpRVK3emFDixM8FxFbhtA19+Newzcp6mS0K0T
+4wqAI5jH/a+eHJb6hDWuCrm9RICxXwqpUdn3HfJrYQzsOCuecTAy60jfxjjZeZxA
+MRIARkhaHQeOdX3rg5v9kWzFXRD0Y7nlqqqCa0el2gs9yZqjDYRMFzz8z4qN9q2N
+zjohhxgIwuWK+0R6qOIg3XK53kdh6YjgWqlGn1aX9z7ztqlDmiD7LIsPgWL2BARh
+WOexDVOg4wTpACOt4c84VVAVt98=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key
new file mode 100644
index 0000000000000..792de4f855b6f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAj8krkiXAGLGjaapfzAOcfYX61cMkpTy417Oi6VgVUynXuIrj
+Ww4jAIid3oMYMk+SAgiShueaneAOCctyTMCkLwKT3GaeOYRB2D6Bk7WC2AFVTouP
+yFoLM9QxWzMah2Wtp5pExMgGs0boY8yj883mGPa/ILfzeOIOUT0rcj5jSvGGc+Po
+hzdpM79teIwbgWIi9qD9WIhOe+QvWZETAX0kRHjR2BChokkkVN+NxZid3Wr0UOUi
+HKWPYonBeE6RJnnOcYz3dvcZnlVlSbzPMyWr9HPLLAN63aPI5KsGf1feWunR8+N+
+FNb47/7Yb+TptU/b0Fgnkib4lWONphtgUjmlDQIDAQABAoIBAFxba5FEjk9OSxjk
+RwRUjYBTPvtvjWpVxqVGza301j3E8rum+sLSIcmF92Gl8zTdx3tm5jehLL+b22iP
+Nav3UX6guTko8RkgyuqLu8+VF6aHtLPvETwwGmIAOnPKL5Aoz5KIlQORODyPn1X7
+bJR9/P6r/7iUb9+F1Ix6Lh4D1OOdqDKuEyIJBOjMuxtECq15tth6Ufmvp5mupT4x
+J5z9RqX36hoOWf4NvAv2YochDSvjYHZMVXsYgu2445ws36I3MGnMuIixDdvVbbtx
+NkUup+tNOZA2669OKwH+D71MGVKKiPq1W5R6J0LuBKdai1LgNLfFcNB27mcOXwaL
+BdRd6EECgYEA1lZYvobU27TdQkA/s+PaUAKw/3/tF/9sVe5V2vA4n15Fgd8KY3BO
+LzPSehBL1EOnyLccKi1K7PHOC/3NrM5Hxon9R5SIY5L8hfK1hVVONdUFgAsElzlf
+OOhsmRXs3Qq1qVfzW7ViipOUTGOYbTo7wSFZr9+rdh2WolhestmCFgMCgYEAq7wb
+VFOyyL267r9CvFsMB8Yy3xMTFJGLadVZJw0N0Em/nt9JEHxFhrr3lSJaYjAW+BHG
+82n3TUODdejwCSFGpI3E/nibhvgopmnQZm8uE8mtULDN388eKbh3iGR7He2dLJEM
+5bG8p5eLGM99god2hu54VYIsgf5jT2WcsAFOM68CgYAjPHK+b8ASlntBZvqSAkVj
+ne2nM0qxBUa0Ichvg8prOL5IiXhVvKK909EoTAGLVcwBjUjODkAhD+eFxSXI4Oif
+1ROUbvC1HfbxtmLtFocTBoAu+qC4k6/51Qv1ZstX02jl/BV/4CPhED3zCPSIEGi+
+aVMrPKQdeOPIsKpw0J04LQKBgCbRqGPRX5JcwMqC1TT6Z6fCN3GRQDjgBWFw5mwb
+WUoBwZzJ4Bwn2xdvX9OIJmIXeLmuWwhepZYDcs3OT6Pgr7U2jpbu8Ej8A0RKmt7s
+tr+mUNTygjba/Hh1yB8+h03mjiaqyv2IxZokeT6seDRvJm2trem2ORVRSWWDFH38
+bY7lAoGAGTsziUtoWbuly66fBjT7XTYDiKgYqn7xz4vyH+KoQ1meqpRqvuC8jkEn
+Ai2ne441btCrsZsH44msc/ss543XHv+Ybqp4aNK0VU72lsYcSMGYtsWCpDAGkZU0
+4zjkGDa+AzdAf2XDAsL+SiaMuDJ8wKovfLpgCkoryQ0Iwdfjkh0=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml
new file mode 100644
index 0000000000000..d5e00352b3e73
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml
@@ -0,0 +1,6 @@
+# A basic role that is used to call the IdP's APIs
+idp_role:
+  cluster:
+    - monitor
+    - "cluster:admin/idp/*"
+
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
new file mode 100644
index 0000000000000..b11a1e806c630
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.ssl.X509KeyPairSettings;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.action.SamlMetadataAction;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestAction;
+import org.elasticsearch.xpack.idp.action.TransportDeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.TransportPutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.TransportSamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.action.TransportSamlMetadataAction;
+import org.elasticsearch.xpack.idp.action.TransportSamlValidateAuthnRequestAction;
+import org.elasticsearch.xpack.idp.privileges.UserPrivilegeResolver;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestDeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestSamlMetadataAction;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestSamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestSamlValidateAuthenticationRequestAction;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestPutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+/**
+ * This plugin provides the backend for an IdP built on top of Elasticsearch security features.
+ * It is used internally within Elastic and is not intended for general use.
+ */
+public class IdentityProviderPlugin extends Plugin implements ActionPlugin {
+
+    private static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("xpack.idp.enabled", false, Setting.Property.NodeScope);
+
+    private final Logger logger = LogManager.getLogger();
+    private boolean enabled;
+    private Settings settings;
+
+    @Override
+    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry, Environment environment,
+                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
+                                               IndexNameExpressionResolver indexNameExpressionResolver) {
+        settings = environment.settings();
+        enabled = ENABLED_SETTING.get(settings);
+        if (enabled == false) {
+            return List.of();
+        }
+
+        SamlInit.initialize();
+        final SamlServiceProviderIndex index = new SamlServiceProviderIndex(client, clusterService);
+        final SecurityContext securityContext = new SecurityContext(settings, threadPool.getThreadContext());
+        final UserPrivilegeResolver userPrivilegeResolver = new UserPrivilegeResolver(client, securityContext);
+
+
+        final ServiceProviderDefaults serviceProviderDefaults = ServiceProviderDefaults.forSettings(settings);
+        final SamlServiceProviderResolver resolver = new SamlServiceProviderResolver(settings, index, serviceProviderDefaults);
+        final SamlIdentityProvider idp = SamlIdentityProvider.builder(resolver)
+            .fromSettings(environment)
+            .serviceProviderDefaults(serviceProviderDefaults)
+            .build();
+
+        final SamlFactory factory = new SamlFactory();
+
+        return List.of(
+            index,
+            idp,
+            factory,
+            userPrivilegeResolver
+        );
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        if (enabled == false) {
+            return List.of();
+        }
+        return List.of(
+            new ActionHandler<>(SamlInitiateSingleSignOnAction.INSTANCE, TransportSamlInitiateSingleSignOnAction.class),
+            new ActionHandler<>(SamlValidateAuthnRequestAction.INSTANCE, TransportSamlValidateAuthnRequestAction.class),
+            new ActionHandler<>(SamlMetadataAction.INSTANCE, TransportSamlMetadataAction.class),
+            new ActionHandler<>(PutSamlServiceProviderAction.INSTANCE, TransportPutSamlServiceProviderAction.class),
+            new ActionHandler<>(DeleteSamlServiceProviderAction.INSTANCE, TransportDeleteSamlServiceProviderAction.class)
+        );
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
+        if (enabled == false) {
+            return List.of();
+        }
+        return List.of(
+            new RestSamlInitiateSingleSignOnAction(getLicenseState()),
+            new RestSamlValidateAuthenticationRequestAction(getLicenseState()),
+            new RestSamlMetadataAction(getLicenseState()),
+            new RestPutSamlServiceProviderAction(getLicenseState()),
+            new RestDeleteSamlServiceProviderAction(getLicenseState())
+        );
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        List<Setting<?>> settings = new ArrayList<>();
+        settings.add(ENABLED_SETTING);
+        settings.addAll(SamlIdentityProviderBuilder.getSettings());
+        settings.addAll(ServiceProviderDefaults.getSettings());
+        settings.addAll(X509KeyPairSettings.withPrefix("xpack.idp.signing.", false).getAllSettings());
+        settings.addAll(X509KeyPairSettings.withPrefix("xpack.idp.metadata_signing.", false).getAllSettings());
+        return Collections.unmodifiableList(settings);
+    }
+
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..99ac5a6773534
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+/**
+ * Action to remove a service provider from the IdP.
+ */
+public class DeleteSamlServiceProviderAction extends ActionType<DeleteSamlServiceProviderResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/sp/delete";
+    public static final DeleteSamlServiceProviderAction INSTANCE = new DeleteSamlServiceProviderAction(NAME);
+
+    public DeleteSamlServiceProviderAction(String name) {
+        super(name, DeleteSamlServiceProviderResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java
new file mode 100644
index 0000000000000..d76b678f5652e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request object to remove a service provider (by Entity ID) from the IdP.
+ */
+public class DeleteSamlServiceProviderRequest extends ActionRequest {
+
+    private final String entityId;
+    private final WriteRequest.RefreshPolicy refreshPolicy;
+
+    public DeleteSamlServiceProviderRequest(String entityId, WriteRequest.RefreshPolicy refreshPolicy) {
+        this.entityId = entityId;
+        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "Refresh policy may not be null");
+    }
+
+    public DeleteSamlServiceProviderRequest(StreamInput in) throws IOException {
+        this.entityId = in.readString();
+        this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(entityId);
+        refreshPolicy.writeTo(out);
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(entityId)) {
+            validationException = addValidationError("The Service Provider Entity ID is required", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final DeleteSamlServiceProviderRequest that = (DeleteSamlServiceProviderRequest) o;
+        return Objects.equals(entityId, that.entityId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(entityId, refreshPolicy);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" + entityId + "," + refreshPolicy + "}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java
new file mode 100644
index 0000000000000..957b464d2a145
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+
+/**
+ * Response object after removing a service provider from the IdP.
+ */
+public class DeleteSamlServiceProviderResponse extends ActionResponse implements ToXContentObject {
+
+    private final String docId;
+    private final long seqNo;
+    private final long primaryTerm;
+    private final String entityId;
+
+    public DeleteSamlServiceProviderResponse(String docId, long seqNo, long primaryTerm, String entityId) {
+        this.docId = docId;
+        this.seqNo = seqNo;
+        this.primaryTerm = primaryTerm;
+        this.entityId = Objects.requireNonNull(entityId, "Entity Id cannot be null");
+    }
+
+    public DeleteSamlServiceProviderResponse(DeleteResponse deleteResponse, String entityId) {
+        this(deleteResponse == null ? null : deleteResponse.getId(),
+            deleteResponse == null ? UNASSIGNED_SEQ_NO : deleteResponse.getSeqNo(),
+            deleteResponse == null ? UNASSIGNED_PRIMARY_TERM : deleteResponse.getPrimaryTerm(),
+            entityId);
+    }
+
+    public DeleteSamlServiceProviderResponse(StreamInput in) throws IOException {
+        docId = in.readString();
+        seqNo = in.readZLong();
+        primaryTerm = in.readVLong();
+        entityId = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(docId);
+        out.writeZLong(seqNo);
+        out.writeVLong(primaryTerm);
+        out.writeString(entityId);
+    }
+
+    @Nullable
+    public String getDocId() {
+        return docId;
+    }
+
+    public long getSeqNo() {
+        return seqNo;
+    }
+
+    public long getPrimaryTerm() {
+        return primaryTerm;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startObject("document");
+        builder.field("_id", docId);
+        builder.field("_seq_no", seqNo);
+        builder.field("_primary_term", primaryTerm);
+        builder.endObject();
+
+        builder.startObject("service_provider");
+        builder.field("entity_id", entityId);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+
+    public boolean found() {
+        return docId != null;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..6ffc8c14c1d1a
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class PutSamlServiceProviderAction extends ActionType<PutSamlServiceProviderResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/sp/put";
+    public static final PutSamlServiceProviderAction INSTANCE = new PutSamlServiceProviderAction(NAME);
+
+    public PutSamlServiceProviderAction(String name) {
+        super(name, PutSamlServiceProviderResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java
new file mode 100644
index 0000000000000..ae7dd1485b8fe
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class PutSamlServiceProviderRequest extends ActionRequest {
+    public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.NONE;
+
+    private final SamlServiceProviderDocument document;
+    private final WriteRequest.RefreshPolicy refreshPolicy;
+
+    public static PutSamlServiceProviderRequest fromXContent(String entityId, WriteRequest.RefreshPolicy refreshPolicy,
+                                                             XContentParser parser) throws IOException {
+        final SamlServiceProviderDocument document = SamlServiceProviderDocument.fromXContent(null, parser);
+        if (document.entityId == null) {
+            document.setEntityId(entityId);
+        } else if (entityId != null) {
+            if (entityId.equals(document.entityId) == false) {
+                throw new ElasticsearchParseException(
+                    "Entity id [{}] inside request body and entity id [{}] from parameter do not match", document.entityId, entityId);
+            }
+        }
+        if (document.created != null) {
+            throw new ElasticsearchParseException(
+                "Field [{}] may not be specified in a request", SamlServiceProviderDocument.Fields.CREATED_DATE);
+        }
+        if (document.lastModified != null) {
+            throw new ElasticsearchParseException(
+                "Field [{}] may not be specified in a request", SamlServiceProviderDocument.Fields.LAST_MODIFIED);
+        }
+        document.setCreatedMillis(System.currentTimeMillis());
+        document.setLastModifiedMillis(System.currentTimeMillis());
+        return new PutSamlServiceProviderRequest(document, refreshPolicy);
+    }
+
+    public PutSamlServiceProviderRequest(SamlServiceProviderDocument document, WriteRequest.RefreshPolicy refreshPolicy) {
+        this.document = document;
+        this.refreshPolicy = refreshPolicy;
+    }
+
+    public PutSamlServiceProviderRequest(StreamInput in) throws IOException {
+        this.document = new SamlServiceProviderDocument(in);
+        this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        document.writeTo(out);
+        refreshPolicy.writeTo(out);
+    }
+
+    public SamlServiceProviderDocument getDocument() {
+        return document;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        final ValidationException docException = document.validate();
+        ActionRequestValidationException validationException = null;
+        if (docException != null) {
+            validationException = new ActionRequestValidationException();
+            validationException.addValidationErrors(docException.validationErrors());
+        }
+
+        if (Strings.hasText(document.acs)) { // if this is blank the document validation will fail
+            try {
+                final URL url = new URL(document.acs);
+                if (url.getProtocol().equals("https") == false) {
+                    validationException = addValidationError(
+                        "[" + SamlServiceProviderDocument.Fields.ACS + "] must use the [https] protocol", validationException);
+                }
+            } catch (MalformedURLException e) {
+                String error = "[" + SamlServiceProviderDocument.Fields.ACS + "] must be a valid URL";
+                if (e.getMessage() != null) {
+                    error += " - " + e.getMessage();
+                }
+                validationException = addValidationError(error, validationException);
+            }
+        }
+
+        if (document.certificates.identityProviderSigning.isEmpty() == false) {
+            validationException = addValidationError(
+                "[" + SamlServiceProviderDocument.Fields.Certificates.IDP_SIGNING + "] certificates may not be specified",
+                validationException);
+        }
+
+        if (document.certificates.identityProviderMetadataSigning.isEmpty() == false) {
+            validationException = addValidationError(
+                "[" + SamlServiceProviderDocument.Fields.Certificates.IDP_METADATA + "] certificates may not be specified",
+                validationException);
+        }
+
+        return validationException;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final PutSamlServiceProviderRequest that = (PutSamlServiceProviderRequest) o;
+        return Objects.equals(document, that.document);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(document);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" + document + "}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java
new file mode 100644
index 0000000000000..f6a89a9e4804f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutSamlServiceProviderResponse extends ActionResponse implements ToXContentObject {
+
+    private final String docId;
+    private final boolean created;
+    private final long seqNo;
+    private final long primaryTerm;
+    private final String entityId;
+    private final boolean enabled;
+
+    public PutSamlServiceProviderResponse(String docId, boolean created, long seqNo, long primaryTerm, String entityId, boolean enabled) {
+        this.docId = Objects.requireNonNull(docId, "Document Id cannot be null");
+        this.created = created;
+        this.seqNo = seqNo;
+        this.primaryTerm = primaryTerm;
+        this.entityId = Objects.requireNonNull(entityId, "Entity Id cannot be null");
+        this.enabled = enabled;
+    }
+
+    public PutSamlServiceProviderResponse(StreamInput in) throws IOException {
+        docId = in.readString();
+        created = in.readBoolean();
+        seqNo = in.readZLong();
+        primaryTerm = in.readVLong();
+        entityId = in.readString();
+        enabled = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(docId);
+        out.writeBoolean(created);
+        out.writeZLong(seqNo);
+        out.writeVLong(primaryTerm);
+        out.writeString(entityId);
+        out.writeBoolean(enabled);
+    }
+
+    public String getDocId() {
+        return docId;
+    }
+
+    public boolean isCreated() {
+        return created;
+    }
+
+    public long getSeqNo() {
+        return seqNo;
+    }
+
+    public long getPrimaryTerm() {
+        return primaryTerm;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startObject("document");
+        builder.field("_id", docId);
+        builder.field("_created", created);
+        builder.field("_seq_no", seqNo);
+        builder.field("_primary_term", primaryTerm);
+        builder.endObject();
+
+        builder.startObject("service_provider");
+        builder.field("entity_id", entityId);
+        builder.field("enabled", enabled);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java
new file mode 100644
index 0000000000000..92c492842ee34
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+/**
+ * ActionType to create a SAML Response in the context of IDP initiated SSO for a given SP
+ */
+public class SamlInitiateSingleSignOnAction extends ActionType<SamlInitiateSingleSignOnResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/init";
+    public static final SamlInitiateSingleSignOnAction INSTANCE = new SamlInitiateSingleSignOnAction();
+
+    private SamlInitiateSingleSignOnAction() {
+        super(NAME, SamlInitiateSingleSignOnResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java
new file mode 100644
index 0000000000000..f29f9cf066f38
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+import java.io.IOException;
+
+public class SamlInitiateSingleSignOnRequest extends ActionRequest {
+
+    private String spEntityId;
+    private SamlAuthenticationState samlAuthenticationState;
+
+    public SamlInitiateSingleSignOnRequest(StreamInput in) throws IOException {
+        super(in);
+        spEntityId = in.readString();
+        samlAuthenticationState = in.readOptionalWriteable(SamlAuthenticationState::new);
+    }
+
+    public SamlInitiateSingleSignOnRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(spEntityId)) {
+            validationException = addValidationError("entity_id is missing", validationException);
+        }
+        if (samlAuthenticationState != null) {
+            final ValidationException authnStateException = samlAuthenticationState.validate();
+            if (validationException != null) {
+                ActionRequestValidationException actionRequestValidationException = new ActionRequestValidationException();
+                actionRequestValidationException.addValidationErrors(authnStateException.validationErrors());
+                validationException = addValidationError("entity_id is missing", actionRequestValidationException);
+            }
+        }
+        return validationException;
+    }
+
+    public String getSpEntityId() {
+        return spEntityId;
+    }
+
+    public void setSpEntityId(String spEntityId) {
+        this.spEntityId = spEntityId;
+    }
+
+    public SamlAuthenticationState getSamlAuthenticationState() {
+        return samlAuthenticationState;
+    }
+
+    public void setSamlAuthenticationState(SamlAuthenticationState samlAuthenticationState) {
+        this.samlAuthenticationState = samlAuthenticationState;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(spEntityId);
+        out.writeOptionalWriteable(samlAuthenticationState);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{spEntityId='" + spEntityId + "'}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java
new file mode 100644
index 0000000000000..7d978152d0377
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public class SamlInitiateSingleSignOnResponse extends ActionResponse {
+
+    private String postUrl;
+    private String samlResponse;
+    private String entityId;
+
+    public SamlInitiateSingleSignOnResponse(StreamInput in) throws IOException {
+        super(in);
+        this.postUrl = in.readString();
+        this.samlResponse = in.readString();
+        this.entityId = in.readString();
+    }
+
+    public SamlInitiateSingleSignOnResponse(String postUrl, String samlResponse, String entityId) {
+        this.postUrl = postUrl;
+        this.samlResponse = samlResponse;
+        this.entityId = entityId;
+    }
+
+    public String getPostUrl() {
+        return postUrl;
+    }
+
+    public String getSamlResponse() {
+        return samlResponse;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(postUrl);
+        out.writeString(samlResponse);
+        out.writeString(entityId);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java
new file mode 100644
index 0000000000000..4c816ef122a97
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class SamlMetadataAction extends ActionType<SamlMetadataResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/metadata";
+    public static final SamlMetadataAction INSTANCE = new SamlMetadataAction();
+
+    private SamlMetadataAction() {
+        super(NAME, SamlMetadataResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java
new file mode 100644
index 0000000000000..86ba297629a3f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SamlMetadataRequest extends ActionRequest {
+
+    private String spEntityId;
+
+    public SamlMetadataRequest(StreamInput in) throws IOException {
+        super(in);
+        spEntityId = in.readString();
+    }
+
+    public SamlMetadataRequest(String spEntityId) {
+        this.spEntityId = Objects.requireNonNull(spEntityId, "Service Provider entity id must be provided");
+    }
+
+    public SamlMetadataRequest() {
+        this.spEntityId = null;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    public String getSpEntityId() {
+        return spEntityId;
+    }
+
+    public void setSpEntityId(String spEntityId) {
+        this.spEntityId = spEntityId;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{spEntityId='" + spEntityId + "'}";
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java
new file mode 100644
index 0000000000000..0839dc70b4803
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SamlMetadataResponse extends ActionResponse {
+
+    private final String xmlString;
+
+    public SamlMetadataResponse(StreamInput in) throws IOException {
+        super(in);
+        this.xmlString = in.readString();
+    }
+
+    public SamlMetadataResponse(String xmlString) {
+        this.xmlString = Objects.requireNonNull(xmlString, "Metadata XML string must be provided");
+    }
+
+    public String getXmlString() {
+        return xmlString;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(xmlString);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java
new file mode 100644
index 0000000000000..bd246d0f9ff7f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class SamlValidateAuthnRequestAction extends ActionType<SamlValidateAuthnRequestResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/validate";
+    public static final SamlValidateAuthnRequestAction INSTANCE = new SamlValidateAuthnRequestAction();
+
+    private SamlValidateAuthnRequestAction() {
+        super(NAME, SamlValidateAuthnRequestResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java
new file mode 100644
index 0000000000000..1fffcf23871d0
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class SamlValidateAuthnRequestRequest extends ActionRequest {
+
+    private String queryString;
+
+    public SamlValidateAuthnRequestRequest(StreamInput in) throws IOException {
+        super(in);
+        queryString = in.readString();
+    }
+
+    public SamlValidateAuthnRequestRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(queryString)) {
+            validationException = addValidationError("Authentication request query string must be provided", validationException);
+        }
+        return validationException;
+    }
+
+    public String getQueryString() {
+        return queryString;
+    }
+
+    public void setQueryString(String queryString) {
+        this.queryString = queryString;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(queryString);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{queryString='" + queryString + "'}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java
new file mode 100644
index 0000000000000..cf06efc53084f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class SamlValidateAuthnRequestResponse extends ActionResponse {
+
+    private final String spEntityId;
+    private final boolean forceAuthn;
+    private final Map<String, Object> authnState;
+
+    public SamlValidateAuthnRequestResponse(StreamInput in) throws IOException {
+        super(in);
+        this.spEntityId = in.readString();
+        this.forceAuthn = in.readBoolean();
+        this.authnState = in.readMap();
+    }
+
+    public SamlValidateAuthnRequestResponse(String spEntityId, boolean forceAuthn, Map<String, Object> authnState) {
+        this.spEntityId = Objects.requireNonNull(spEntityId, "spEntityId is required for successful responses");
+        this.forceAuthn = forceAuthn;
+        this.authnState = Map.copyOf(Objects.requireNonNull(authnState));
+    }
+
+    public String getSpEntityId() {
+        return spEntityId;
+    }
+
+    public boolean isForceAuthn() {
+        return forceAuthn;
+    }
+
+    public Map<String, Object> getAuthnState() {
+        return authnState;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(spEntityId);
+        out.writeBoolean(forceAuthn);
+        out.writeMap(authnState);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{ spEntityId='" + getSpEntityId() + "',\n" +
+            " forceAuthn='" + isForceAuthn() + "',\n" +
+            " authnState='" + getAuthnState() + "' }";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..f32505046c1ad
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+
+import java.util.stream.Collectors;
+
+/**
+ * Transport action to remove a service provider from the IdP
+ */
+public class TransportDeleteSamlServiceProviderAction
+    extends HandledTransportAction<DeleteSamlServiceProviderRequest, DeleteSamlServiceProviderResponse> {
+
+    private final Logger logger = LogManager.getLogger();
+    private final SamlServiceProviderIndex index;
+
+    @Inject
+    public TransportDeleteSamlServiceProviderAction(TransportService transportService, ActionFilters actionFilters,
+                                                    SamlServiceProviderIndex index) {
+        super(DeleteSamlServiceProviderAction.NAME, transportService, actionFilters, DeleteSamlServiceProviderRequest::new);
+        this.index = index;
+    }
+
+    @Override
+    protected void doExecute(Task task, final DeleteSamlServiceProviderRequest request,
+                             final ActionListener<DeleteSamlServiceProviderResponse> listener) {
+        final String entityId = request.getEntityId();
+        index.findByEntityId(entityId, ActionListener.wrap(matchingDocuments -> {
+            if (matchingDocuments.isEmpty()) {
+                listener.onResponse(new DeleteSamlServiceProviderResponse(null, entityId));
+            } else if (matchingDocuments.size() == 1) {
+                final SamlServiceProviderIndex.DocumentSupplier docInfo = Iterables.get(matchingDocuments, 0);
+                final SamlServiceProviderDocument existingDoc = docInfo.getDocument();
+                assert existingDoc.docId != null : "Loaded document with no doc id";
+                assert existingDoc.entityId.equals(entityId) : "Loaded document with non-matching entity-id";
+                logger.info("Deleting Service Provider [{}]", existingDoc);
+                index.deleteDocument(docInfo.version, request.getRefreshPolicy(), ActionListener.wrap(
+                    deleteResponse -> listener.onResponse(new DeleteSamlServiceProviderResponse(deleteResponse, entityId)),
+                    listener::onFailure
+                ));
+            } else {
+                logger.warn("Found multiple existing service providers in [{}] with entity id [{}] - [{}]",
+                    index, entityId, matchingDocuments.stream().map(d -> d.getDocument().docId).collect(Collectors.joining(",")));
+                listener.onFailure(new IllegalStateException("Multiple service providers exist with entity id [" + entityId + "]"));
+            }
+        }, listener::onFailure));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..e2556240b5755
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.stream.Collectors;
+
+public class TransportPutSamlServiceProviderAction
+    extends HandledTransportAction<PutSamlServiceProviderRequest, PutSamlServiceProviderResponse> {
+
+    private final Logger logger = LogManager.getLogger();
+    private final SamlServiceProviderIndex index;
+    private final SamlIdentityProvider identityProvider;
+    private final Clock clock;
+
+    @Inject
+    public TransportPutSamlServiceProviderAction(TransportService transportService, ActionFilters actionFilters,
+                                                 SamlServiceProviderIndex index, SamlIdentityProvider identityProvider) {
+        this(transportService, actionFilters, index, identityProvider, Clock.systemUTC());
+    }
+
+    TransportPutSamlServiceProviderAction(TransportService transportService, ActionFilters actionFilters,
+                                          SamlServiceProviderIndex index, SamlIdentityProvider identityProvider, Clock clock) {
+        super(PutSamlServiceProviderAction.NAME, transportService, actionFilters, PutSamlServiceProviderRequest::new);
+        this.index = index;
+        this.identityProvider = identityProvider;
+        this.clock = clock;
+    }
+
+    @Override
+    protected void doExecute(Task task, final PutSamlServiceProviderRequest request,
+                             final ActionListener<PutSamlServiceProviderResponse> listener) {
+        final SamlServiceProviderDocument document = request.getDocument();
+        if (document.docId != null) {
+            listener.onFailure(new IllegalArgumentException("request document must not have an id [" + document.docId + "]"));
+            return;
+        }
+        if (document.nameIdFormat != null && identityProvider.getAllowedNameIdFormats().contains(document.nameIdFormat) == false) {
+            listener.onFailure(new IllegalArgumentException("NameID format [" + document.nameIdFormat + "] is not supported."));
+            return;
+        }
+        index.findByEntityId(document.entityId, ActionListener.wrap(matchingDocuments -> {
+            if (matchingDocuments.isEmpty()) {
+                // derive a document id from the entity id so that don't accidentally create duplicate entities due to a race condition
+                document.docId = deriveDocumentId(document);
+                // force a create in case there are concurrent requests. This way, if two nodes/threads are trying to create the SP at
+                // the same time, one will fail. That's not ideal, but it's better than having 1 silently overwrite the other.
+                writeDocument(document, DocWriteRequest.OpType.CREATE, request.getRefreshPolicy(), listener);
+            } else if (matchingDocuments.size() == 1) {
+                final SamlServiceProviderDocument existingDoc = Iterables.get(matchingDocuments, 0).getDocument();
+                assert existingDoc.docId != null : "Loaded document with no doc id";
+                assert existingDoc.entityId.equals(document.entityId) : "Loaded document with non-matching entity-id";
+                document.setDocId(existingDoc.docId);
+                document.setCreated(existingDoc.created);
+                writeDocument(document, DocWriteRequest.OpType.INDEX, request.getRefreshPolicy(), listener);
+            } else {
+                logger.warn("Found multiple existing service providers in [{}] with entity id [{}] - [{}]",
+                    index, document.entityId, matchingDocuments.stream().map(d -> d.getDocument().docId).collect(Collectors.joining(",")));
+                listener.onFailure(new IllegalStateException(
+                    "Multiple service providers already exist with entity id [" + document.entityId + "]"));
+            }
+        }, listener::onFailure));
+    }
+
+    private void writeDocument(SamlServiceProviderDocument document, DocWriteRequest.OpType opType,
+                               WriteRequest.RefreshPolicy refreshPolicy, ActionListener<PutSamlServiceProviderResponse> listener) {
+
+        final Instant now = clock.instant();
+        if (document.created == null || opType == DocWriteRequest.OpType.CREATE) {
+            document.created = now;
+        }
+        document.lastModified = now;
+        final ValidationException validationException = document.validate();
+        if (validationException != null) {
+            listener.onFailure(validationException);
+            return;
+        }
+        logger.debug("[{}] service provider [{}] in document [{}] of [{}]", opType, document.entityId, document.docId, index);
+        index.writeDocument(document, opType, refreshPolicy, ActionListener.wrap(
+            response -> listener.onResponse(new PutSamlServiceProviderResponse(
+                response.getId(),
+                response.getResult() == DocWriteResponse.Result.CREATED,
+                response.getSeqNo(),
+                response.getPrimaryTerm(),
+                document.entityId,
+                document.enabled)),
+            listener::onFailure
+        ));
+    }
+
+    private String deriveDocumentId(SamlServiceProviderDocument document) {
+        final byte[] sha256 = MessageDigests.sha256().digest(document.entityId.getBytes(StandardCharsets.UTF_8));
+        return Base64.getUrlEncoder().withoutPadding().encodeToString(sha256);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
new file mode 100644
index 0000000000000..11aff7bf81d1b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authc.support.SecondaryAuthentication;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.idp.privileges.UserPrivilegeResolver;
+import org.elasticsearch.xpack.idp.saml.authn.FailedAuthenticationResponseMessageBuilder;
+import org.elasticsearch.xpack.idp.saml.authn.SuccessfulAuthenticationResponseMessageBuilder;
+import org.elasticsearch.xpack.idp.saml.authn.UserServiceAuthentication;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.StatusCode;
+
+import java.time.Clock;
+
+public class TransportSamlInitiateSingleSignOnAction
+    extends HandledTransportAction<SamlInitiateSingleSignOnRequest, SamlInitiateSingleSignOnResponse> {
+
+    private final Logger logger = LogManager.getLogger(TransportSamlInitiateSingleSignOnAction.class);
+
+    private final SecurityContext securityContext;
+    private final SamlIdentityProvider identityProvider;
+    private final SamlFactory samlFactory;
+    private final UserPrivilegeResolver privilegeResolver;
+
+    @Inject
+    public TransportSamlInitiateSingleSignOnAction(TransportService transportService, ActionFilters actionFilters,
+                                                   SecurityContext securityContext, SamlIdentityProvider idp, SamlFactory factory,
+                                                   UserPrivilegeResolver privilegeResolver) {
+        super(SamlInitiateSingleSignOnAction.NAME, transportService, actionFilters, SamlInitiateSingleSignOnRequest::new);
+        this.securityContext = securityContext;
+        this.identityProvider = idp;
+        this.samlFactory = factory;
+        this.privilegeResolver = privilegeResolver;
+    }
+
+    @Override
+    protected void doExecute(Task task, SamlInitiateSingleSignOnRequest request,
+                             ActionListener<SamlInitiateSingleSignOnResponse> listener) {
+        final SamlAuthenticationState authenticationState = request.getSamlAuthenticationState();
+        identityProvider.getRegisteredServiceProvider(request.getSpEntityId(), false, ActionListener.wrap(
+            sp -> {
+                if (null == sp) {
+                    final String message = "Service Provider with Entity ID [" + request.getSpEntityId()
+                        + "] is not registered with this Identity Provider";
+                    logger.debug(message);
+                    possiblyReplyWithSamlFailure(authenticationState, StatusCode.RESPONDER, new IllegalArgumentException(message),
+                        listener);
+                    return;
+                }
+                final SecondaryAuthentication secondaryAuthentication = SecondaryAuthentication.readFromContext(securityContext);
+                if (secondaryAuthentication == null) {
+                    possiblyReplyWithSamlFailure(authenticationState,
+                        StatusCode.REQUESTER,
+                        new ElasticsearchSecurityException("Request is missing secondary authentication", RestStatus.FORBIDDEN),
+                        listener);
+                    return;
+                }
+                buildUserFromAuthentication(secondaryAuthentication, sp, ActionListener.wrap(
+                    user -> {
+                        if (user == null) {
+                            possiblyReplyWithSamlFailure(authenticationState,
+                                StatusCode.REQUESTER,
+                                new ElasticsearchSecurityException("User [{}] is not permitted to access service [{}]",
+                                    RestStatus.FORBIDDEN, secondaryAuthentication.getUser(), sp),
+                                listener);
+                            return;
+                        }
+                        final SuccessfulAuthenticationResponseMessageBuilder builder =
+                            new SuccessfulAuthenticationResponseMessageBuilder(samlFactory, Clock.systemUTC(), identityProvider);
+                        try {
+                            final Response response = builder.build(user, authenticationState);
+                            listener.onResponse(new SamlInitiateSingleSignOnResponse(
+                                user.getServiceProvider().getAssertionConsumerService().toString(),
+                                samlFactory.getXmlContent(response),
+                                user.getServiceProvider().getEntityId()));
+                        } catch (ElasticsearchException e) {
+                            listener.onFailure(e);
+                        }
+                    },
+                    e -> possiblyReplyWithSamlFailure(authenticationState, StatusCode.RESPONDER, e, listener)
+                ));
+            },
+            e -> possiblyReplyWithSamlFailure(authenticationState, StatusCode.RESPONDER, e, listener)
+        ));
+    }
+
+    private void buildUserFromAuthentication(SecondaryAuthentication secondaryAuthentication, SamlServiceProvider serviceProvider,
+                                             ActionListener<UserServiceAuthentication> listener) {
+        User user = secondaryAuthentication.getUser();
+        secondaryAuthentication.execute(ignore -> {
+                privilegeResolver.resolve(serviceProvider.getPrivileges(), ActionListener.wrap(
+                    userPrivileges -> {
+                        if (userPrivileges.hasAccess == false) {
+                            listener.onResponse(null);
+                        } else {
+                            logger.debug("Resolved [{}] for [{}]", userPrivileges, user);
+                            listener.onResponse(new UserServiceAuthentication(user.principal(), user.fullName(), user.email(),
+                                userPrivileges.roles, serviceProvider));
+                        }
+                    },
+                    listener::onFailure
+                ));
+                return null;
+            }
+        );
+    }
+
+    private void possiblyReplyWithSamlFailure(SamlAuthenticationState authenticationState, String statusCode, Exception e,
+                                              ActionListener<SamlInitiateSingleSignOnResponse> listener) {
+        if (authenticationState != null) {
+            final FailedAuthenticationResponseMessageBuilder builder =
+                new FailedAuthenticationResponseMessageBuilder(samlFactory, Clock.systemUTC(), identityProvider)
+                    .setInResponseTo(authenticationState.getAuthnRequestId())
+                    .setAcsUrl(authenticationState.getRequestedAcsUrl())
+                    .setPrimaryStatusCode(statusCode);
+            final Response response = builder.build();
+            //TODO: Log and indicate SAML Response status is failure in the response
+            listener.onResponse(new SamlInitiateSingleSignOnResponse(
+                authenticationState.getRequestedAcsUrl(),
+                samlFactory.getXmlContent(response),
+                authenticationState.getEntityId()));
+        } else {
+            listener.onFailure(e);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
new file mode 100644
index 0000000000000..9d17fae407a7e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.idp.SamlMetadataGenerator;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+
+public class TransportSamlMetadataAction extends HandledTransportAction<SamlMetadataRequest, SamlMetadataResponse> {
+
+    private final SamlIdentityProvider identityProvider;
+    private final SamlFactory samlFactory;
+
+    @Inject
+    public TransportSamlMetadataAction(TransportService transportService, ActionFilters actionFilters,
+                                       SamlIdentityProvider idp, SamlFactory factory) {
+        super(SamlMetadataAction.NAME, transportService, actionFilters, SamlMetadataRequest::new);
+        this.identityProvider = idp;
+        this.samlFactory = factory;
+    }
+
+    @Override
+    protected void doExecute(Task task, SamlMetadataRequest request, ActionListener<SamlMetadataResponse> listener) {
+        final String spEntityId = request.getSpEntityId();
+        final SamlMetadataGenerator generator = new SamlMetadataGenerator(samlFactory, identityProvider);
+        generator.generateMetadata(spEntityId, listener);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
new file mode 100644
index 0000000000000..9b2b1d3a819ab
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.authn.SamlAuthnRequestValidator;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+
+public class TransportSamlValidateAuthnRequestAction
+    extends HandledTransportAction<SamlValidateAuthnRequestRequest, SamlValidateAuthnRequestResponse> {
+
+    private final SamlIdentityProvider identityProvider;
+    private final SamlFactory samlFactory;
+
+    @Inject
+    public TransportSamlValidateAuthnRequestAction(TransportService transportService, ActionFilters actionFilters,
+                                                   SamlIdentityProvider idp, SamlFactory factory) {
+        super(SamlValidateAuthnRequestAction.NAME, transportService, actionFilters, SamlValidateAuthnRequestRequest::new);
+        this.identityProvider = idp;
+        this.samlFactory = factory;
+    }
+
+    @Override
+    protected void doExecute(Task task, SamlValidateAuthnRequestRequest request,
+                             ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        final SamlAuthnRequestValidator validator = new SamlAuthnRequestValidator(samlFactory, identityProvider);
+        try {
+            validator.processQueryString(request.getQueryString(), listener);
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java
new file mode 100644
index 0000000000000..32d93261f0cf2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.authc;
+
+/**
+ * Denotes support authentication methods for users
+ */
+public enum AuthenticationMethod {
+    PASSWORD,
+    KERBEROS,
+    TLS_CLIENT_AUTH,
+    PRIOR_SESSION,
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java
new file mode 100644
index 0000000000000..00f53dff6f032
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.authc;
+
+/**
+ * Denotes network based controls that were applied during authentication of a user
+ */
+public enum NetworkControl {
+    IP_FILTER,
+    TLS,
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java
new file mode 100644
index 0000000000000..794cdc7b72ac1
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.privileges;
+
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+
+import java.util.Map;
+import java.util.Objects;
+
+public class ServiceProviderPrivileges {
+
+    private final String applicationName;
+    private final String resource;
+    private final Map<String, String> roles;
+
+    public ServiceProviderPrivileges(String applicationName, String resource, Map<String, String> roles) {
+        this.applicationName = Objects.requireNonNull(applicationName, "Application name cannot be null");
+        this.resource = Objects.requireNonNull(resource, "Resource cannot be null");
+        this.roles = Map.copyOf(roles);
+    }
+
+    /**
+     * Returns the "application" (see {@link RoleDescriptor.ApplicationResourcePrivileges#getApplication()}) in the IdP's security cluster
+     * under which this service provider is defined.
+     */
+    public String getApplicationName() {
+        return applicationName;
+    }
+
+    /**
+     * Returns the "resource" (see {@link RoleDescriptor.ApplicationResourcePrivileges#getResources()}) that represents this
+     * Service Provider in the IdP's security cluster.
+     */
+    public String getResource() {
+        return resource;
+    }
+
+    /**
+     * Returns a mapping from "role name" (key) to "{@link ApplicationPrivilegeDescriptor#getActions() action name}" (value)
+     * that represents the roles that should be exposed to this Service Provider.
+     * The "role name" (but not the action name) will be provided to the service provider.
+     * These roles have no semantic meaning within the IdP, they are simply metadata that we pass to the Service Provider. They may not
+     * have any relationship to the roles that the user has in this cluster, and the service provider may refer to them using a different
+     * terminology (e.g. "groups").
+     * The actions will be resolved as application privileges from the IdP's security cluster.
+     */
+    public Map<String, String> getRoleActions() {
+        return roles;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java
new file mode 100644
index 0000000000000..8c0e67dbbe166
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.privileges;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Determines what privileges a user has within a given {@link ServiceProviderPrivileges service}.
+ */
+public class UserPrivilegeResolver {
+
+    public static class UserPrivileges {
+        public final String principal;
+        public final boolean hasAccess;
+        public final Set<String> roles;
+
+        public UserPrivileges(String principal, boolean hasAccess, Set<String> roles) {
+            this.principal = Objects.requireNonNull(principal, "principal may not be null");
+            if (hasAccess == false && roles.isEmpty() == false) {
+                throw new IllegalArgumentException("a user without access ([" + hasAccess + "]) may not have roles ([" + roles + "])");
+            }
+            this.hasAccess = hasAccess;
+            this.roles = Set.copyOf(Objects.requireNonNull(roles, "roles may not be null"));
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder str = new StringBuilder()
+                .append(getClass().getSimpleName())
+                .append("{")
+                .append(principal)
+                .append(", ")
+                .append(hasAccess);
+            if (hasAccess) {
+                str.append(", ").append(roles);
+            }
+            str.append("}");
+            return str.toString();
+        }
+
+        public static UserPrivileges noAccess(String principal) {
+            return new UserPrivileges(principal, false, Set.of());
+        }
+    }
+
+    private final Logger logger = LogManager.getLogger();
+    private final Client client;
+    private final SecurityContext securityContext;
+
+    public UserPrivilegeResolver(Client client, SecurityContext securityContext) {
+        this.client = client;
+        this.securityContext = securityContext;
+    }
+
+    /**
+     * Resolves the user's privileges for the specified service.
+     * Requires that the active user is set in the {@link org.elasticsearch.xpack.core.security.SecurityContext}.
+     */
+    public void resolve(ServiceProviderPrivileges service, ActionListener<UserPrivileges> listener) {
+        HasPrivilegesRequest request = new HasPrivilegesRequest();
+        final String username = securityContext.requireUser().principal();
+        request.username(username);
+        request.applicationPrivileges(buildResourcePrivilege(service));
+        request.clusterPrivileges(Strings.EMPTY_ARRAY);
+        request.indexPrivileges(new RoleDescriptor.IndicesPrivileges[0]);
+        client.execute(HasPrivilegesAction.INSTANCE, request, ActionListener.wrap(
+            response -> {
+                logger.debug("Checking access for user [{}] to application [{}] resource [{}]",
+                    username, service.getApplicationName(), service.getResource());
+                UserPrivileges privileges = buildResult(response, service);
+                logger.debug("Resolved service privileges [{}]", privileges);
+                listener.onResponse(privileges);
+            },
+            listener::onFailure
+        ));
+    }
+
+    private UserPrivileges buildResult(HasPrivilegesResponse response, ServiceProviderPrivileges service) {
+        final Set<ResourcePrivileges> appPrivileges = response.getApplicationPrivileges().get(service.getApplicationName());
+        if (appPrivileges == null || appPrivileges.isEmpty()) {
+            return UserPrivileges.noAccess(response.getUsername());
+        }
+        final Set<String> roles = service.getRoleActions().entrySet().stream()
+            .filter(entry -> checkAccess(appPrivileges, entry.getValue(), service.getResource()))
+            .map(Map.Entry::getKey)
+            .collect(Collectors.toUnmodifiableSet());
+        final boolean hasAccess = roles.isEmpty() == false;
+        return new UserPrivileges(response.getUsername(), hasAccess, roles);
+    }
+
+    private boolean checkAccess(Set<ResourcePrivileges> userPrivileges, String action, String resource) {
+        final Optional<ResourcePrivileges> match = userPrivileges.stream()
+            .filter(rp -> rp.getResource().equals(resource))
+            .filter(rp -> rp.isAllowed(action))
+            .findAny();
+        match.ifPresent(rp -> logger.debug("User has access to [{} on {}] via [{}]", action, resource, rp));
+        return match.isPresent();
+    }
+
+    private RoleDescriptor.ApplicationResourcePrivileges buildResourcePrivilege(ServiceProviderPrivileges service) {
+        final RoleDescriptor.ApplicationResourcePrivileges.Builder builder = RoleDescriptor.ApplicationResourcePrivileges.builder();
+        builder.application(service.getApplicationName());
+        builder.resources(service.getResource());
+        builder.privileges(service.getRoleActions().values());
+        return builder.build();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java
new file mode 100644
index 0000000000000..60781526492ab
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.Status;
+import org.opensaml.saml.saml2.core.StatusCode;
+import org.opensaml.saml.saml2.core.StatusMessage;
+
+import java.time.Clock;
+
+public class FailedAuthenticationResponseMessageBuilder {
+
+    private final Clock clock;
+    private final SamlFactory samlFactory;
+    private final SamlIdentityProvider idp;
+    private String primaryStatusCode;
+    private String statusMessage;
+    private String acsUrl;
+    private String inResponseTo;
+    private String secondaryStatusCode;
+
+    public FailedAuthenticationResponseMessageBuilder(SamlFactory samlFactory, Clock clock, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.samlFactory = samlFactory;
+        this.clock = clock;
+        this.idp = idp;
+        this.primaryStatusCode = StatusCode.REQUESTER;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setAcsUrl(String acsUrl) {
+        this.acsUrl = acsUrl;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setPrimaryStatusCode(String primaryStatusCode) {
+        this.primaryStatusCode = primaryStatusCode;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setStatusMessage(String statusMessage) {
+        this.statusMessage = statusMessage;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setSecondaryStatusCode(String secondaryStatusCode) {
+        this.secondaryStatusCode = secondaryStatusCode;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setInResponseTo(String inResponseTo) {
+        this.inResponseTo = inResponseTo;
+        return this;
+    }
+
+    public Response build() {
+        final DateTime now = now();
+        final Response response = samlFactory.object(Response.class, Response.DEFAULT_ELEMENT_NAME);
+        response.setID(samlFactory.secureIdentifier());
+        response.setInResponseTo(inResponseTo);
+        response.setIssuer(buildIssuer());
+        response.setIssueInstant(now);
+        response.setStatus(buildStatus());
+        response.setDestination(acsUrl);
+        return response;
+    }
+
+    private Issuer buildIssuer() {
+        final Issuer issuer = samlFactory.object(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(this.idp.getEntityId());
+        return issuer;
+    }
+
+    private Status buildStatus() {
+        final StatusCode firstLevelCode = samlFactory.object(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+        firstLevelCode.setValue(primaryStatusCode);
+
+        if (Strings.hasText(secondaryStatusCode)) {
+            final StatusCode secondLevelCode = samlFactory.object(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+            secondLevelCode.setValue(secondaryStatusCode);
+            firstLevelCode.setStatusCode(secondLevelCode);
+        }
+
+        final Status status = samlFactory.object(Status.class, Status.DEFAULT_ELEMENT_NAME);
+        if (Strings.hasText(statusMessage)) {
+            final StatusMessage firstLevelMessage = samlFactory.object(StatusMessage.class, StatusMessage.DEFAULT_ELEMENT_NAME);
+            firstLevelMessage.setMessage(statusMessage);
+            status.setStatusMessage(firstLevelMessage);
+        }
+        status.setStatusCode(firstLevelCode);
+        return status;
+    }
+
+    private DateTime now() {
+        return new DateTime(clock.millis(), DateTimeZone.UTC);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java
new file mode 100644
index 0000000000000..eba6fe390e91e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.internal.io.Streams;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestResponse;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.opensaml.saml.saml2.core.AuthnRequest;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameIDPolicy;
+import org.opensaml.security.x509.X509Credential;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.UNSPECIFIED;
+
+/**
+ * Processes a SAML AuthnRequest, validates it, extracts necessary information and returns a {@link SamlValidateAuthnRequestResponse}
+ */
+public class SamlAuthnRequestValidator {
+
+    private final SamlFactory samlFactory;
+    private final SamlIdentityProvider idp;
+    private final Logger logger = LogManager.getLogger(SamlAuthnRequestValidator.class);
+    private static final String[] XSD_FILES = new String[]{"/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd",
+        "/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd",
+        "/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd",
+        "/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd"};
+
+    private static final ThreadLocal<DocumentBuilder> THREAD_LOCAL_DOCUMENT_BUILDER = ThreadLocal.withInitial(() -> {
+        try {
+            return SamlFactory.getHardenedBuilder(XSD_FILES);
+        } catch (Exception e) {
+            throw new ElasticsearchSecurityException("Could not load XSD schema file", e);
+        }
+    });
+
+    public SamlAuthnRequestValidator(SamlFactory samlFactory, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.samlFactory = samlFactory;
+        this.idp = idp;
+    }
+
+    public void processQueryString(String queryString, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+
+        final ParsedQueryString parsedQueryString;
+        try {
+            parsedQueryString = parseQueryString(queryString);
+        } catch (ElasticsearchSecurityException e) {
+            logger.debug("Failed to parse query string for SAML AuthnRequest", e);
+            listener.onFailure(e);
+            return;
+        }
+
+        try {
+            // We consciously parse the AuthnRequest before we validate its signature as we need to get the Issuer, in order to
+            // verify if we know of this SP and get its credentials for signature verification
+            final Element root = parseSamlMessage(inflate(decodeBase64(parsedQueryString.samlRequest)));
+            if (samlFactory.elementNameMatches(root, "urn:oasis:names:tc:SAML:2.0:protocol", "AuthnRequest") == false) {
+                logAndRespond(new ParameterizedMessage("SAML message [{}] is not an AuthnRequest", samlFactory.text(root, 128)), listener);
+                return;
+            }
+            final AuthnRequest authnRequest = samlFactory.buildXmlObject(root, AuthnRequest.class);
+            getSpFromIssuer(authnRequest.getIssuer(), ActionListener.wrap(
+                sp -> {
+                    try {
+                        validateAuthnRequest(authnRequest, sp, parsedQueryString, listener);
+                    } catch (ElasticsearchSecurityException e) {
+                        logger.debug("Could not validate AuthnRequest", e);
+                        listener.onFailure(e);
+                    } catch (Exception e) {
+                        logAndRespond("Could not validate AuthnRequest", e, listener);
+                    }
+                },
+                listener::onFailure
+            ));
+        } catch (ElasticsearchSecurityException e) {
+            logger.debug("Could not process AuthnRequest", e);
+            listener.onFailure(e);
+        } catch (Exception e) {
+            logAndRespond("Could not process AuthnRequest", e, listener);
+        }
+    }
+
+    private ParsedQueryString parseQueryString(String queryString) throws ElasticsearchSecurityException {
+
+        final Map<String, String> parameters = new HashMap<>();
+        RestUtils.decodeQueryString(queryString, 0, parameters);
+        if (parameters.isEmpty()) {
+            throw new ElasticsearchSecurityException("Invalid Authentication Request query string (zero parameters)");
+        }
+        logger.trace(new ParameterizedMessage("Parsed the following parameters from the query string: {}", parameters));
+        final String samlRequest = parameters.get("SAMLRequest");
+        if (null == samlRequest) {
+            throw new ElasticsearchSecurityException("Query string [{}] does not contain a SAMLRequest parameter",
+                RestStatus.BAD_REQUEST, queryString);
+        }
+        return new ParsedQueryString(
+            queryString,
+            samlRequest,
+            parameters.get("RelayState"),
+            parameters.get("SigAlg"),
+            parameters.get("Signature"));
+    }
+
+    private void validateAuthnRequest(AuthnRequest authnRequest, SamlServiceProvider sp, ParsedQueryString parsedQueryString,
+                                         ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        // If the Service Provider should not sign requests, do not try to handle signatures even if they are added to the request
+        if (sp.shouldSignAuthnRequests()) {
+            if (Strings.hasText(parsedQueryString.signature)) {
+                if (Strings.hasText(parsedQueryString.sigAlg) == false) {
+                    logAndRespond(new ParameterizedMessage("Query string [{}] contains a Signature but SigAlg parameter is missing",
+                        parsedQueryString.queryString), listener);
+                    return;
+                }
+                final Set<X509Credential> spSigningCredentials = sp.getSpSigningCredentials();
+                if (spSigningCredentials == null || spSigningCredentials.isEmpty()) {
+                    logAndRespond(new ParameterizedMessage("Unable to validate signature of authentication request, " +
+                        "Service Provider [{}] hasn't registered signing credentials", sp.getEntityId()), listener);
+                    return;
+                }
+                if (validateSignature(parsedQueryString, spSigningCredentials) == false) {
+                    logAndRespond(
+                        new ParameterizedMessage("Unable to validate signature of authentication request [{}] using credentials [{}]",
+                            parsedQueryString.queryString, samlFactory.describeCredentials(spSigningCredentials)), listener);
+                    return;
+                }
+            } else if (Strings.hasText(parsedQueryString.sigAlg)) {
+                logAndRespond(new ParameterizedMessage("Query string [{}] contains a SigAlg parameter but Signature is missing",
+                    parsedQueryString.queryString), listener);
+                return;
+            } else {
+                logAndRespond(
+                    new ParameterizedMessage(
+                        "The Service Provider [{}] must sign authentication requests but no signature was found", sp.getEntityId()),
+                    listener);
+                return;
+            }
+        }
+        final Map<String, Object> authnState = new HashMap<>();
+        checkDestination(authnRequest);
+        checkAcs(authnRequest, sp, authnState);
+        validateNameIdPolicy(authnRequest, sp, authnState);
+        authnState.put(SamlAuthenticationState.Fields.ENTITY_ID.getPreferredName(), sp.getEntityId());
+        authnState.put(SamlAuthenticationState.Fields.AUTHN_REQUEST_ID.getPreferredName(), authnRequest.getID());
+        final SamlValidateAuthnRequestResponse response = new SamlValidateAuthnRequestResponse(sp.getEntityId(),
+            authnRequest.isForceAuthn(), authnState);
+        logger.trace(new ParameterizedMessage("Validated AuthnResponse from queryString [{}] and extracted [{}]",
+            parsedQueryString.queryString, response));
+        listener.onResponse(response);
+    }
+
+    private void validateNameIdPolicy(AuthnRequest request, SamlServiceProvider sp, Map<String, Object> authnState) {
+        final NameIDPolicy nameIDPolicy = request.getNameIDPolicy();
+        if (null != nameIDPolicy) {
+            final String requestedFormat = nameIDPolicy.getFormat();
+            final String allowedFormat = sp.getAllowedNameIdFormat();
+            if (Strings.hasText(requestedFormat)) {
+                if (allowedFormat != null && requestedFormat.equals(UNSPECIFIED) == false
+                    && requestedFormat.equals(allowedFormat) == false) {
+                    throw new ElasticsearchSecurityException("The requested NameID format [{}] doesn't match the allowed NameID format" +
+                        " for this Service Provider which is [{}]", requestedFormat, sp.getAllowedNameIdFormat());
+                } else {
+                    authnState.put(SamlAuthenticationState.Fields.NAMEID_FORMAT.getPreferredName(), requestedFormat);
+                }
+            }
+        }
+    }
+
+    private boolean validateSignature(ParsedQueryString queryString, Collection<X509Credential> credentials) {
+        final String javaSigAlgorithm = samlFactory.getJavaAlorithmNameFromUri(queryString.sigAlg);
+        final byte[] contentBytes = queryString.reconstructQueryParameters().getBytes(StandardCharsets.UTF_8);
+        final byte[] signatureBytes = Base64.getDecoder().decode(queryString.signature);
+        return credentials.stream().anyMatch(credential -> {
+            try {
+                Signature sig = Signature.getInstance(javaSigAlgorithm);
+                sig.initVerify(credential.getEntityCertificate().getPublicKey());
+                sig.update(contentBytes);
+                return sig.verify(signatureBytes);
+            } catch (NoSuchAlgorithmException e) {
+                throw new ElasticsearchSecurityException("Java signature algorithm [{}] is not available for SAML/XML-Sig algorithm [{}]",
+                    e, javaSigAlgorithm, queryString.sigAlg);
+            } catch (InvalidKeyException | SignatureException e) {
+                logger.warn(new ParameterizedMessage("Signature verification failed for credential [{}]",
+                    samlFactory.describeCredentials(Set.of(credential))), e);
+                return false;
+            }
+        });
+    }
+
+    private void getSpFromIssuer(Issuer issuer, ActionListener<SamlServiceProvider> listener) {
+        if (issuer == null || issuer.getValue() == null) {
+            throw new ElasticsearchSecurityException("SAML authentication request has no issuer", RestStatus.BAD_REQUEST);
+        }
+        final String issuerString = issuer.getValue();
+        idp.getRegisteredServiceProvider(issuerString, false, ActionListener.wrap(
+            serviceProvider -> {
+                if (null == serviceProvider) {
+                    throw new ElasticsearchSecurityException(
+                        "Service Provider with Entity ID [{}] is not registered with this Identity Provider", RestStatus.BAD_REQUEST,
+                        issuerString);
+                }
+                listener.onResponse(serviceProvider);
+            },
+            listener::onFailure
+        ));
+    }
+
+    private void checkDestination(AuthnRequest request) {
+        final String url = idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI).toString();
+        if (url.equals(request.getDestination()) == false) {
+            throw new ElasticsearchSecurityException(
+                "SAML authentication request [{}] is for destination [{}] but the SSO endpoint of this Identity Provider is [{}]",
+                RestStatus.BAD_REQUEST, request.getID(), request.getDestination(), url);
+        }
+    }
+
+    private void checkAcs(AuthnRequest request, SamlServiceProvider sp, Map<String, Object> authnState) {
+        final String acs = request.getAssertionConsumerServiceURL();
+        if (Strings.hasText(acs) == false) {
+            final String message = request.getAssertionConsumerServiceIndex() == null ?
+                "SAML authentication does not contain an AssertionConsumerService URL" :
+                "SAML authentication does not contain an AssertionConsumerService URL. It contains an Assertion Consumer Service Index " +
+                    "but this IDP doesn't support multiple AssertionConsumerService URLs.";
+            throw new ElasticsearchSecurityException(message, RestStatus.BAD_REQUEST);
+        }
+        if (acs.equals(sp.getAssertionConsumerService().toString()) == false) {
+            throw new ElasticsearchSecurityException("The registered ACS URL for this Service Provider is [{}] but the authentication " +
+                "request contained [{}]", RestStatus.BAD_REQUEST, sp.getAssertionConsumerService(), acs);
+        }
+        authnState.put(SamlAuthenticationState.Fields.ACS_URL.getPreferredName(), acs);
+    }
+
+    protected Element parseSamlMessage(byte[] content) {
+        final Element root;
+        try (ByteArrayInputStream input = new ByteArrayInputStream(content)) {
+            // This will parse and validate the input against the schemas
+            final Document doc = THREAD_LOCAL_DOCUMENT_BUILDER.get().parse(input);
+            root = doc.getDocumentElement();
+            if (logger.isTraceEnabled()) {
+                logger.trace("Received SAML Message: {} \n", samlFactory.toString(root, true));
+            }
+        } catch (SAXException | IOException e) {
+            throw new ElasticsearchSecurityException("Failed to parse SAML message", RestStatus.BAD_REQUEST, e);
+        }
+        return root;
+    }
+
+    private byte[] decodeBase64(String content) {
+        try {
+            return Base64.getDecoder().decode(content.replaceAll("\\s+", ""));
+        } catch (IllegalArgumentException e) {
+            logger.info("Failed to decode base64 string [{}] - {}", content, e);
+            throw new ElasticsearchSecurityException("SAML message cannot be Base64 decoded", RestStatus.BAD_REQUEST, e);
+        }
+    }
+
+    private byte[] inflate(byte[] bytes) {
+        Inflater inflater = new Inflater(true);
+        try (ByteArrayInputStream in = new ByteArrayInputStream(bytes);
+             InflaterInputStream inflate = new InflaterInputStream(in, inflater);
+             ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length * 3 / 2)) {
+            Streams.copy(inflate, out);
+            return out.toByteArray();
+        } catch (IOException e) {
+            throw new ElasticsearchSecurityException("SAML message cannot be inflated", RestStatus.BAD_REQUEST, e);
+        }
+    }
+
+    private String urlEncode(String param) throws UnsupportedEncodingException {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8.name());
+    }
+
+    private void logAndRespond(String message, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        logger.debug(message);
+        listener.onFailure(new ElasticsearchSecurityException(message));
+    }
+
+    private void logAndRespond(ParameterizedMessage message, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        logAndRespond(message.getFormattedMessage(), listener);
+    }
+
+    private void logAndRespond(String message, Throwable e, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        logger.debug(message);
+        listener.onFailure(new ElasticsearchSecurityException(message, e));
+    }
+
+    private class ParsedQueryString {
+        private final String queryString;
+        private final String samlRequest;
+        @Nullable
+        private final String relayState;
+        @Nullable
+        private final String sigAlg;
+        @Nullable
+        private final String signature;
+
+        private ParsedQueryString(String queryString, String samlRequest, String relayState, String sigAlg, String signature) {
+            this.queryString = Objects.requireNonNull(queryString, "Query string may not be null");
+            this.samlRequest = Objects.requireNonNull(samlRequest, "SAML request parameter may not be null");
+            this.relayState = relayState;
+            this.sigAlg = sigAlg;
+            this.signature = signature;
+        }
+
+        public String reconstructQueryParameters() throws ElasticsearchSecurityException {
+            try {
+                return relayState == null ?
+                    "SAMLRequest=" + urlEncode(samlRequest) + "&SigAlg=" + urlEncode(sigAlg) :
+                    "SAMLRequest=" + urlEncode(samlRequest) + "&RelayState=" + urlEncode(relayState) + "&SigAlg=" + urlEncode(sigAlg);
+            } catch (UnsupportedEncodingException e) {
+                throw new ElasticsearchSecurityException("Cannot reconstruct query for signature verification", e);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java
new file mode 100644
index 0000000000000..470834b23b177
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java
@@ -0,0 +1,274 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.idp.authc.AuthenticationMethod;
+import org.elasticsearch.xpack.idp.authc.NetworkControl;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.SamlObjectSigner;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.opensaml.core.xml.schema.XSString;
+import org.opensaml.saml.saml2.core.Assertion;
+import org.opensaml.saml.saml2.core.Attribute;
+import org.opensaml.saml.saml2.core.AttributeStatement;
+import org.opensaml.saml.saml2.core.AttributeValue;
+import org.opensaml.saml.saml2.core.Audience;
+import org.opensaml.saml.saml2.core.AudienceRestriction;
+import org.opensaml.saml.saml2.core.AuthnContext;
+import org.opensaml.saml.saml2.core.AuthnContextClassRef;
+import org.opensaml.saml.saml2.core.AuthnStatement;
+import org.opensaml.saml.saml2.core.Conditions;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameID;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.Status;
+import org.opensaml.saml.saml2.core.StatusCode;
+import org.opensaml.saml.saml2.core.Subject;
+import org.opensaml.saml.saml2.core.SubjectConfirmation;
+import org.opensaml.saml.saml2.core.SubjectConfirmationData;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+/**
+ * Builds SAML 2.0 {@link Response} objects for successful authentication results.
+ */
+public class SuccessfulAuthenticationResponseMessageBuilder {
+
+    private final Logger logger = LogManager.getLogger();
+
+    private final Clock clock;
+    private final SamlIdentityProvider idp;
+    private final SamlFactory samlFactory;
+
+    public SuccessfulAuthenticationResponseMessageBuilder(SamlFactory samlFactory, Clock clock, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.samlFactory = samlFactory;
+        this.clock = clock;
+        this.idp = idp;
+    }
+
+    public Response build(UserServiceAuthentication user, @Nullable SamlAuthenticationState authnState) {
+        logger.debug("Building success response for [{}] from [{}]", user, authnState);
+        final DateTime now = now();
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+
+        final Response response = samlFactory.object(Response.class, Response.DEFAULT_ELEMENT_NAME);
+        response.setID(samlFactory.secureIdentifier());
+        if (authnState != null && authnState.getAuthnRequestId() != null) {
+            response.setInResponseTo(authnState.getAuthnRequestId());
+        }
+        response.setIssuer(buildIssuer());
+        response.setIssueInstant(now);
+        response.setStatus(buildStatus());
+        response.setDestination(serviceProvider.getAssertionConsumerService().toString());
+
+        final Assertion assertion = samlFactory.object(Assertion.class, Assertion.DEFAULT_ELEMENT_NAME);
+        assertion.setID(samlFactory.secureIdentifier());
+        assertion.setIssuer(buildIssuer());
+        assertion.setIssueInstant(now);
+        assertion.setConditions(buildConditions(now, serviceProvider));
+        assertion.setSubject(buildSubject(now, user, authnState));
+        assertion.getAuthnStatements().add(buildAuthnStatement(now, user));
+        final AttributeStatement attributes = buildAttributes(user);
+        if (attributes != null) {
+            assertion.getAttributeStatements().add(attributes);
+        }
+        response.getAssertions().add(assertion);
+        return sign(response);
+    }
+
+    private Response sign(Response response) {
+        final SamlObjectSigner signer = new SamlObjectSigner(samlFactory, idp);
+        return samlFactory.buildXmlObject(signer.sign(response), Response.class);
+    }
+
+    private Conditions buildConditions(DateTime now, SamlServiceProvider serviceProvider) {
+        final Audience spAudience = samlFactory.object(Audience.class, Audience.DEFAULT_ELEMENT_NAME);
+        spAudience.setAudienceURI(serviceProvider.getEntityId());
+
+        final AudienceRestriction restriction = samlFactory.object(AudienceRestriction.class, AudienceRestriction.DEFAULT_ELEMENT_NAME);
+        restriction.getAudiences().add(spAudience);
+
+        final Conditions conditions = samlFactory.object(Conditions.class, Conditions.DEFAULT_ELEMENT_NAME);
+        conditions.setNotBefore(now);
+        conditions.setNotOnOrAfter(now.plus(serviceProvider.getAuthnExpiry()));
+        conditions.getAudienceRestrictions().add(restriction);
+        return conditions;
+    }
+
+    private DateTime now() {
+        return new DateTime(clock.millis(), DateTimeZone.UTC);
+    }
+
+    private Subject buildSubject(DateTime now, UserServiceAuthentication user, SamlAuthenticationState authnState) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+
+        final NameID nameID = buildNameId(user, authnState);
+
+        final Subject subject = samlFactory.object(Subject.class, Subject.DEFAULT_ELEMENT_NAME);
+        subject.setNameID(nameID);
+
+        final SubjectConfirmationData data = samlFactory.object(SubjectConfirmationData.class,
+            SubjectConfirmationData.DEFAULT_ELEMENT_NAME);
+        if (authnState != null && authnState.getAuthnRequestId() != null) {
+            data.setInResponseTo(authnState.getAuthnRequestId());
+        }
+        data.setNotBefore(now);
+        data.setNotOnOrAfter(now.plus(serviceProvider.getAuthnExpiry()));
+        data.setRecipient(serviceProvider.getAssertionConsumerService().toString());
+
+        final SubjectConfirmation confirmation = samlFactory.object(SubjectConfirmation.class, SubjectConfirmation.DEFAULT_ELEMENT_NAME);
+        confirmation.setMethod(SubjectConfirmation.METHOD_BEARER);
+        confirmation.setSubjectConfirmationData(data);
+
+        subject.getSubjectConfirmations().add(confirmation);
+        return subject;
+    }
+
+    private AuthnStatement buildAuthnStatement(DateTime now, UserServiceAuthentication user) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+        final AuthnStatement statement = samlFactory.object(AuthnStatement.class, AuthnStatement.DEFAULT_ELEMENT_NAME);
+        statement.setAuthnInstant(now);
+        statement.setSessionNotOnOrAfter(now.plus(serviceProvider.getAuthnExpiry()));
+
+        final AuthnContext context = samlFactory.object(AuthnContext.class, AuthnContext.DEFAULT_ELEMENT_NAME);
+        final AuthnContextClassRef classRef = samlFactory.object(AuthnContextClassRef.class, AuthnContextClassRef.DEFAULT_ELEMENT_NAME);
+        classRef.setAuthnContextClassRef(resolveAuthnClass(user.getAuthenticationMethods(), user.getNetworkControls()));
+        context.setAuthnContextClassRef(classRef);
+        statement.setAuthnContext(context);
+
+        return statement;
+    }
+
+    private String resolveAuthnClass(Set<AuthenticationMethod> authenticationMethods, Set<NetworkControl> networkControls) {
+        if (authenticationMethods.contains(AuthenticationMethod.PASSWORD)) {
+            if (networkControls.contains(NetworkControl.IP_FILTER)) {
+                return AuthnContext.IP_PASSWORD_AUTHN_CTX;
+            } else if (networkControls.contains(NetworkControl.TLS)) {
+                return AuthnContext.PPT_AUTHN_CTX;
+            } else {
+                return AuthnContext.PASSWORD_AUTHN_CTX;
+            }
+        } else if (authenticationMethods.contains(AuthenticationMethod.KERBEROS)) {
+            return AuthnContext.KERBEROS_AUTHN_CTX;
+        } else if (authenticationMethods.contains(AuthenticationMethod.TLS_CLIENT_AUTH) && networkControls.contains(NetworkControl.TLS)) {
+            return AuthnContext.TLS_CLIENT_AUTHN_CTX;
+        } else if (authenticationMethods.contains(AuthenticationMethod.PRIOR_SESSION)) {
+            return AuthnContext.PREVIOUS_SESSION_AUTHN_CTX;
+        } else if (networkControls.contains(NetworkControl.IP_FILTER)) {
+            return AuthnContext.IP_AUTHN_CTX;
+        } else {
+            return AuthnContext.UNSPECIFIED_AUTHN_CTX;
+        }
+    }
+
+    private AttributeStatement buildAttributes(UserServiceAuthentication user) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+        final AttributeStatement statement = samlFactory.object(AttributeStatement.class, AttributeStatement.DEFAULT_ELEMENT_NAME);
+        final List<Attribute> attributes = new ArrayList<>();
+        final Attribute roles = buildAttribute(serviceProvider.getAttributeNames().roles, "roles", user.getRoles());
+        if (roles != null) {
+            attributes.add(roles);
+        }
+        final Attribute principal = buildAttribute(serviceProvider.getAttributeNames().principal, "principal", user.getPrincipal());
+        if (principal != null) {
+            attributes.add(principal);
+        }
+        final Attribute email = buildAttribute(serviceProvider.getAttributeNames().email, "email", user.getEmail());
+        if (email != null) {
+            attributes.add(email);
+        }
+        final Attribute name = buildAttribute(serviceProvider.getAttributeNames().name, "name", user.getName());
+        if (name != null) {
+            attributes.add(name);
+        }
+        if (attributes.isEmpty()) {
+            return null;
+        }
+        statement.getAttributes().addAll(attributes);
+        return statement;
+    }
+
+    private Attribute buildAttribute(String formalName, String friendlyName, String value) {
+        if (Strings.isNullOrEmpty(value)) {
+            return null;
+        }
+        return buildAttribute(formalName, friendlyName, List.of(value));
+    }
+
+    private Attribute buildAttribute(String formalName, String friendlyName, Collection<String> values) {
+        if (values.isEmpty() || Strings.isNullOrEmpty(formalName)) {
+            return null;
+        }
+        final Attribute attribute = samlFactory.object(Attribute.class, Attribute.DEFAULT_ELEMENT_NAME);
+        attribute.setName(formalName);
+        attribute.setFriendlyName(friendlyName);
+        attribute.setNameFormat(Attribute.URI_REFERENCE);
+        for (String val : values) {
+            final XSString string = samlFactory.object(XSString.class, AttributeValue.DEFAULT_ELEMENT_NAME, XSString.TYPE_NAME);
+            string.setValue(val);
+            attribute.getAttributeValues().add(string);
+        }
+        return attribute;
+    }
+
+    private Issuer buildIssuer() {
+        final Issuer issuer = samlFactory.object(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(this.idp.getEntityId());
+        return issuer;
+    }
+
+    private Status buildStatus() {
+        final StatusCode code = samlFactory.object(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+        code.setValue(StatusCode.SUCCESS);
+
+        final Status status = samlFactory.object(Status.class, Status.DEFAULT_ELEMENT_NAME);
+        status.setStatusCode(code);
+
+        return status;
+    }
+
+    private NameID buildNameId(UserServiceAuthentication user, @Nullable SamlAuthenticationState authnState) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+        final NameID nameID = samlFactory.object(NameID.class, NameID.DEFAULT_ELEMENT_NAME);
+        final String nameIdFormat;
+        if (authnState != null && authnState.getRequestedNameidFormat() != null) {
+            nameIdFormat = authnState.getRequestedNameidFormat();
+        } else {
+            nameIdFormat = serviceProvider.getAllowedNameIdFormat() != null ? serviceProvider.getAllowedNameIdFormat() :
+                idp.getServiceProviderDefaults().nameIdFormat;
+        }
+        nameID.setFormat(nameIdFormat);
+        nameID.setValue(getNameIdValueForFormat(nameIdFormat, user));
+        return nameID;
+    }
+
+    private String getNameIdValueForFormat(String format, UserServiceAuthentication user) {
+        switch (format) {
+            case TRANSIENT:
+                // See SAML 2.0 Core 8.3.8 & 1.3.4
+                return samlFactory.secureIdentifier();
+            default:
+                throw new IllegalStateException("Unsupported NameID Format: " + format);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java
new file mode 100644
index 0000000000000..4223b26810d55
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.xpack.idp.authc.AuthenticationMethod;
+import org.elasticsearch.xpack.idp.authc.NetworkControl;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+
+import java.util.Set;
+
+/**
+ * Lightweight representation of a user that has authenticated to the IdP in the context of a specific service provider
+ */
+public class UserServiceAuthentication {
+
+    private final String principal;
+    private final String name;
+    private final String email;
+    private final Set<String> roles;
+
+    private final SamlServiceProvider serviceProvider;
+    private final Set<AuthenticationMethod> authenticationMethods;
+    private final Set<NetworkControl> networkControls;
+
+    public UserServiceAuthentication(String principal, String name, String email, Set<String> roles,
+                                     SamlServiceProvider serviceProvider,
+                                     Set<AuthenticationMethod> authenticationMethods, Set<NetworkControl> networkControls) {
+        this.principal = principal;
+        this.name = name;
+        this.email = email;
+        this.roles = Set.copyOf(roles);
+        this.serviceProvider = serviceProvider;
+        this.authenticationMethods = authenticationMethods;
+        this.networkControls = networkControls;
+    }
+
+    public UserServiceAuthentication(String principal, String name, String email, Set<String> roles, SamlServiceProvider serviceProvider) {
+        this(principal, name, email, roles, serviceProvider, Set.of(AuthenticationMethod.PASSWORD), Set.of(NetworkControl.TLS));
+    }
+
+    public String getPrincipal() {
+        return principal;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getEmail() {
+        return email;
+    }
+
+    public Set<String> getRoles() {
+        return roles;
+    }
+
+    public SamlServiceProvider getServiceProvider() {
+        return serviceProvider;
+    }
+
+    public Set<AuthenticationMethod> getAuthenticationMethods() {
+        return authenticationMethods;
+    }
+
+    public Set<NetworkControl> getNetworkControls() {
+        return networkControls;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" +
+            "principal='" + principal + '\'' +
+            ", name='" + name + '\'' +
+            ", email='" + email + '\'' +
+            ", roles=" + roles +
+            ", serviceProvider=" + serviceProvider +
+            ", authenticationMethods=" + authenticationMethods +
+            ", networkControls=" + networkControls +
+            '}';
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java
new file mode 100644
index 0000000000000..ac60f6f906e05
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java
@@ -0,0 +1,264 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.common.Strings;
+import org.opensaml.saml.common.xml.SAMLConstants;
+import org.opensaml.saml.saml2.metadata.ContactPerson;
+import org.opensaml.saml.saml2.metadata.EmailAddress;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.saml2.metadata.GivenName;
+import org.opensaml.saml.saml2.metadata.IDPSSODescriptor;
+import org.opensaml.saml.saml2.metadata.KeyDescriptor;
+import org.opensaml.saml.saml2.metadata.NameIDFormat;
+import org.opensaml.saml.saml2.metadata.Organization;
+import org.opensaml.saml.saml2.metadata.OrganizationDisplayName;
+import org.opensaml.saml.saml2.metadata.OrganizationName;
+import org.opensaml.saml.saml2.metadata.OrganizationURL;
+import org.opensaml.saml.saml2.metadata.SingleLogoutService;
+import org.opensaml.saml.saml2.metadata.SingleSignOnService;
+import org.opensaml.saml.saml2.metadata.SurName;
+import org.opensaml.saml.saml2.metadata.impl.ContactPersonBuilder;
+import org.opensaml.saml.saml2.metadata.impl.EmailAddressBuilder;
+import org.opensaml.saml.saml2.metadata.impl.EntityDescriptorBuilder;
+import org.opensaml.saml.saml2.metadata.impl.GivenNameBuilder;
+import org.opensaml.saml.saml2.metadata.impl.IDPSSODescriptorBuilder;
+import org.opensaml.saml.saml2.metadata.impl.KeyDescriptorBuilder;
+import org.opensaml.saml.saml2.metadata.impl.NameIDFormatBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationDisplayNameBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationNameBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationURLBuilder;
+import org.opensaml.saml.saml2.metadata.impl.SingleLogoutServiceBuilder;
+import org.opensaml.saml.saml2.metadata.impl.SingleSignOnServiceBuilder;
+import org.opensaml.saml.saml2.metadata.impl.SurNameBuilder;
+import org.opensaml.security.credential.UsageType;
+import org.opensaml.xmlsec.keyinfo.KeyInfoSupport;
+import org.opensaml.xmlsec.signature.KeyInfo;
+import org.opensaml.xmlsec.signature.impl.KeyInfoBuilder;
+
+import java.net.URL;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+public class SamlIdPMetadataBuilder {
+
+    private Locale locale;
+    private final String entityId;
+    private Set<String> nameIdFormats;
+    private boolean wantAuthnRequestsSigned;
+    private Map<String, URL> singleSignOnServiceUrls = new HashMap<>();
+    private Map<String, URL> singleLogoutServiceUrls = new HashMap<>();
+    private List<X509Certificate> signingCertificates;
+    private SamlIdentityProvider.OrganizationInfo organization;
+    private final List<SamlIdentityProvider.ContactInfo> contacts;
+
+
+    public SamlIdPMetadataBuilder(String entityId) {
+        this.entityId = entityId;
+        this.locale = Locale.getDefault();
+        this.contacts = new ArrayList<>();
+        this.nameIdFormats = new HashSet<>();
+        this.signingCertificates = new ArrayList<>();
+        this.wantAuthnRequestsSigned = false;
+    }
+
+    public SamlIdPMetadataBuilder withLocale(Locale locale) {
+        this.locale = locale;
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withNameIdFormat(String nameIdFormat) {
+        if (Strings.isNullOrEmpty(nameIdFormat) == false) {
+            this.nameIdFormats.add(nameIdFormat);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder wantAuthnRequestsSigned(boolean wants) {
+        this.wantAuthnRequestsSigned = wants;
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSingleSignOnServiceUrl(String binding, URL url) {
+        if ( null != url) {
+            this.singleSignOnServiceUrls.put(binding, url);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSingleLogoutServiceUrl(String binding, URL url) {
+        if (null != url) {
+            this.singleLogoutServiceUrls.put(binding, url);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSigningCertificates(List<X509Certificate> signingCertificates) {
+        if (null != signingCertificates) {
+            this.signingCertificates.addAll(signingCertificates);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSigningCertificate(X509Certificate signingCertificate) {
+        if ( null != signingCertificate ) {
+            return withSigningCertificates(Collections.singletonList(signingCertificate));
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder organization(SamlIdentityProvider.OrganizationInfo organization) {
+        if (null != organization) {
+            this.organization = organization;
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder organization(String orgName, String displayName, String url) {
+        return organization(new SamlIdentityProvider.OrganizationInfo(orgName, displayName, url));
+    }
+
+    public SamlIdPMetadataBuilder withContact(SamlIdentityProvider.ContactInfo contact) {
+        if (null != contact) {
+            this.contacts.add(contact);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withContact(String type, String givenName, String surName, String email) {
+        return withContact(new SamlIdentityProvider.ContactInfo(SamlIdentityProvider.ContactInfo.getType(type), givenName, surName, email));
+    }
+
+    public EntityDescriptor build() throws CertificateEncodingException {
+        final IDPSSODescriptor idpSsoDescriptor = new IDPSSODescriptorBuilder().buildObject();
+        idpSsoDescriptor.removeAllSupportedProtocols();
+        idpSsoDescriptor.addSupportedProtocol(SAMLConstants.SAML20P_NS);
+        idpSsoDescriptor.setWantAuthnRequestsSigned(this.wantAuthnRequestsSigned);
+        if (nameIdFormats.isEmpty() == false) {
+            idpSsoDescriptor.getNameIDFormats().addAll(buildNameIDFormats());
+        }
+        if (singleSignOnServiceUrls.isEmpty() == false) {
+            idpSsoDescriptor.getSingleSignOnServices().addAll(buildSingleSignOnServices());
+        }
+        if (singleLogoutServiceUrls.isEmpty() == false) {
+            idpSsoDescriptor.getSingleLogoutServices().addAll(buildSingleLogoutServices());
+        }
+        idpSsoDescriptor.getKeyDescriptors().addAll(buildKeyDescriptors());
+
+        final EntityDescriptor descriptor = new EntityDescriptorBuilder().buildObject();
+        descriptor.setEntityID(this.entityId);
+        descriptor.getRoleDescriptors().add(idpSsoDescriptor);
+        if (organization != null) {
+            descriptor.setOrganization(buildOrganization());
+        }
+        for (SamlIdentityProvider.ContactInfo contact : contacts) {
+            descriptor.getContactPersons().add(buildContact(contact));
+        }
+
+        return descriptor;
+    }
+
+    private List<SingleSignOnService> buildSingleSignOnServices() {
+        List<SingleSignOnService> ssoServices = new ArrayList<>();
+        if (singleSignOnServiceUrls.isEmpty()) {
+            throw new IllegalStateException("At least one SingleSignOnService URL should be specified");
+        }
+        for (Map.Entry<String, URL> entry : singleSignOnServiceUrls.entrySet()) {
+            final SingleSignOnService sso = new SingleSignOnServiceBuilder().buildObject();
+            sso.setBinding(entry.getKey());
+            sso.setLocation(entry.getValue().toString());
+            ssoServices.add(sso);
+        }
+        return ssoServices;
+    }
+
+    private List<SingleLogoutService> buildSingleLogoutServices() {
+        List<SingleLogoutService> sloServices = new ArrayList<>();
+        for (Map.Entry<String, URL> entry : singleLogoutServiceUrls.entrySet()) {
+            final SingleLogoutService slo = new SingleLogoutServiceBuilder().buildObject();
+            slo.setBinding(entry.getKey());
+            slo.setLocation(entry.getValue().toString());
+            sloServices.add(slo);
+        }
+        return sloServices;
+    }
+
+    private List<NameIDFormat> buildNameIDFormats() {
+        List<NameIDFormat> formats = new ArrayList();
+        if (nameIdFormats.isEmpty()) {
+            throw new IllegalStateException("NameID format has not been specified");
+        }
+        for (String nameIdFormat : nameIdFormats) {
+            final NameIDFormat format = new NameIDFormatBuilder().buildObject();
+            format.setFormat(nameIdFormat);
+            formats.add(format);
+
+        }
+        return formats;
+    }
+
+    private List<? extends KeyDescriptor> buildKeyDescriptors() throws CertificateEncodingException {
+        if (signingCertificates.isEmpty()) {
+            return Collections.emptyList();
+        }
+        List<KeyDescriptor> keys = new ArrayList<>();
+        for (X509Certificate signingCertificate : signingCertificates) {
+            if (signingCertificate != null) {
+                final KeyDescriptor descriptor = new KeyDescriptorBuilder().buildObject();
+                descriptor.setUse(UsageType.SIGNING);
+                final KeyInfo keyInfo = new KeyInfoBuilder().buildObject();
+                KeyInfoSupport.addCertificate(keyInfo, signingCertificate);
+                descriptor.setKeyInfo(keyInfo);
+                keys.add(descriptor);
+            }
+        }
+        return keys;
+    }
+
+    private Organization buildOrganization() {
+        final String lang = locale.toLanguageTag();
+        final OrganizationName name = new OrganizationNameBuilder().buildObject();
+        name.setValue(this.organization.organizationName);
+        name.setXMLLang(lang);
+        final OrganizationDisplayName displayName = new OrganizationDisplayNameBuilder().buildObject();
+        displayName.setValue(this.organization.displayName);
+        displayName.setXMLLang(lang);
+        final OrganizationURL url = new OrganizationURLBuilder().buildObject();
+        url.setValue(this.organization.url);
+        url.setXMLLang(lang);
+
+        final Organization org = new OrganizationBuilder().buildObject();
+        org.getOrganizationNames().add(name);
+        org.getDisplayNames().add(displayName);
+        org.getURLs().add(url);
+        return org;
+    }
+
+    private ContactPerson buildContact(SamlIdentityProvider.ContactInfo contact) {
+        final GivenName givenName = new GivenNameBuilder().buildObject();
+        givenName.setName(contact.givenName);
+        final SurName surName = new SurNameBuilder().buildObject();
+        surName.setName(contact.surName);
+        final EmailAddress email = new EmailAddressBuilder().buildObject();
+        email.setAddress(contact.email);
+
+        final ContactPerson person = new ContactPersonBuilder().buildObject();
+        person.setType(contact.type);
+        person.setGivenName(givenName);
+        person.setSurName(surName);
+        person.getEmailAddresses().add(email);
+        return person;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java
new file mode 100644
index 0000000000000..037894cd3b685
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.idp;
+
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.opensaml.saml.saml2.metadata.ContactPersonTypeEnumeration;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * SAML 2.0 configuration information about this IdP
+ */
+public class SamlIdentityProvider {
+
+    private final Logger logger = LogManager.getLogger();
+
+    private final String entityId;
+    private final Map<String, URL> ssoEndpoints;
+    private final Map<String, URL> sloEndpoints;
+    private final Set<String> allowedNameIdFormats;
+    private final ServiceProviderDefaults serviceProviderDefaults;
+    private final X509Credential signingCredential;
+    private final SamlServiceProviderResolver serviceProviderResolver;
+    private final X509Credential metadataSigningCredential;
+    private ContactInfo technicalContact;
+    private OrganizationInfo organization;
+
+    // Package access - use Builder instead
+    SamlIdentityProvider(String entityId, Map<String, URL> ssoEndpoints, Map<String, URL> sloEndpoints, Set<String> allowedNameIdFormats,
+                         X509Credential signingCredential, X509Credential metadataSigningCredential,
+                         ContactInfo technicalContact, OrganizationInfo organization,
+                         ServiceProviderDefaults serviceProviderDefaults, SamlServiceProviderResolver serviceProviderResolver) {
+        this.entityId = entityId;
+        this.ssoEndpoints = ssoEndpoints;
+        this.sloEndpoints = sloEndpoints;
+        this.allowedNameIdFormats = allowedNameIdFormats;
+        this.signingCredential = signingCredential;
+        this.serviceProviderDefaults = serviceProviderDefaults;
+        this.metadataSigningCredential = metadataSigningCredential;
+        this.technicalContact = technicalContact;
+        this.organization = organization;
+        this.serviceProviderResolver = serviceProviderResolver;
+    }
+
+    public static SamlIdentityProviderBuilder builder(SamlServiceProviderResolver resolver) {
+        return new SamlIdentityProviderBuilder(resolver);
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public URL getSingleSignOnEndpoint(String binding) {
+        return ssoEndpoints.get(binding);
+    }
+
+    public URL getSingleLogoutEndpoint(String binding) {
+        return sloEndpoints.get(binding);
+    }
+
+    public Set<String> getAllowedNameIdFormats() {
+        return allowedNameIdFormats;
+    }
+
+    public X509Credential getSigningCredential() {
+        return signingCredential;
+    }
+
+    public X509Credential getMetadataSigningCredential() {
+        return metadataSigningCredential;
+    }
+
+    public OrganizationInfo getOrganization() {
+        return organization;
+    }
+
+    public ContactInfo getTechnicalContact() {
+        return technicalContact;
+    }
+
+    public ServiceProviderDefaults getServiceProviderDefaults() {
+        return serviceProviderDefaults;
+    }
+
+    /**
+     * Asynchronously lookup the specified {@link SamlServiceProvider} by entity-id.
+     * @param allowDisabled whether to return service providers that are not {@link SamlServiceProvider#isEnabled() enabled}.
+     *                      For security reasons, callers should typically avoid working with disabled service providers.
+     * @param listener Responds with the requested Service Provider object, or {@code null} if no such SP exists.
+     *                 {@link ActionListener#onFailure} is only used for fatal errors (e.g. being unable to access
+     *                 the backing store (elasticsearch index) that hold the SP data).
+     */
+    public void getRegisteredServiceProvider(String spEntityId, boolean allowDisabled, ActionListener<SamlServiceProvider> listener) {
+        serviceProviderResolver.resolve(spEntityId, ActionListener.wrap(
+            sp -> {
+                if (sp == null) {
+                    logger.info("No service provider exists for entityId [{}]", spEntityId);
+                    listener.onResponse(null);
+                } else if (allowDisabled == false && sp.isEnabled() == false) {
+                    logger.info("Service provider [{}][{}] is not enabled", sp.getEntityId(), sp.getName());
+                    listener.onResponse(null);
+                } else {
+                    logger.debug("Service provider for [{}] is [{}]", sp.getEntityId(), sp);
+                    listener.onResponse(sp);
+                }
+            },
+            listener::onFailure
+        ));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final SamlIdentityProvider that = (SamlIdentityProvider) o;
+        return Objects.equals(entityId, that.entityId) &&
+            Objects.equals(ssoEndpoints, that.ssoEndpoints) &&
+            Objects.equals(sloEndpoints, that.sloEndpoints) &&
+            Objects.equals(allowedNameIdFormats, that.allowedNameIdFormats) &&
+            Objects.equals(signingCredential, that.signingCredential) &&
+            Objects.equals(metadataSigningCredential, that.metadataSigningCredential) &&
+            Objects.equals(technicalContact, that.technicalContact) &&
+            Objects.equals(organization, that.organization);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(entityId);
+    }
+
+    public static class ContactInfo {
+        static final Map<String, ContactPersonTypeEnumeration> TYPES = Collections.unmodifiableMap(
+            MapBuilder.newMapBuilder(new LinkedHashMap<String, ContactPersonTypeEnumeration>())
+                .put(ContactPersonTypeEnumeration.ADMINISTRATIVE.toString(), ContactPersonTypeEnumeration.ADMINISTRATIVE)
+                .put(ContactPersonTypeEnumeration.BILLING.toString(), ContactPersonTypeEnumeration.BILLING)
+                .put(ContactPersonTypeEnumeration.SUPPORT.toString(), ContactPersonTypeEnumeration.SUPPORT)
+                .put(ContactPersonTypeEnumeration.TECHNICAL.toString(), ContactPersonTypeEnumeration.TECHNICAL)
+                .put(ContactPersonTypeEnumeration.OTHER.toString(), ContactPersonTypeEnumeration.OTHER)
+                .map());
+
+        public final ContactPersonTypeEnumeration type;
+        public final String givenName;
+        public final String surName;
+        public final String email;
+
+        public ContactInfo(ContactPersonTypeEnumeration type, String givenName, String surName, String email) {
+            this.type = Objects.requireNonNull(type, "Contact Person Type is required");
+            this.givenName = givenName;
+            this.surName = surName;
+            this.email = Objects.requireNonNull(email, "Contact Person email is required");
+        }
+
+        public static ContactPersonTypeEnumeration getType(String name) {
+            final ContactPersonTypeEnumeration type = TYPES.get(name.toLowerCase(Locale.ROOT));
+            if (type == null) {
+                throw new IllegalArgumentException("Invalid contact type " + name + " allowed values are "
+                    + Strings.collectionToCommaDelimitedString(TYPES.keySet()));
+            }
+            return type;
+        }
+    }
+
+    public static class OrganizationInfo {
+        public final String organizationName;
+        public final String displayName;
+        public final String url;
+
+        public OrganizationInfo(String organizationName, String displayName, String url) {
+            this.organizationName = organizationName;
+            this.displayName = displayName;
+            this.url = url;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            OrganizationInfo that = (OrganizationInfo) o;
+            return Objects.equals(organizationName, that.organizationName) &&
+                Objects.equals(displayName, that.displayName) &&
+                Objects.equals(url, that.url);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(organizationName, displayName, url);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java
new file mode 100644
index 0000000000000..6ddbf533b9904
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java
@@ -0,0 +1,356 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.X509KeyPairSettings;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.opensaml.saml.saml2.metadata.ContactPersonTypeEnumeration;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.security.x509.impl.X509KeyManagerX509CredentialAdapter;
+
+import javax.net.ssl.X509KeyManager;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.PrivateKey;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+/**
+ * Builds a {@link SamlIdentityProvider} instance, from either direct properties, or defined node {@link Settings}.
+ */
+public class SamlIdentityProviderBuilder {
+
+    private static final List<String> ALLOWED_NAMEID_FORMATS = List.of(TRANSIENT);
+    public static final Setting<String> IDP_ENTITY_ID = Setting.simpleString("xpack.idp.entity_id", Setting.Property.NodeScope);
+
+    public static final Setting<URL> IDP_SSO_REDIRECT_ENDPOINT = new Setting<>("xpack.idp.sso_endpoint.redirect", "https:",
+        value -> parseUrl("xpack.idp.sso_endpoint.redirect", value), Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_SSO_POST_ENDPOINT = new Setting<>("xpack.idp.sso_endpoint.post", "https:",
+        value -> parseUrl("xpack.idp.sso_endpoint.post", value), Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_SLO_REDIRECT_ENDPOINT = new Setting<>("xpack.idp.slo_endpoint.redirect", "https:",
+        value -> parseUrl("xpack.idp.slo_endpoint.redirect", value), Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_SLO_POST_ENDPOINT = new Setting<>("xpack.idp.slo_endpoint.post", "https:",
+        value -> parseUrl("xpack.idp.slo_endpoint.post", value), Setting.Property.NodeScope);
+    public static final Setting<List<String>> IDP_ALLOWED_NAMEID_FORMATS = Setting.listSetting("xpack.idp.allowed_nameid_formats",
+        List.of(TRANSIENT), Function.identity(), SamlIdentityProviderBuilder::validateNameIDs, Setting.Property.NodeScope);
+
+    public static final Setting<String> IDP_SIGNING_KEY_ALIAS = Setting.simpleString("xpack.idp.signing.keystore.alias",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_METADATA_SIGNING_KEY_ALIAS = Setting.simpleString("xpack.idp.metadata.signing.keystore.alias",
+        Setting.Property.NodeScope);
+
+    public static final Setting<String> IDP_ORGANIZATION_NAME = Setting.simpleString("xpack.idp.organization.name",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_ORGANIZATION_DISPLAY_NAME = Setting.simpleString("xpack.idp.organization.display_name",
+        IDP_ORGANIZATION_NAME, Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_ORGANIZATION_URL = new Setting<>("xpack.idp.organization.url", "http:",
+        value -> parseUrl("xpack.idp.organization.url", value), Setting.Property.NodeScope);
+
+    public static final Setting<String> IDP_CONTACT_GIVEN_NAME = Setting.simpleString("xpack.idp.contact.given_name",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_CONTACT_SURNAME = Setting.simpleString("xpack.idp.contact.surname",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_CONTACT_EMAIL = Setting.simpleString("xpack.idp.contact.email", Setting.Property.NodeScope);
+
+    private final SamlServiceProviderResolver serviceProviderResolver;
+
+    private String entityId;
+    private Map<String, URL> ssoEndpoints;
+    private Map<String, URL> sloEndpoints;
+    private Set<String> allowedNameIdFormats;
+    private X509Credential signingCredential;
+    private X509Credential metadataSigningCredential;
+    private SamlIdentityProvider.ContactInfo technicalContact;
+    private SamlIdentityProvider.OrganizationInfo organization;
+    private ServiceProviderDefaults serviceProviderDefaults;
+
+    SamlIdentityProviderBuilder(SamlServiceProviderResolver serviceProviderResolver) {
+        this.serviceProviderResolver = serviceProviderResolver;
+        this.ssoEndpoints = new HashMap<>();
+        this.sloEndpoints = new HashMap<>();
+    }
+
+    public SamlIdentityProvider build() throws ValidationException {
+        ValidationException ex = new ValidationException();
+
+        if (Strings.isNullOrEmpty(entityId)) {
+            ex.addValidationError("IDP Entity ID must be set (was [" + entityId + "])");
+        }
+
+        if (ssoEndpoints == null || ssoEndpoints.containsKey(SAML2_REDIRECT_BINDING_URI) == false) {
+            ex.addValidationError("The redirect ([ " + SAML2_REDIRECT_BINDING_URI + "]) SSO binding is required");
+        }
+
+        if (signingCredential == null) {
+            ex.addValidationError("Signing credential must be specified");
+        } else {
+            try {
+                validateSigningKey(signingCredential.getPrivateKey());
+            } catch (ElasticsearchSecurityException e) {
+                ex.addValidationError("Signing credential is invalid - " + e.getMessage());
+            }
+        }
+
+        if (metadataSigningCredential != null) {
+            try {
+                validateSigningKey(metadataSigningCredential.getPrivateKey());
+            } catch (ElasticsearchSecurityException e) {
+                ex.addValidationError("Metadata signing credential is invalid - " + e.getMessage());
+            }
+        }
+
+        if (serviceProviderDefaults == null) {
+            ex.addValidationError("Service provider defaults must be specified");
+        }
+
+        if (ex.validationErrors().isEmpty() == false) {
+            throw ex;
+        }
+
+        return new SamlIdentityProvider(
+            entityId,
+            Map.copyOf(ssoEndpoints),
+            sloEndpoints == null ? Map.of() : Map.copyOf(sloEndpoints),
+            Set.copyOf(allowedNameIdFormats),
+            signingCredential, metadataSigningCredential,
+            technicalContact, organization,
+            serviceProviderDefaults,
+            serviceProviderResolver);
+    }
+
+    public SamlIdentityProviderBuilder fromSettings(Environment env) {
+        final Settings settings = env.settings();
+        this.entityId = require(settings, IDP_ENTITY_ID);
+        this.ssoEndpoints = new HashMap<>();
+        this.sloEndpoints = new HashMap<>();
+        this.ssoEndpoints.put(SAML2_REDIRECT_BINDING_URI, requiredUrl(settings, IDP_SSO_REDIRECT_ENDPOINT));
+        if (IDP_SSO_POST_ENDPOINT.exists(settings)) {
+            this.ssoEndpoints.put(SAML2_POST_BINDING_URI, IDP_SSO_POST_ENDPOINT.get(settings));
+        }
+        if (IDP_SLO_POST_ENDPOINT.exists(settings)) {
+            this.sloEndpoints.put(SAML2_POST_BINDING_URI, IDP_SLO_POST_ENDPOINT.get(settings));
+        }
+        if (IDP_SLO_REDIRECT_ENDPOINT.exists(settings)) {
+            this.sloEndpoints.put(SAML2_REDIRECT_BINDING_URI, IDP_SLO_REDIRECT_ENDPOINT.get(settings));
+        }
+        this.allowedNameIdFormats = new HashSet<>(IDP_ALLOWED_NAMEID_FORMATS.get(settings));
+        this.signingCredential = buildSigningCredential(env, settings, "xpack.idp.signing.");
+        this.metadataSigningCredential = buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        this.technicalContact = buildContactInfo(settings);
+        this.organization = buildOrganization(settings);
+        return this;
+    }
+
+    public static List<? extends Setting<?>> getSettings() {
+        return List.of(
+            IDP_ENTITY_ID,
+            IDP_SLO_REDIRECT_ENDPOINT,
+            IDP_SLO_POST_ENDPOINT,
+            IDP_SSO_REDIRECT_ENDPOINT,
+            IDP_SSO_POST_ENDPOINT,
+            IDP_ALLOWED_NAMEID_FORMATS,
+            IDP_SIGNING_KEY_ALIAS,
+            IDP_METADATA_SIGNING_KEY_ALIAS,
+            IDP_ORGANIZATION_NAME,
+            IDP_ORGANIZATION_DISPLAY_NAME,
+            IDP_ORGANIZATION_URL,
+            IDP_CONTACT_GIVEN_NAME,
+            IDP_CONTACT_SURNAME,
+            IDP_CONTACT_EMAIL);
+    }
+
+    public SamlIdentityProviderBuilder serviceProviderDefaults(ServiceProviderDefaults serviceProviderDefaults) {
+        this.serviceProviderDefaults = serviceProviderDefaults;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder entityId(String entityId) {
+        this.entityId = entityId;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleSignOnEndpoints(Map<String, URL> ssoEndpoints) {
+        this.ssoEndpoints = ssoEndpoints;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleLogoutEndpoints(Map<String, URL> sloEndpoints) {
+        this.sloEndpoints = sloEndpoints;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleSignOnEndpoint(String binding, URL endpoint) {
+        this.ssoEndpoints.put(binding, endpoint);
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleLogoutEndpoint(String binding, URL endpoint) {
+        this.sloEndpoints.put(binding, endpoint);
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder allowedNameIdFormat(String nameIdFormat) {
+        this.allowedNameIdFormats.add(nameIdFormat);
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder signingCredential(X509Credential signingCredential) {
+        this.signingCredential = signingCredential;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder metadataSigningCredential(X509Credential metadataSigningCredential) {
+        this.metadataSigningCredential = metadataSigningCredential;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder technicalContact(SamlIdentityProvider.ContactInfo technicalContact) {
+        this.technicalContact = technicalContact;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder organization(SamlIdentityProvider.OrganizationInfo organization) {
+        this.organization = organization;
+        return this;
+    }
+
+    private static URL parseUrl(String key, String value) {
+        try {
+            return new URL(value);
+        } catch (MalformedURLException e) {
+            throw new IllegalArgumentException("Invalid value [" + value + "] for [" + key + "]. Not a valid URL", e);
+        }
+    }
+
+    private static void validateNameIDs(List<String> values) {
+        final Set<String> invalidFormats =
+            values.stream().distinct().filter(e -> ALLOWED_NAMEID_FORMATS.contains(e) == false).collect(Collectors.toSet());
+        if (invalidFormats.size() > 0) {
+            throw new IllegalArgumentException(
+                invalidFormats + " are not valid NameID formats. Allowed values are " + ALLOWED_NAMEID_FORMATS);
+        }
+    }
+
+    static String require(Settings settings, Setting<String> setting) {
+        if (settings.hasValue(setting.getKey())) {
+            return setting.get(settings);
+        } else {
+            throw new IllegalArgumentException("The configuration setting [" + setting.getKey() + "] is required");
+        }
+    }
+
+    static URL requiredUrl(Settings settings, Setting<URL> setting) {
+        if (settings.hasValue(setting.getKey())) {
+            return setting.get(settings);
+        } else {
+            throw new IllegalArgumentException("The configuration setting [" + setting.getKey() + "] is required");
+        }
+    }
+
+    // Package protected for testing
+    static X509Credential buildSigningCredential(Environment environment, Settings settings, String prefix) {
+        List<X509Credential> credentials = buildCredentials(environment, settings, prefix, false);
+        if (credentials.isEmpty()) {
+            return null;
+        }
+        return credentials.get(0);
+    }
+
+    static List<X509Credential> buildCredentials(Environment env, Settings settings, String prefix, boolean allowMultiple) {
+        final X509KeyPairSettings keyPairSettings = X509KeyPairSettings.withPrefix(prefix, false);
+        final X509KeyManager keyManager = CertParsingUtils.getKeyManager(keyPairSettings, settings, null, env);
+        if (keyManager == null) {
+            return Collections.emptyList();
+        }
+
+        final List<X509Credential> credentials = new ArrayList<>();
+        final Set<String> selectedAliases = new HashSet<>();
+        final String configAlias = settings.get(prefix + "keystore.alias");
+        if (Strings.isNullOrEmpty(configAlias)) {
+            final String[] rsaAliases = keyManager.getServerAliases("RSA", null);
+            if (null != rsaAliases) {
+                selectedAliases.addAll(Arrays.asList(rsaAliases));
+            }
+            final String[] ecAliases = keyManager.getServerAliases("EC", null);
+            if (null != ecAliases) {
+                selectedAliases.addAll(Arrays.asList(ecAliases));
+            }
+            if (selectedAliases.isEmpty()) {
+                throw new IllegalArgumentException(
+                    "The configured keystore for [" + prefix + "keystore] does not contain any RSA or EC key pairs.");
+            }
+            if (selectedAliases.size() > 1 && allowMultiple == false) {
+                throw new IllegalArgumentException(
+                    "The configured keystore for [" + prefix + "keystore] contains multiple private key entries, when one was expected.");
+            }
+        } else {
+            selectedAliases.add(configAlias);
+        }
+        for (String alias : selectedAliases) {
+            try {
+                validateSigningKey(keyManager.getPrivateKey(alias));
+            } catch (ElasticsearchSecurityException e) {
+                throw new IllegalArgumentException("The configured credential [" + prefix + "keystore] with alias [" + alias
+                    + "] is not a valid signing key - " + e.getMessage());
+            }
+            credentials.add(new X509KeyManagerX509CredentialAdapter(keyManager, alias));
+        }
+        return credentials;
+    }
+
+    private static void validateSigningKey(PrivateKey privateKey) {
+        if (privateKey == null) {
+            throw new ElasticsearchSecurityException("There is no private key available for this credential");
+        }
+        final String keyType = privateKey.getAlgorithm();
+        if (keyType.equals("RSA") == false && keyType.equals("EC") == false) {
+            throw new ElasticsearchSecurityException("The private key uses unsupported key algorithm type [" + keyType
+                + "], only RSA and EC are supported");
+        }
+    }
+
+    private static SamlIdentityProvider.OrganizationInfo buildOrganization(Settings settings) {
+        final String name = settings.hasValue(IDP_ORGANIZATION_NAME.getKey()) ? IDP_ORGANIZATION_NAME.get(settings) : null;
+        final String displayName = settings.hasValue(IDP_ORGANIZATION_DISPLAY_NAME.getKey()) ?
+            IDP_ORGANIZATION_DISPLAY_NAME.get(settings) : null;
+        final String url = settings.hasValue(IDP_ORGANIZATION_URL.getKey()) ? IDP_ORGANIZATION_URL.get(settings).toString() : null;
+        if (Stream.of(name, displayName, url).allMatch(Objects::isNull) == false) {
+            return new SamlIdentityProvider.OrganizationInfo(name, displayName, url);
+        }
+        return null;
+    }
+
+    private static SamlIdentityProvider.ContactInfo buildContactInfo(Settings settings) {
+        if (settings.hasValue(IDP_CONTACT_EMAIL.getKey())) {
+            return new SamlIdentityProvider.ContactInfo(ContactPersonTypeEnumeration.TECHNICAL,
+                IDP_CONTACT_GIVEN_NAME.get(settings), IDP_CONTACT_SURNAME.get(settings), IDP_CONTACT_EMAIL.get(settings));
+        }
+        return null;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java
new file mode 100644
index 0000000000000..a6425e341e366
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.idp.action.SamlMetadataResponse;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.opensaml.core.xml.io.MarshallingException;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.saml2.metadata.impl.EntityDescriptorMarshaller;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.signature.Signature;
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.xmlsec.signature.support.Signer;
+import org.w3c.dom.Element;
+
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+import static org.opensaml.xmlsec.signature.Signature.DEFAULT_ELEMENT_NAME;
+import static org.opensaml.xmlsec.signature.support.SignatureConstants.ALGO_ID_C14N_EXCL_OMIT_COMMENTS;
+import static org.opensaml.xmlsec.signature.support.SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256;
+
+public class SamlMetadataGenerator {
+
+    private final SamlFactory samlFactory;
+    private final SamlIdentityProvider idp;
+    private final Logger logger = LogManager.getLogger(SamlMetadataGenerator.class);
+
+    public SamlMetadataGenerator(SamlFactory samlFactory, SamlIdentityProvider idp) {
+        this.samlFactory = samlFactory;
+        this.idp = idp;
+        SamlInit.initialize();
+    }
+
+    public void generateMetadata(String spEntityId, ActionListener<SamlMetadataResponse> listener) {
+        idp.getRegisteredServiceProvider(spEntityId, true, ActionListener.wrap(
+            sp -> {
+                try {
+                    if (null == sp) {
+                        listener.onFailure(new IllegalArgumentException("Service provider with Entity ID [" + spEntityId
+                            + "] is not registered with this Identity Provider"));
+                        return;
+                    }
+                    EntityDescriptor metadata = buildEntityDescriptor(sp);
+                    final X509Credential signingCredential = idp.getMetadataSigningCredential();
+                    Element metadataElement = possiblySignDescriptor(metadata, signingCredential);
+                    listener.onResponse(new SamlMetadataResponse(samlFactory.toString(metadataElement, false)));
+                } catch (Exception e) {
+                    logger.debug("Error generating IDP metadata to share with [" + spEntityId + "]", e);
+                    listener.onFailure(e);
+                }
+            },
+            listener::onFailure
+        ));
+    }
+
+    EntityDescriptor buildEntityDescriptor(SamlServiceProvider sp) throws Exception {
+        final SamlIdPMetadataBuilder builder = new SamlIdPMetadataBuilder(idp.getEntityId())
+            .wantAuthnRequestsSigned(sp.shouldSignAuthnRequests())
+            .withSingleSignOnServiceUrl(SAML2_REDIRECT_BINDING_URI,
+                idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI))
+            .withSingleSignOnServiceUrl(SAML2_POST_BINDING_URI,
+                idp.getSingleSignOnEndpoint(SAML2_POST_BINDING_URI))
+            .withSingleLogoutServiceUrl(SAML2_REDIRECT_BINDING_URI,
+                idp.getSingleLogoutEndpoint(SAML2_REDIRECT_BINDING_URI))
+            .withSingleLogoutServiceUrl(SAML2_POST_BINDING_URI,
+                idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI))
+            .withNameIdFormat(PERSISTENT)
+            .withNameIdFormat(TRANSIENT)
+            .organization(idp.getOrganization())
+            .withContact(idp.getTechnicalContact());
+        final X509Credential signingCredential = idp.getSigningCredential();
+        if (null != signingCredential) {
+            builder.withSigningCertificate(signingCredential.getEntityCertificate());
+        }
+        return builder.build();
+    }
+
+    Element possiblySignDescriptor(EntityDescriptor descriptor, X509Credential signingCredential) throws MarshallingException,
+        SignatureException {
+        EntityDescriptorMarshaller marshaller = new EntityDescriptorMarshaller();
+        if (null == signingCredential) {
+            return marshaller.marshall(descriptor);
+        } else {
+            Signature signature = samlFactory.buildObject(Signature.class, DEFAULT_ELEMENT_NAME);
+            signature.setSigningCredential(signingCredential);
+            signature.setSignatureAlgorithm(ALGO_ID_SIGNATURE_RSA_SHA256);
+            signature.setCanonicalizationAlgorithm(ALGO_ID_C14N_EXCL_OMIT_COMMENTS);
+            descriptor.setSignature(signature);
+            Element element = new EntityDescriptorMarshaller().marshall(descriptor);
+            Signer.signObject(signature);
+            return element;
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java
new file mode 100644
index 0000000000000..0ef8c4b61ac84
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+
+import java.io.IOException;
+
+public abstract class IdpBaseRestHandler extends BaseRestHandler {
+
+    private static License.OperationMode MINIMUM_ALLOWED_LICENSE = License.OperationMode.ENTERPRISE;
+
+    protected final XPackLicenseState licenseState;
+
+    protected IdpBaseRestHandler(XPackLicenseState licenseState) {
+        this.licenseState = licenseState;
+    }
+
+    protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        RestChannelConsumer consumer = innerPrepareRequest(request, client);
+        if (isIdpFeatureAllowed()) {
+            return consumer;
+        } else {
+            return channel -> channel.sendResponse(new BytesRestResponse(channel,
+                LicenseUtils.newComplianceException("Identity Provider")));
+        }
+    }
+
+    protected boolean isIdpFeatureAllowed() {
+        return licenseState.isAllowedByLicense(MINIMUM_ALLOWED_LICENSE);
+    }
+
+    /**
+     * Implementers should implement this method as they normally would for
+     * {@link BaseRestHandler#prepareRequest(RestRequest, NodeClient)} and ensure that all request
+     * parameters are consumed prior to returning a value.
+     */
+    protected abstract RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException;
+}
+
+
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..a270f9409004b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderRequest;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderResponse;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Rest endpoint to remove a service provider (by Entity ID) from the IdP.
+ */
+public class RestDeleteSamlServiceProviderAction extends IdpBaseRestHandler {
+
+    public RestDeleteSamlServiceProviderAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "idp_delete_saml_sp_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(Method.DELETE, "/_idp/saml/sp/{sp_entity_id}"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        final String entityId = restRequest.param("sp_entity_id");
+        final WriteRequest.RefreshPolicy refresh = restRequest.hasParam("refresh")
+            ? WriteRequest.RefreshPolicy.parse(restRequest.param("refresh")) : WriteRequest.RefreshPolicy.NONE;
+        final DeleteSamlServiceProviderRequest request = new DeleteSamlServiceProviderRequest(entityId, refresh);
+        return channel -> client.execute(DeleteSamlServiceProviderAction.INSTANCE, request,
+            new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(DeleteSamlServiceProviderResponse response, XContentBuilder builder) throws Exception {
+                    response.toXContent(builder, restRequest);
+                    return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND, builder);
+                }
+            });
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..26f9e2c7d81d5
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderRequest;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderResponse;
+
+import java.io.IOException;
+import java.util.List;
+
+public class RestPutSamlServiceProviderAction extends IdpBaseRestHandler {
+
+    public RestPutSamlServiceProviderAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "idp_put_saml_sp_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(Method.PUT, "/_idp/saml/sp/{sp_entity_id}"),
+            new Route(Method.POST, "/_idp/saml/sp/{sp_entity_id}")
+        );
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        final String entityId = restRequest.param("sp_entity_id");
+        final WriteRequest.RefreshPolicy refreshPolicy = restRequest.hasParam("refresh")
+            ? WriteRequest.RefreshPolicy.parse(restRequest.param("refresh")) : PutSamlServiceProviderRequest.DEFAULT_REFRESH_POLICY;
+        try (XContentParser parser = restRequest.contentParser()) {
+            final PutSamlServiceProviderRequest request = PutSamlServiceProviderRequest.fromXContent(entityId, refreshPolicy, parser);
+            return channel -> client.execute(PutSamlServiceProviderAction.INSTANCE, request,
+                new RestBuilderListener<>(channel) {
+                    @Override
+                    public RestResponse buildResponse(PutSamlServiceProviderResponse response, XContentBuilder builder) throws Exception {
+                        response.toXContent(builder, restRequest);
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java
new file mode 100644
index 0000000000000..1903f4bca7b44
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnRequest;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnResponse;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestSamlInitiateSingleSignOnAction extends IdpBaseRestHandler {
+    static final ObjectParser<SamlInitiateSingleSignOnRequest, Void> PARSER = new ObjectParser<>("idp_init_sso",
+        SamlInitiateSingleSignOnRequest::new);
+
+    static {
+        PARSER.declareString(SamlInitiateSingleSignOnRequest::setSpEntityId, new ParseField("entity_id"));
+        PARSER.declareObject(SamlInitiateSingleSignOnRequest::setSamlAuthenticationState, (p, c) -> SamlAuthenticationState.fromXContent(p),
+            new ParseField("authn_state"));
+    }
+
+    public RestSamlInitiateSingleSignOnAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(
+            new Route(POST, "/_idp/saml/init")
+        );
+    }
+
+    @Override
+    public String getName() {
+        return "saml_idp_init_sso_action";
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        try (XContentParser parser = request.contentParser()) {
+            final SamlInitiateSingleSignOnRequest initRequest = PARSER.parse(parser, null);
+            return channel -> client.execute(SamlInitiateSingleSignOnAction.INSTANCE, initRequest,
+                new RestBuilderListener<SamlInitiateSingleSignOnResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(SamlInitiateSingleSignOnResponse response, XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        builder.field("post_url", response.getPostUrl());
+                        builder.field("saml_response", response.getSamlResponse());
+                        builder.startObject("service_provider");
+                        builder.field("entity_id", response.getEntityId());
+                        builder.endObject();
+                        builder.endObject();
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java
new file mode 100644
index 0000000000000..9212d956d1609
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.SamlMetadataAction;
+import org.elasticsearch.xpack.idp.action.SamlMetadataRequest;
+import org.elasticsearch.xpack.idp.action.SamlMetadataResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+public class RestSamlMetadataAction extends IdpBaseRestHandler {
+
+    public RestSamlMetadataAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "saml_metadata_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(new Route(GET, "/_idp/saml/metadata/{sp_entity_id}"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        final String spEntityId = request.param("sp_entity_id");
+        final SamlMetadataRequest metadataRequest = new SamlMetadataRequest(spEntityId);
+        return channel -> client.execute(SamlMetadataAction.INSTANCE, metadataRequest,
+            new RestBuilderListener<SamlMetadataResponse>(channel) {
+                @Override
+                public RestResponse buildResponse(SamlMetadataResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    builder.field("metadata", response.getXmlString());
+                    builder.endObject();
+                    return new BytesRestResponse(RestStatus.OK, builder);
+                }
+            });
+
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java
new file mode 100644
index 0000000000000..14fb48ad2fc16
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestAction;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestRequest;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestSamlValidateAuthenticationRequestAction extends IdpBaseRestHandler {
+
+    static final ObjectParser<SamlValidateAuthnRequestRequest, Void> PARSER =
+        new ObjectParser<>("idp_validate_authn_request", SamlValidateAuthnRequestRequest::new);
+
+    static {
+        PARSER.declareString(SamlValidateAuthnRequestRequest::setQueryString, new ParseField("authn_request_query"));
+    }
+
+    public RestSamlValidateAuthenticationRequestAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "saml_idp_validate_authn_request_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(new Route(POST, "/_idp/saml/validate"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        try (XContentParser parser = request.contentParser()) {
+            final SamlValidateAuthnRequestRequest validateRequest = PARSER.parse(parser, null);
+            return channel -> client.execute(SamlValidateAuthnRequestAction.INSTANCE, validateRequest,
+                new RestBuilderListener<SamlValidateAuthnRequestResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(SamlValidateAuthnRequestResponse response, XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        builder.startObject("service_provider");
+                        builder.field("entity_id", response.getSpEntityId());
+                        builder.endObject();
+                        builder.field("force_authn", response.isForceAuthn());
+                        builder.field("authn_state", response.getAuthnState());
+                        builder.endObject();
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java
new file mode 100644
index 0000000000000..4e99ac5004dc4
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.joda.time.ReadableDuration;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Set;
+
+
+public class CloudServiceProvider implements SamlServiceProvider {
+
+    private final String entityId;
+    private final String name;
+    private final boolean enabled;
+    private final URL assertionConsumerService;
+    private final String allowedNameIdFormat;
+    private final ReadableDuration authnExpiry;
+    private final ServiceProviderPrivileges privileges;
+    private final AttributeNames attributeNames;
+    private final Set<X509Credential> spSigningCredentials;
+    private final boolean signAuthnRequests;
+    private final boolean signLogoutRequests;
+
+    public CloudServiceProvider(String entityId, String name, boolean enabled, URL assertionConsumerService, String allowedNameIdFormat,
+                                ReadableDuration authnExpiry, ServiceProviderPrivileges privileges, AttributeNames attributeNames,
+                                Set<X509Credential> spSigningCredentials, boolean signAuthnRequests, boolean signLogoutRequests) {
+        if (Strings.isNullOrEmpty(entityId)) {
+            throw new IllegalArgumentException("Service Provider Entity ID cannot be null or empty");
+        }
+        this.entityId = entityId;
+        this.name = name;
+        this.enabled = enabled;
+        this.assertionConsumerService = assertionConsumerService;
+        this.allowedNameIdFormat = allowedNameIdFormat;
+        this.authnExpiry = authnExpiry;
+        this.privileges = privileges;
+        this.attributeNames = attributeNames;
+        this.spSigningCredentials = spSigningCredentials == null ? Set.of() : Set.copyOf(spSigningCredentials);
+        this.signLogoutRequests = signLogoutRequests;
+        this.signAuthnRequests = signAuthnRequests;
+    }
+
+    @Override
+    public String getEntityId() {
+        return entityId;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    @Override
+    public String getAllowedNameIdFormat() {
+        return allowedNameIdFormat;
+    }
+
+    @Override
+    public URL getAssertionConsumerService() {
+        return assertionConsumerService;
+    }
+
+    @Override
+    public ReadableDuration getAuthnExpiry() {
+        return authnExpiry;
+    }
+
+    @Override
+    public AttributeNames getAttributeNames() {
+        return attributeNames;
+    }
+
+    @Override
+    public Set<X509Credential> getSpSigningCredentials() {
+        return spSigningCredentials;
+    }
+
+    @Override
+    public boolean shouldSignAuthnRequests() {
+        return signAuthnRequests;
+    }
+
+    @Override
+    public boolean shouldSignLogoutRequests() {
+        return signLogoutRequests;
+    }
+
+    @Override
+    public ServiceProviderPrivileges getPrivileges() {
+        return privileges;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName()
+            + "{"
+            + "entityId=[" + entityId + ']'
+            + " name=[" + name + ']'
+            + " enabled=" + enabled
+            + " acs=[" + assertionConsumerService + "]"
+            + "}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java
new file mode 100644
index 0000000000000..bb22857cacf45
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.joda.time.ReadableDuration;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Set;
+
+/**
+ * SAML 2.0 configuration information about a specific service provider
+ */
+public interface SamlServiceProvider {
+
+    String getName();
+
+    boolean isEnabled();
+
+    String getEntityId();
+
+    String getAllowedNameIdFormat();
+
+    URL getAssertionConsumerService();
+
+    ReadableDuration getAuthnExpiry();
+
+    class AttributeNames {
+        public final String principal;
+        public final String name;
+        public final String email;
+        public final String roles;
+
+        public AttributeNames(String principal, String name, String email, String roles) {
+            this.principal = principal;
+            this.name = name;
+            this.email = email;
+            this.roles = roles;
+        }
+    }
+
+    AttributeNames getAttributeNames();
+
+    ServiceProviderPrivileges getPrivileges();
+
+    Set<X509Credential> getSpSigningCredentials();
+
+    boolean shouldSignAuthnRequests();
+
+    boolean shouldSignLogoutRequests();
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java
new file mode 100644
index 0000000000000..cde39f13cdebd
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java
@@ -0,0 +1,539 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.joda.time.Duration;
+import org.joda.time.ReadableDuration;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+/**
+ * This class models the storage of a {@link SamlServiceProvider} as an Elasticsearch document.
+ */
+public class SamlServiceProviderDocument implements ToXContentObject, Writeable {
+
+    public static final String SIGN_AUTHN = "authn";
+    public static final String SIGN_LOGOUT = "logout";
+    private static final Set<String> ALLOWED_SIGN_MESSAGES = Set.of(SIGN_AUTHN, SIGN_LOGOUT);
+
+    public static class Privileges {
+        public String resource;
+        public Map<String, String> roleActions = Map.of();
+
+        public void setResource(String resource) {
+            this.resource = resource;
+        }
+
+        public void setRoleActions(Map<String, String> roleActions) {
+            this.roleActions = roleActions;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final Privileges that = (Privileges) o;
+            return Objects.equals(resource, that.resource) &&
+                Objects.equals(roleActions, that.roleActions);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(resource, roleActions);
+        }
+    }
+
+    public static class AttributeNames {
+        public String principal;
+        @Nullable
+        public String email;
+        @Nullable
+        public String name;
+        @Nullable
+        public String roles;
+
+        public void setPrincipal(String principal) {
+            this.principal = principal;
+        }
+
+        public void setEmail(String email) {
+            this.email = email;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public void setRoles(String roles) {
+            this.roles = roles;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final AttributeNames that = (AttributeNames) o;
+            return Objects.equals(principal, that.principal) &&
+                Objects.equals(email, that.email) &&
+                Objects.equals(name, that.name) &&
+                Objects.equals(roles, that.roles);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(principal, email, name, roles);
+        }
+    }
+
+    public static class Certificates {
+        public List<String> serviceProviderSigning = List.of();
+        public List<String> identityProviderSigning = List.of();
+        public List<String> identityProviderMetadataSigning = List.of();
+
+        public void setServiceProviderSigning(Collection<String> serviceProviderSigning) {
+            this.serviceProviderSigning = serviceProviderSigning == null ? List.of() : List.copyOf(serviceProviderSigning);
+        }
+
+        public void setIdentityProviderSigning(Collection<String> identityProviderSigning) {
+            this.identityProviderSigning = identityProviderSigning == null ? List.of() : List.copyOf(identityProviderSigning);
+        }
+
+        public void setIdentityProviderMetadataSigning(Collection<String> identityProviderMetadataSigning) {
+            this.identityProviderMetadataSigning
+                = identityProviderMetadataSigning == null ? List.of() : List.copyOf(identityProviderMetadataSigning);
+        }
+
+        public void setServiceProviderX509SigningCertificates(Collection<X509Certificate> certificates) {
+            this.serviceProviderSigning = encodeCertificates(certificates);
+        }
+
+        public List<X509Certificate> getServiceProviderX509SigningCertificates() {
+            return decodeCertificates(this.serviceProviderSigning);
+        }
+
+        public void setIdentityProviderX509SigningCertificates(Collection<X509Certificate> certificates) {
+            this.identityProviderSigning = encodeCertificates(certificates);
+        }
+
+        public List<X509Certificate> getIdentityProviderX509SigningCertificates() {
+            return decodeCertificates(this.identityProviderSigning);
+        }
+
+        public void setIdentityProviderX509MetadataSigningCertificates(Collection<X509Certificate> certificates) {
+            this.identityProviderMetadataSigning = encodeCertificates(certificates);
+        }
+
+        public List<X509Certificate> getIdentityProviderX509MetadataSigningCertificates() {
+            return decodeCertificates(this.identityProviderMetadataSigning);
+        }
+
+        private List<String> encodeCertificates(Collection<X509Certificate> certificates) {
+            return certificates == null ? List.of() : certificates.stream()
+                .map(cert -> {
+                    try {
+                        return cert.getEncoded();
+                    } catch (CertificateEncodingException e) {
+                        throw new ElasticsearchException("Cannot read certificate", e);
+                    }
+                })
+                .map(Base64.getEncoder()::encodeToString)
+                .collect(Collectors.toUnmodifiableList());
+        }
+
+        private List<X509Certificate> decodeCertificates(List<String> encodedCertificates) {
+            if (encodedCertificates == null || encodedCertificates.isEmpty()) {
+                return List.of();
+            }
+            return encodedCertificates.stream().map(this::decodeCertificate).collect(Collectors.toUnmodifiableList());
+        }
+
+        private X509Certificate decodeCertificate(String base64Cert) {
+            final byte[] bytes = base64Cert.getBytes(StandardCharsets.UTF_8);
+            try (InputStream stream = new ByteArrayInputStream(bytes)) {
+                final List<Certificate> certificates = CertParsingUtils.readCertificates(Base64.getDecoder().wrap(stream));
+                if (certificates.size() == 1) {
+                    final Certificate certificate = certificates.get(0);
+                    if (certificate instanceof X509Certificate) {
+                        return (X509Certificate) certificate;
+                    } else {
+                        throw new ElasticsearchException("Certificate ({}) is not a X.509 certificate", certificate.getClass());
+                    }
+                } else {
+                    throw new ElasticsearchException("Expected a single certificate, but found {}", certificates.size());
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            } catch (CertificateException e) {
+                throw new ElasticsearchException("Cannot parse certificate(s)", e);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final Certificates that = (Certificates) o;
+            return Objects.equals(serviceProviderSigning, that.serviceProviderSigning) &&
+                Objects.equals(identityProviderSigning, that.identityProviderSigning) &&
+                Objects.equals(identityProviderMetadataSigning, that.identityProviderMetadataSigning);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(serviceProviderSigning, identityProviderSigning, identityProviderMetadataSigning);
+        }
+    }
+
+    @Nullable
+    public String docId;
+
+    public String name;
+
+    public String entityId;
+
+    public String acs;
+    
+    @Nullable
+    public String nameIdFormat;
+
+    public boolean enabled = true;
+    public Instant created;
+    public Instant lastModified;
+
+    public Set<String> signMessages = Set.of();
+
+    @Nullable
+    public Long authenticationExpiryMillis;
+
+    public final Privileges privileges = new Privileges();
+    public final AttributeNames attributeNames = new AttributeNames();
+    public final Certificates certificates = new Certificates();
+
+    public SamlServiceProviderDocument() {
+    }
+
+    public SamlServiceProviderDocument(StreamInput in) throws IOException {
+        docId = in.readOptionalString();
+        name = in.readString();
+        entityId = in.readString();
+        acs = in.readString();
+        enabled = in.readBoolean();
+        created = in.readInstant();
+        lastModified = in.readInstant();
+        nameIdFormat = in.readOptionalString();
+        authenticationExpiryMillis = in.readOptionalVLong();
+
+        privileges.resource = in.readString();
+        privileges.roleActions = in.readMap(StreamInput::readString, StreamInput::readString);
+
+        attributeNames.principal = in.readString();
+        attributeNames.email = in.readOptionalString();
+        attributeNames.name = in.readOptionalString();
+        attributeNames.roles = in.readOptionalString();
+
+        certificates.serviceProviderSigning = in.readStringList();
+        certificates.identityProviderSigning = in.readStringList();
+        certificates.identityProviderMetadataSigning = in.readStringList();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(docId);
+        out.writeString(name);
+        out.writeString(entityId);
+        out.writeString(acs);
+        out.writeBoolean(enabled);
+        out.writeInstant(created);
+        out.writeInstant(lastModified);
+        out.writeOptionalString(nameIdFormat);
+        out.writeOptionalVLong(authenticationExpiryMillis);
+
+        out.writeString(privileges.resource);
+        out.writeMap(privileges.roleActions == null ? Map.of() : privileges.roleActions,
+            StreamOutput::writeString, StreamOutput::writeString);
+
+        out.writeString(attributeNames.principal);
+        out.writeOptionalString(attributeNames.email);
+        out.writeOptionalString(attributeNames.name);
+        out.writeOptionalString(attributeNames.roles);
+
+        out.writeStringCollection(certificates.serviceProviderSigning);
+        out.writeStringCollection(certificates.identityProviderSigning);
+        out.writeStringCollection(certificates.identityProviderMetadataSigning);
+    }
+
+    public String getDocId() {
+        return docId;
+    }
+
+    public void setDocId(String docId) {
+        this.docId = docId;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public void setEntityId(String entityId) {
+        this.entityId = entityId;
+    }
+
+    public void setAcs(String acs) {
+        this.acs = acs;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public void setCreated(Instant created) {
+        this.created = created;
+    }
+
+    public void setCreatedMillis(Long millis) {
+        this.created = Instant.ofEpochMilli(millis);
+    }
+
+    public void setLastModifiedMillis(Long millis) {
+        this.lastModified = Instant.ofEpochMilli(millis);
+    }
+
+    public void setNameIdFormat(String nameIdFormat) {
+        this.nameIdFormat = nameIdFormat;
+    }
+
+    public void setSignMessages(Collection<String> signMessages) {
+        this.signMessages = signMessages == null ? Set.of() : Set.copyOf(signMessages);
+    }
+
+    public void setAuthenticationExpiryMillis(Long authenticationExpiryMillis) {
+        this.authenticationExpiryMillis = authenticationExpiryMillis;
+    }
+
+    public void setAuthenticationExpiry(ReadableDuration authnExpiry) {
+        this.authenticationExpiryMillis = authnExpiry == null ? null : authnExpiry.getMillis();
+    }
+
+    public ReadableDuration getAuthenticationExpiry() {
+        return authenticationExpiryMillis == null ? null : Duration.millis(this.authenticationExpiryMillis);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final SamlServiceProviderDocument that = (SamlServiceProviderDocument) o;
+        return Objects.equals(docId, that.docId) &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(entityId, that.entityId) &&
+            Objects.equals(acs, that.acs) &&
+            Objects.equals(enabled, that.enabled) &&
+            Objects.equals(created, that.created) &&
+            Objects.equals(lastModified, that.lastModified) &&
+            Objects.equals(nameIdFormat, that.nameIdFormat) &&
+            Objects.equals(authenticationExpiryMillis, that.authenticationExpiryMillis) &&
+            Objects.equals(certificates, that.certificates) &&
+            Objects.equals(privileges, that.privileges) &&
+            Objects.equals(attributeNames, that.attributeNames);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(docId, name, entityId, acs, enabled, created, lastModified, nameIdFormat, authenticationExpiryMillis,
+            certificates, privileges, attributeNames);
+    }
+
+    private static final ObjectParser<SamlServiceProviderDocument, SamlServiceProviderDocument> DOC_PARSER
+        = new ObjectParser<>("service_provider_doc", true, SamlServiceProviderDocument::new);
+    private static final ObjectParser<Privileges, Void> PRIVILEGES_PARSER = new ObjectParser<>("service_provider_priv", true, null);
+    private static final ObjectParser<AttributeNames, Void> ATTRIBUTES_PARSER = new ObjectParser<>("service_provider_attr", true, null);
+    private static final ObjectParser<Certificates, Void> CERTIFICATES_PARSER = new ObjectParser<>("service_provider_cert", true, null);
+
+    private static final BiConsumer<SamlServiceProviderDocument, Object> NULL_CONSUMER = (doc, obj) -> {
+    };
+
+    static {
+        DOC_PARSER.declareString(SamlServiceProviderDocument::setName, Fields.NAME);
+        DOC_PARSER.declareString(SamlServiceProviderDocument::setEntityId, Fields.ENTITY_ID);
+        DOC_PARSER.declareString(SamlServiceProviderDocument::setAcs, Fields.ACS);
+        DOC_PARSER.declareBoolean(SamlServiceProviderDocument::setEnabled, Fields.ENABLED);
+        DOC_PARSER.declareLong(SamlServiceProviderDocument::setCreatedMillis, Fields.CREATED_DATE);
+        DOC_PARSER.declareLong(SamlServiceProviderDocument::setLastModifiedMillis, Fields.LAST_MODIFIED);
+        DOC_PARSER.declareStringOrNull(SamlServiceProviderDocument::setNameIdFormat, Fields.NAME_ID);
+        DOC_PARSER.declareStringArray(SamlServiceProviderDocument::setSignMessages, Fields.SIGN_MSGS);
+        DOC_PARSER.declareField(SamlServiceProviderDocument::setAuthenticationExpiryMillis,
+            parser -> parser.currentToken() == XContentParser.Token.VALUE_NULL ? null : parser.longValue(),
+            Fields.AUTHN_EXPIRY, ObjectParser.ValueType.LONG_OR_NULL);
+
+        DOC_PARSER.declareObject(NULL_CONSUMER, (parser, doc) -> PRIVILEGES_PARSER.parse(parser, doc.privileges, null), Fields.PRIVILEGES);
+        PRIVILEGES_PARSER.declareString(Privileges::setResource, Fields.Privileges.RESOURCE);
+        PRIVILEGES_PARSER.declareField(Privileges::setRoleActions,
+            (parser, ignore) -> parser.currentToken() == XContentParser.Token.VALUE_NULL ? null : parser.mapStrings(),
+            Fields.Privileges.ROLES, ObjectParser.ValueType.OBJECT_OR_NULL);
+
+        DOC_PARSER.declareObject(NULL_CONSUMER, (p, doc) -> ATTRIBUTES_PARSER.parse(p, doc.attributeNames, null), Fields.ATTRIBUTES);
+        ATTRIBUTES_PARSER.declareString(AttributeNames::setPrincipal, Fields.Attributes.PRINCIPAL);
+        ATTRIBUTES_PARSER.declareStringOrNull(AttributeNames::setEmail, Fields.Attributes.EMAIL);
+        ATTRIBUTES_PARSER.declareStringOrNull(AttributeNames::setName, Fields.Attributes.NAME);
+        ATTRIBUTES_PARSER.declareStringOrNull(AttributeNames::setRoles, Fields.Attributes.ROLES);
+
+        DOC_PARSER.declareObject(NULL_CONSUMER, (p, doc) -> CERTIFICATES_PARSER.parse(p, doc.certificates, null), Fields.CERTIFICATES);
+        CERTIFICATES_PARSER.declareStringArray(Certificates::setServiceProviderSigning, Fields.Certificates.SP_SIGNING);
+        CERTIFICATES_PARSER.declareStringArray(Certificates::setIdentityProviderSigning, Fields.Certificates.IDP_SIGNING);
+        CERTIFICATES_PARSER.declareStringArray(Certificates::setIdentityProviderMetadataSigning, Fields.Certificates.IDP_METADATA);
+    }
+
+    public static SamlServiceProviderDocument fromXContent(String docId, XContentParser parser) throws IOException {
+        SamlServiceProviderDocument doc = new SamlServiceProviderDocument();
+        doc.setDocId(docId);
+        return DOC_PARSER.parse(parser, doc, doc);
+    }
+
+    public ValidationException validate() {
+        final ValidationException validation = new ValidationException();
+        if (Strings.isNullOrEmpty(name)) {
+            validation.addValidationError("field [" + Fields.NAME + "] is required, but was [" + name + "]");
+        }
+        if (Strings.isNullOrEmpty(entityId)) {
+            validation.addValidationError("field [" + Fields.ENTITY_ID + "] is required, but was [" + entityId + "]");
+        }
+        if (Strings.isNullOrEmpty(acs)) {
+            validation.addValidationError("field [" + Fields.ACS + "] is required, but was [" + acs + "]");
+        }
+        if (created == null) {
+            validation.addValidationError("field [" + Fields.CREATED_DATE + "] is required, but was [" + created + "]");
+        }
+        if (lastModified == null) {
+            validation.addValidationError("field [" + Fields.LAST_MODIFIED + "] is required, but was [" + lastModified + "]");
+        }
+
+        final Set<String> invalidSignOptions = Sets.difference(signMessages, ALLOWED_SIGN_MESSAGES);
+        if (invalidSignOptions.isEmpty() == false) {
+            validation.addValidationError("the values [" + invalidSignOptions + "] are not permitted for [" + Fields.SIGN_MSGS
+                + "] - permitted values are [" + ALLOWED_SIGN_MESSAGES + "]");
+        }
+
+        if (Strings.isNullOrEmpty(privileges.resource)) {
+            validation.addValidationError("field [" + Fields.PRIVILEGES + "." + Fields.Privileges.RESOURCE
+                + "] is required, but was [" + privileges.resource + "]");
+        }
+        if (Strings.isNullOrEmpty(attributeNames.principal)) {
+            validation.addValidationError("field [" + Fields.ATTRIBUTES + "." + Fields.Attributes.PRINCIPAL
+                + "] is required, but was [" + attributeNames.principal + "]");
+        }
+        if (validation.validationErrors().isEmpty()) {
+            return null;
+        } else {
+            return validation;
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Fields.NAME.getPreferredName(), name);
+        builder.field(Fields.ENTITY_ID.getPreferredName(), entityId);
+        builder.field(Fields.ACS.getPreferredName(), acs);
+        builder.field(Fields.ENABLED.getPreferredName(), enabled);
+        builder.field(Fields.CREATED_DATE.getPreferredName(), created == null ? null : created.toEpochMilli());
+        builder.field(Fields.LAST_MODIFIED.getPreferredName(), lastModified == null ? null : lastModified.toEpochMilli());
+        builder.field(Fields.NAME_ID.getPreferredName(), nameIdFormat);
+        builder.field(Fields.SIGN_MSGS.getPreferredName(), signMessages == null ? List.of() : signMessages);
+        builder.field(Fields.AUTHN_EXPIRY.getPreferredName(), authenticationExpiryMillis);
+
+        builder.startObject(Fields.PRIVILEGES.getPreferredName());
+        builder.field(Fields.Privileges.RESOURCE.getPreferredName(), privileges.resource);
+        builder.field(Fields.Privileges.ROLES.getPreferredName(), privileges.roleActions);
+        builder.endObject();
+
+        builder.startObject(Fields.ATTRIBUTES.getPreferredName());
+        builder.field(Fields.Attributes.PRINCIPAL.getPreferredName(), attributeNames.principal);
+        builder.field(Fields.Attributes.EMAIL.getPreferredName(), attributeNames.email);
+        builder.field(Fields.Attributes.NAME.getPreferredName(), attributeNames.name);
+        builder.field(Fields.Attributes.ROLES.getPreferredName(), attributeNames.roles);
+        builder.endObject();
+
+        builder.startObject(Fields.CERTIFICATES.getPreferredName());
+        builder.field(Fields.Certificates.SP_SIGNING.getPreferredName(), certificates.serviceProviderSigning);
+        builder.field(Fields.Certificates.IDP_SIGNING.getPreferredName(), certificates.identityProviderSigning);
+        builder.field(Fields.Certificates.IDP_METADATA.getPreferredName(), certificates.identityProviderMetadataSigning);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+
+    public interface Fields {
+        ParseField NAME = new ParseField("name");
+        ParseField ENTITY_ID = new ParseField("entity_id");
+        ParseField ACS = new ParseField("acs");
+        ParseField ENABLED = new ParseField("enabled");
+        ParseField NAME_ID = new ParseField("name_id_format");
+        ParseField SIGN_MSGS = new ParseField("sign_messages");
+        ParseField AUTHN_EXPIRY = new ParseField("authn_expiry_ms");
+
+        ParseField CREATED_DATE = new ParseField("created");
+        ParseField LAST_MODIFIED = new ParseField("last_modified");
+
+        ParseField PRIVILEGES = new ParseField("privileges");
+        ParseField ATTRIBUTES = new ParseField("attributes");
+        ParseField CERTIFICATES = new ParseField("certificates");
+
+        interface Privileges {
+            ParseField RESOURCE = new ParseField("resource");
+            ParseField ROLES = new ParseField("roles");
+        }
+
+        interface Attributes {
+            ParseField PRINCIPAL = new ParseField("principal");
+            ParseField EMAIL = new ParseField("email");
+            ParseField NAME = new ParseField("name");
+            ParseField ROLES = new ParseField("roles");
+        }
+
+        interface Certificates {
+            ParseField SP_SIGNING = new ParseField("sp_signing");
+            ParseField IDP_SIGNING = new ParseField("idp_signing");
+            ParseField IDP_METADATA = new ParseField("idp_metadata");
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{docId=" + docId + ", name=" + name + ", entityId=" + entityId + "}@" + hashCode();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java
new file mode 100644
index 0000000000000..e509654520d22
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java
@@ -0,0 +1,337 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.util.CachedSupplier;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * This class provides utility methods to read/write {@link SamlServiceProviderDocument} to an Elasticsearch index.
+ */
+public class SamlServiceProviderIndex implements Closeable {
+
+    private final Logger logger = LogManager.getLogger();
+
+    private final Client client;
+    private final ClusterService clusterService;
+    private final ClusterStateListener clusterStateListener;
+    private volatile boolean aliasExists;
+    private volatile boolean templateInstalled;
+
+    public static final String ALIAS_NAME = "saml-service-provider";
+    public static final String INDEX_NAME = "saml-service-provider-v1";
+    static final String TEMPLATE_NAME = ALIAS_NAME;
+
+    private static final String TEMPLATE_RESOURCE = "/org/elasticsearch/xpack/idp/saml-service-provider-template.json";
+    private static final String TEMPLATE_META_VERSION_KEY = "idp-version";
+    private static final String TEMPLATE_VERSION_SUBSTITUTE = "idp.template.version";
+
+    public static final class DocumentVersion {
+        public final String id;
+        public final long primaryTerm;
+        public final long seqNo;
+
+        public DocumentVersion(String id, long primaryTerm, long seqNo) {
+            this.id = id;
+            this.primaryTerm = primaryTerm;
+            this.seqNo = seqNo;
+        }
+
+        public DocumentVersion(GetResponse get) {
+            this(get.getId(), get.getPrimaryTerm(), get.getSeqNo());
+        }
+
+        public DocumentVersion(GetResult get) {
+            this(get.getId(), get.getPrimaryTerm(), get.getSeqNo());
+        }
+
+        public DocumentVersion(SearchHit hit) {
+            this(hit.getId(), hit.getPrimaryTerm(), hit.getSeqNo());
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final DocumentVersion that = (DocumentVersion) o;
+            return Objects.equals(this.id, that.id) && primaryTerm == that.primaryTerm &&
+                seqNo == that.seqNo;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, primaryTerm, seqNo);
+        }
+    }
+
+    public static final class DocumentSupplier {
+        public final DocumentVersion version;
+        public final Supplier<SamlServiceProviderDocument> document;
+
+        public DocumentSupplier(DocumentVersion version, Supplier<SamlServiceProviderDocument> document) {
+            this.version = version;
+            this.document = new CachedSupplier<>(document);
+        }
+
+        public SamlServiceProviderDocument getDocument() {
+            return document.get();
+        }
+    }
+
+    public SamlServiceProviderIndex(Client client, ClusterService clusterService) {
+        this.client = new OriginSettingClient(client, ClientHelper.IDP_ORIGIN);
+        this.clusterService = clusterService;
+        this.clusterStateListener = this::clusterChanged;
+        clusterService.addListener(clusterStateListener);
+    }
+
+    private void clusterChanged(ClusterChangedEvent clusterChangedEvent) {
+        final ClusterState state = clusterChangedEvent.state();
+        installTemplateIfRequired(state);
+        checkForAliasStateChange(state);
+    }
+
+    private void installTemplateIfRequired(ClusterState state) {
+        if (templateInstalled) {
+            return;
+        }
+        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            return;
+        }
+        if (isTemplateUpToDate(state)) {
+            templateInstalled = true;
+            return;
+        }
+        if (state.nodes().isLocalNodeElectedMaster() == false) {
+            return;
+        }
+        installIndexTemplate(ActionListener.wrap(
+            installed -> {
+                templateInstalled = true;
+                if (installed) {
+                    logger.debug("Template [{}] has been updated", TEMPLATE_NAME);
+                } else {
+                    logger.debug("Template [{}] appears to be up to date", TEMPLATE_NAME);
+                }
+            }, e -> logger.warn(new ParameterizedMessage("Failed to install template [{}]", TEMPLATE_NAME), e)
+        ));
+    }
+
+    private void checkForAliasStateChange(ClusterState state) {
+        final AliasOrIndex aliasInfo = state.getMetaData().getAliasAndIndexLookup().get(ALIAS_NAME);
+        final boolean previousState = aliasExists;
+        this.aliasExists = aliasInfo != null;
+        if (aliasExists != previousState) {
+            logChangedAliasState(aliasInfo);
+        }
+    }
+
+    @Override
+    public void close() {
+        logger.debug("Closing ... removing cluster state listener");
+        clusterService.removeListener(clusterStateListener);
+    }
+
+    private void logChangedAliasState(AliasOrIndex aliasInfo) {
+        if (aliasInfo == null) {
+            logger.warn("service provider index/alias [{}] no longer exists", ALIAS_NAME);
+        } else if (aliasInfo.isAlias() == false) {
+            logger.warn("service provider index [{}] exists as a concrete index, but it should be an alias", ALIAS_NAME);
+        } else if (aliasInfo.getIndices().size() != 1) {
+            logger.warn("service provider alias [{}] refers to multiple indices [{}] - this is unexpected and is likely to cause problems",
+                ALIAS_NAME, Strings.collectionToCommaDelimitedString(aliasInfo.getIndices()));
+        } else {
+            logger.info("service provider alias [{}] refers to [{}]", ALIAS_NAME, aliasInfo.getIndices().get(0).getIndex());
+        }
+    }
+
+    public void installIndexTemplate(ActionListener<Boolean> listener) {
+        final ClusterState state = clusterService.state();
+        if (isTemplateUpToDate(state)) {
+            listener.onResponse(false);
+        }
+        final String template = TemplateUtils.loadTemplate(TEMPLATE_RESOURCE, Version.CURRENT.toString(), TEMPLATE_VERSION_SUBSTITUTE);
+        final PutIndexTemplateRequest request = new PutIndexTemplateRequest(TEMPLATE_NAME).source(template, XContentType.JSON);
+        client.admin().indices().putTemplate(request, ActionListener.wrap(response -> {
+            logger.info("Installed template [{}]", TEMPLATE_NAME);
+            listener.onResponse(true);
+        }, listener::onFailure));
+    }
+
+    private boolean isTemplateUpToDate(ClusterState state) {
+        return TemplateUtils.checkTemplateExistsAndIsUpToDate(TEMPLATE_NAME, TEMPLATE_META_VERSION_KEY, state, logger);
+    }
+
+    public void deleteDocument(DocumentVersion version, WriteRequest.RefreshPolicy refreshPolicy, ActionListener<DeleteResponse> listener) {
+        final DeleteRequest request = new DeleteRequest(aliasExists ? ALIAS_NAME : INDEX_NAME)
+            .id(version.id)
+            .setIfSeqNo(version.seqNo)
+            .setIfPrimaryTerm(version.primaryTerm)
+            .setRefreshPolicy(refreshPolicy);
+        client.delete(request, ActionListener.wrap(response -> {
+            logger.debug("Deleted service provider document [{}] ({})", version.id, response.getResult());
+            listener.onResponse(response);
+        }, listener::onFailure));
+    }
+
+    public void writeDocument(SamlServiceProviderDocument document, DocWriteRequest.OpType opType,
+                              WriteRequest.RefreshPolicy refreshPolicy, ActionListener<DocWriteResponse> listener) {
+        final ValidationException exception = document.validate();
+        if (exception != null) {
+            listener.onFailure(exception);
+            return;
+        }
+
+        if (templateInstalled) {
+            _writeDocument(document, opType, refreshPolicy, listener);
+        } else {
+            installIndexTemplate(ActionListener.wrap(installed ->
+                _writeDocument(document, opType, refreshPolicy, listener), listener::onFailure));
+        }
+    }
+
+    private void _writeDocument(SamlServiceProviderDocument document, DocWriteRequest.OpType opType,
+                                WriteRequest.RefreshPolicy refreshPolicy, ActionListener<DocWriteResponse> listener) {
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream();
+             XContentBuilder xContentBuilder = new XContentBuilder(XContentType.JSON.xContent(), out)) {
+            document.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+            // Due to the lack of "alias templates" (at the current time), we cannot write to the alias if it doesn't exist yet
+            // - that would cause the alias to be created as a concrete index, which is not what we want.
+            // So, until we know that the alias exists we have to write to the expected index name instead.
+            final IndexRequest request = new IndexRequest(aliasExists ? ALIAS_NAME : INDEX_NAME)
+                .opType(opType)
+                .source(xContentBuilder)
+                .id(document.docId)
+                .setRefreshPolicy(refreshPolicy);
+            client.index(request, ActionListener.wrap(response -> {
+                logger.debug("Wrote service provider [{}][{}] as document [{}] ({})",
+                    document.name, document.entityId, response.getId(), response.getResult());
+                listener.onResponse(response);
+            }, listener::onFailure));
+        } catch (IOException e) {
+            listener.onFailure(e);
+        }
+    }
+
+    public void readDocument(String documentId, ActionListener<DocumentSupplier> listener) {
+        final GetRequest request = new GetRequest(ALIAS_NAME, documentId);
+        client.get(request, ActionListener.wrap(response -> {
+            if (response.isExists()) {
+                listener.onResponse(
+                    new DocumentSupplier(new DocumentVersion(response), () -> toDocument(documentId, response.getSourceAsBytesRef()))
+                );
+            } else {
+                listener.onResponse(null);
+            }
+        }, listener::onFailure));
+    }
+
+    public void findByEntityId(String entityId, ActionListener<Set<DocumentSupplier>> listener) {
+        final QueryBuilder query = QueryBuilders.termQuery(SamlServiceProviderDocument.Fields.ENTITY_ID.getPreferredName(), entityId);
+        findDocuments(query, listener);
+    }
+
+    public void findAll(ActionListener<Set<DocumentSupplier>> listener) {
+        final QueryBuilder query = QueryBuilders.matchAllQuery();
+        findDocuments(query, listener);
+    }
+
+    public void refresh(ActionListener<Void> listener) {
+        client.admin().indices().refresh(new RefreshRequest(ALIAS_NAME), ActionListener.wrap(
+            response -> listener.onResponse(null), listener::onFailure));
+    }
+
+    private void findDocuments(QueryBuilder query, ActionListener<Set<DocumentSupplier>> listener) {
+        logger.trace("Searching [{}] for [{}]", ALIAS_NAME, query);
+        final SearchRequest request = client.prepareSearch(ALIAS_NAME)
+            .setQuery(query)
+            .setSize(1000)
+            .setFetchSource(true)
+            .request();
+        client.search(request, ActionListener.wrap(response -> {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Search hits: [{}] [{}]", response.getHits().getTotalHits(), Arrays.toString(response.getHits().getHits()));
+            }
+            final Set<DocumentSupplier> docs = Stream.of(response.getHits().getHits())
+                .map(hit -> new DocumentSupplier(new DocumentVersion(hit), () -> toDocument(hit.getId(), hit.getSourceRef())))
+                .collect(Collectors.toUnmodifiableSet());
+            listener.onResponse(docs);
+        }, ex -> {
+            if (ex instanceof IndexNotFoundException) {
+                listener.onResponse(Set.of());
+            } else {
+                listener.onFailure(ex);
+            }
+        }));
+    }
+
+    private SamlServiceProviderDocument toDocument(String documentId, BytesReference source) {
+        try (StreamInput in = source.streamInput();
+             XContentParser parser = XContentType.JSON.xContent().createParser(
+                 NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, in)) {
+            return SamlServiceProviderDocument.fromXContent(documentId, parser);
+        } catch (IOException e) {
+            throw new UncheckedIOException("failed to parse document [" + documentId + "]", e);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{alias=" + ALIAS_NAME + " [" + (aliasExists ? "exists" : "not-found") + "]}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java
new file mode 100644
index 0000000000000..c4d1b89472101
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentSupplier;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+import org.joda.time.ReadableDuration;
+import org.opensaml.security.x509.BasicX509Credential;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class SamlServiceProviderResolver {
+
+    private static final int CACHE_SIZE_DEFAULT = 1000;
+    private static final TimeValue CACHE_TTL_DEFAULT = TimeValue.timeValueMinutes(60);
+
+    public static final Setting<Integer> CACHE_SIZE
+        = Setting.intSetting("xpack.idp.sp.cache.size", CACHE_SIZE_DEFAULT, Setting.Property.NodeScope);
+    public static final Setting<TimeValue> CACHE_TTL
+        = Setting.timeSetting("xpack.idp.sp.cache.ttl", CACHE_TTL_DEFAULT, Setting.Property.NodeScope);
+
+    private final Cache<String, CachedServiceProvider> cache;
+    private final SamlServiceProviderIndex index;
+    private final ServiceProviderDefaults defaults;
+
+    public SamlServiceProviderResolver(Settings settings, SamlServiceProviderIndex index, ServiceProviderDefaults defaults) {
+        this.cache = CacheBuilder.<String, CachedServiceProvider>builder()
+            .setMaximumWeight(CACHE_SIZE.get(settings))
+            .setExpireAfterAccess(CACHE_TTL.get(settings))
+            .build();
+        this.index = index;
+        this.defaults = defaults;
+    }
+
+    /**
+     * Find a {@link SamlServiceProvider} by entity-id.
+     *
+     * @param listener Callback for the service provider object. Calls {@link ActionListener#onResponse} with a {@code null} value if the
+     *                 service provider does not exist.
+     */
+    public void resolve(String entityId, ActionListener<SamlServiceProvider> listener) {
+        index.findByEntityId(entityId, ActionListener.wrap(
+            documentSuppliers -> {
+                if (documentSuppliers.isEmpty()) {
+                    listener.onResponse(null);
+                    return;
+                }
+                if (documentSuppliers.size() > 1) {
+                    listener.onFailure(new IllegalStateException(
+                        "Found multiple service providers with entity ID [" + entityId
+                            + "] - document ids ["
+                            + documentSuppliers.stream().map(s -> s.version.id).collect(Collectors.joining(","))
+                            + "] in index [" + index + "]"));
+                    return;
+                }
+                final DocumentSupplier doc = Iterables.get(documentSuppliers, 0);
+                final CachedServiceProvider cached = cache.get(entityId);
+                if (cached != null && cached.documentVersion.equals(doc.version)) {
+                    listener.onResponse(cached.serviceProvider);
+                    return;
+                } else {
+                    populateCacheAndReturn(entityId, doc, listener);
+                }
+            },
+            listener::onFailure
+        ));
+
+    }
+
+    private void populateCacheAndReturn(String entityId, DocumentSupplier doc, ActionListener<SamlServiceProvider> listener) {
+        final SamlServiceProvider serviceProvider = buildServiceProvider(doc.document.get());
+        final CachedServiceProvider cacheEntry = new CachedServiceProvider(entityId, doc.version, serviceProvider);
+        cache.put(entityId, cacheEntry);
+        listener.onResponse(serviceProvider);
+    }
+
+    private SamlServiceProvider buildServiceProvider(SamlServiceProviderDocument document) {
+        final ServiceProviderPrivileges privileges = buildPrivileges(document.privileges);
+        final SamlServiceProvider.AttributeNames attributes = new SamlServiceProvider.AttributeNames(
+            document.attributeNames.principal, document.attributeNames.name, document.attributeNames.email, document.attributeNames.roles
+        );
+        final Set<X509Credential> credentials = document.certificates.getServiceProviderX509SigningCertificates()
+            .stream()
+            .map(BasicX509Credential::new)
+            .collect(Collectors.toUnmodifiableSet());
+
+        final URL acs = parseUrl(document);
+        String nameIdFormat = document.nameIdFormat;
+        if (nameIdFormat == null) {
+            nameIdFormat = defaults.nameIdFormat;
+        }
+
+        final ReadableDuration authnExpiry = Optional.ofNullable(document.getAuthenticationExpiry())
+            .orElse(defaults.authenticationExpiry);
+
+        final boolean signAuthnRequests = document.signMessages.contains(SamlServiceProviderDocument.SIGN_AUTHN);
+        final boolean signLogoutRequests = document.signMessages.contains(SamlServiceProviderDocument.SIGN_LOGOUT);
+
+        return new CloudServiceProvider(document.entityId, document.name, document.enabled, acs, nameIdFormat, authnExpiry,
+            privileges, attributes, credentials, signAuthnRequests, signLogoutRequests);
+    }
+
+    private ServiceProviderPrivileges buildPrivileges(SamlServiceProviderDocument.Privileges configuredPrivileges) {
+        final String resource = configuredPrivileges.resource;
+        final Map<String, String> roles = Optional.ofNullable(configuredPrivileges.roleActions).orElse(Map.of());
+        return new ServiceProviderPrivileges(defaults.applicationName, resource, roles);
+    }
+
+    private URL parseUrl(SamlServiceProviderDocument document) {
+        final URL acs;
+        try {
+            acs = new URL(document.acs);
+        } catch (MalformedURLException e) {
+            final ServiceProviderException exception = new ServiceProviderException(
+                "Service provider [{}] (doc {}) has an invalid ACS [{}]", e, document.entityId, document.docId, document.acs);
+            exception.setEntityId(document.entityId);
+            throw exception;
+        }
+        return acs;
+    }
+
+    private class CachedServiceProvider {
+        private final String entityId;
+        private final DocumentVersion documentVersion;
+        private final SamlServiceProvider serviceProvider;
+
+        private CachedServiceProvider(String entityId, DocumentVersion documentVersion, SamlServiceProvider serviceProvider) {
+            this.entityId = entityId;
+            this.documentVersion = documentVersion;
+            this.serviceProvider = serviceProvider;
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java
new file mode 100644
index 0000000000000..a4e5523cc0e9f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.joda.time.Duration;
+import org.joda.time.ReadableDuration;
+import org.opensaml.saml.saml2.core.NameID;
+
+import java.util.List;
+
+/**
+ * Defines default values for a service provider if they are not configured in {@link SamlServiceProviderDocument}
+ */
+public final class ServiceProviderDefaults {
+
+    public static final Setting<String> APPLICATION_NAME_SETTING
+        = Setting.simpleString("xpack.idp.privileges.application", Setting.Property.NodeScope);
+    public static final Setting<String> NAMEID_FORMAT_SETTING
+        = Setting.simpleString("xpack.idp.defaults.nameid_format", NameID.TRANSIENT, Setting.Property.NodeScope);
+    public static final Setting<TimeValue> AUTHN_EXPIRY_SETTING
+        = Setting.timeSetting("xpack.idp.defaults.authn_expiry", TimeValue.timeValueMinutes(5), Setting.Property.NodeScope);
+
+    public final String applicationName;
+    public final String nameIdFormat;
+    public final ReadableDuration authenticationExpiry;
+
+    public ServiceProviderDefaults(String applicationName,
+                                   String nameIdFormat,
+                                   ReadableDuration authenticationExpiry) {
+        this.applicationName = applicationName;
+        this.nameIdFormat = nameIdFormat;
+        this.authenticationExpiry = authenticationExpiry;
+    }
+
+    public static ServiceProviderDefaults forSettings(Settings settings) {
+        final String appplication = require(settings, APPLICATION_NAME_SETTING);
+        final String nameId = NAMEID_FORMAT_SETTING.get(settings);
+        final TimeValue expiry = AUTHN_EXPIRY_SETTING.get(settings);
+        return new ServiceProviderDefaults(appplication, nameId, Duration.millis(expiry.millis()));
+    }
+
+    private static <T> T require(Settings settings, Setting<T> setting) {
+        if (setting.exists(settings)) {
+            return setting.get(settings);
+        }
+        throw new IllegalStateException("Setting [" + setting.getKey() + "] must be configured");
+    }
+
+    public static List<Setting<?>> getSettings() {
+        return List.of(APPLICATION_NAME_SETTING, NAMEID_FORMAT_SETTING, AUTHN_EXPIRY_SETTING);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java
new file mode 100644
index 0000000000000..9a6b03c1c73fe
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.ElasticsearchException;
+
+/**
+ * Indicates a configuration or execution problem specific to a SAML ServiceProvider
+ */
+public class ServiceProviderException extends ElasticsearchException {
+    public static final String ENTITY_ID = "es.idp.sp.entity_id";
+
+    public ServiceProviderException(String msg, Throwable cause, Object... args) {
+        super(msg, cause, args);
+    }
+
+    public void setEntityId(String entityId) {
+        super.addMetadata(ENTITY_ID, entityId);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java
new file mode 100644
index 0000000000000..779ffa89a61b2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Represents a state object that can be used during an SP initiated flow in order to pass necessary parameters from the response of the
+ * /validate API to the /init API. State properties are not integrity protected, might end up in a cookie on the user's side and as such
+ * are treated as tainted parameters for informational purposes only. No authentication/authorization decisions should be made based on
+ * these.
+ */
+public class SamlAuthenticationState implements Writeable, ToXContentObject {
+    private String entityId;
+    private String requestedAcsUrl;
+    @Nullable
+    private String requestedNameidFormat;
+    @Nullable
+    private String authnRequestId;
+
+    public SamlAuthenticationState() {
+
+    }
+
+    public SamlAuthenticationState(StreamInput in) throws IOException {
+        entityId = in.readString();
+        requestedAcsUrl = in.readString();
+        requestedNameidFormat = in.readOptionalString();
+        authnRequestId = in.readOptionalString();
+    }
+
+    public String getRequestedNameidFormat() {
+        return requestedNameidFormat;
+    }
+
+    public void setRequestedNameidFormat(String requestedNameidFormat) {
+        this.requestedNameidFormat = requestedNameidFormat;
+    }
+
+    public String getAuthnRequestId() {
+        return authnRequestId;
+    }
+
+    public void setAuthnRequestId(String authnRequestId) {
+        this.authnRequestId = authnRequestId;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public void setEntityId(String entityId) {
+        this.entityId = entityId;
+    }
+
+    public String getRequestedAcsUrl() {
+        return requestedAcsUrl;
+    }
+
+    public void setRequestedAcsUrl(String requestedAcsUrl) {
+        this.requestedAcsUrl = requestedAcsUrl;
+    }
+
+    public ValidationException validate() {
+        final ValidationException validation = new ValidationException();
+        if (Strings.isNullOrEmpty(entityId)) {
+            validation.addValidationError("field [" + Fields.ENTITY_ID + "] is required, but was [" + entityId + "]");
+        }
+        if (Strings.isNullOrEmpty(requestedAcsUrl)) {
+            validation.addValidationError("field [" + Fields.ACS_URL + "] is required, but was [" + requestedAcsUrl + "]");
+        }
+        return validation;
+    }
+
+    public static final ObjectParser<SamlAuthenticationState, SamlAuthenticationState> PARSER
+        = new ObjectParser<>("saml_authn_state", true, SamlAuthenticationState::new);
+
+    static {
+        PARSER.declareString(SamlAuthenticationState::setEntityId, Fields.ENTITY_ID);
+        PARSER.declareString(SamlAuthenticationState::setRequestedAcsUrl, Fields.ACS_URL);
+        PARSER.declareStringOrNull(SamlAuthenticationState::setRequestedNameidFormat, Fields.NAMEID_FORMAT);
+        PARSER.declareStringOrNull(SamlAuthenticationState::setAuthnRequestId, Fields.AUTHN_REQUEST_ID);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Fields.NAMEID_FORMAT.getPreferredName(), requestedNameidFormat);
+        builder.field(Fields.AUTHN_REQUEST_ID.getPreferredName(), authnRequestId);
+        builder.field(Fields.ENTITY_ID.getPreferredName(), entityId);
+        builder.field(Fields.ACS_URL.getPreferredName(), requestedAcsUrl);
+        return builder.endObject();
+    }
+
+    public static SamlAuthenticationState fromXContent(XContentParser parser) throws IOException {
+        SamlAuthenticationState authnState = new SamlAuthenticationState();
+        return PARSER.parse(parser, authnState, null);
+    }
+
+    public interface Fields {
+        ParseField NAMEID_FORMAT = new ParseField("nameid_format");
+        ParseField AUTHN_REQUEST_ID = new ParseField("authn_request_id");
+        ParseField ENTITY_ID = new ParseField("entity_id");
+        ParseField ACS_URL = new ParseField("acs_url");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(entityId);
+        out.writeString(requestedAcsUrl);
+        out.writeOptionalString(requestedNameidFormat);
+        out.writeOptionalString(authnRequestId);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + Strings.toString(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SamlAuthenticationState that = (SamlAuthenticationState) o;
+        return entityId.equals(that.entityId) &&
+            requestedAcsUrl.equals(that.requestedAcsUrl) &&
+            Objects.equals(requestedNameidFormat, that.requestedNameidFormat) &&
+            Objects.equals(authnRequestId, that.authnRequestId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(entityId, requestedAcsUrl, requestedNameidFormat, authnRequestId);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java
new file mode 100644
index 0000000000000..31dd6671cc078
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java
@@ -0,0 +1,359 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.opensaml.core.xml.XMLObject;
+import org.opensaml.core.xml.XMLObjectBuilderFactory;
+import org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport;
+import org.opensaml.core.xml.io.MarshallingException;
+import org.opensaml.core.xml.io.Unmarshaller;
+import org.opensaml.core.xml.io.UnmarshallerFactory;
+import org.opensaml.core.xml.io.UnmarshallingException;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.saml.common.SAMLObject;
+import org.opensaml.security.credential.Credential;
+import org.opensaml.security.x509.X509Credential;
+import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import javax.xml.XMLConstants;
+import javax.xml.namespace.QName;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.URISyntaxException;
+import java.security.SecureRandom;
+import java.security.cert.CertificateEncodingException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport.getUnmarshallerFactory;
+
+/**
+ * Utility object for constructing new objects and values in a SAML 2.0 / OpenSAML context
+ */
+public class SamlFactory {
+
+    private final XMLObjectBuilderFactory builderFactory;
+    private final SecureRandom random;
+    private static final Logger LOGGER = LogManager.getLogger(SamlFactory.class);
+
+    public SamlFactory() {
+        SamlInit.initialize();
+        builderFactory = XMLObjectProviderRegistrySupport.getBuilderFactory();
+        random = new SecureRandom();
+    }
+
+    public <T extends XMLObject> T object(Class<T> type, QName elementName) {
+        final XMLObject obj = builderFactory.getBuilder(elementName).buildObject(elementName);
+        return cast(type, elementName, obj);
+    }
+
+    public <T extends XMLObject> T object(Class<T> type, QName elementName, QName schemaType) {
+        final XMLObject obj = builderFactory.getBuilder(schemaType).buildObject(elementName, schemaType);
+        return cast(type, elementName, obj);
+    }
+
+    private <T extends XMLObject> T cast(Class<T> type, QName elementName, XMLObject obj) {
+        if (type.isInstance(obj)) {
+            return type.cast(obj);
+        } else {
+            throw new IllegalArgumentException("Object for element " + elementName.getLocalPart() + " is of type " + obj.getClass()
+                + " not " + type);
+        }
+    }
+
+    public String secureIdentifier() {
+        return randomNCName(20);
+    }
+
+    private String randomNCName(int numberBytes) {
+        final byte[] randomBytes = new byte[numberBytes];
+        random.nextBytes(randomBytes);
+        // NCNames (https://www.w3.org/TR/xmlschema-2/#NCName) can't start with a number, so start them all with "_" to be safe
+        return "_".concat(MessageDigests.toHexString(randomBytes));
+    }
+
+    public <T extends XMLObject> T buildObject(Class<T> type, QName elementName) {
+        final XMLObject obj = builderFactory.getBuilder(elementName).buildObject(elementName);
+        if (type.isInstance(obj)) {
+            return type.cast(obj);
+        } else {
+            throw new IllegalArgumentException("Object for element " + elementName.getLocalPart() + " is of type " + obj.getClass()
+                + " not " + type);
+        }
+    }
+
+    public String toString(Element element, boolean pretty) {
+        try {
+            StringWriter writer = new StringWriter();
+            print(element, writer, pretty);
+            return writer.toString();
+        } catch (TransformerException e) {
+            return "[" + element.getNamespaceURI() + "]" + element.getLocalName();
+        }
+    }
+
+    public <T extends XMLObject> T buildXmlObject(Element element, Class<T> type) {
+        try {
+            UnmarshallerFactory unmarshallerFactory = getUnmarshallerFactory();
+            Unmarshaller unmarshaller = unmarshallerFactory.getUnmarshaller(element);
+            if (unmarshaller == null) {
+                throw new ElasticsearchSecurityException("XML element [{}] cannot be unmarshalled to SAML type [{}] (no unmarshaller)",
+                    element.getTagName(), type);
+            }
+            final XMLObject object = unmarshaller.unmarshall(element);
+            if (type.isInstance(object)) {
+                return type.cast(object);
+            }
+            Object[] args = new Object[]{element.getTagName(), type.getName(), object.getClass().getName()};
+            throw new ElasticsearchSecurityException("SAML object [{}] is incorrect type. Expected [{}] but was [{}]", args);
+        } catch (UnmarshallingException e) {
+            throw new ElasticsearchSecurityException("Failed to unmarshall SAML content [{}]", e, element.getTagName());
+        }
+    }
+
+    void print(Element element, Writer writer, boolean pretty) throws TransformerException {
+        final Transformer serializer = getHardenedXMLTransformer();
+        if (pretty) {
+            serializer.setOutputProperty(OutputKeys.INDENT, "yes");
+        }
+        serializer.transform(new DOMSource(element), new StreamResult(writer));
+    }
+
+    public String getXmlContent(SAMLObject object){
+        return getXmlContent(object, false);
+    }
+
+    public String getXmlContent(SAMLObject object, boolean prettyPrint) {
+        try {
+            return toString(XMLObjectSupport.marshall(object), prettyPrint);
+        } catch (MarshallingException e) {
+            LOGGER.info("Error marshalling SAMLObject ", e);
+            return "_unserializable_";
+        }
+    }
+
+    public boolean elementNameMatches(Element element, String namespace, String localName) {
+        return localName.equals(element.getLocalName()) && namespace.equals(element.getNamespaceURI());
+    }
+
+    public String text(Element dom, int length) {
+        return text(dom, length, 0);
+    }
+
+    public String text(XMLObject xml, int prefixLength, int suffixLength) {
+        final Element dom = xml.getDOM();
+        if (dom == null) {
+            return null;
+        }
+        return text(dom, prefixLength, suffixLength);
+    }
+
+    public String text(XMLObject xml, int length) {
+        return text(xml, length, 0);
+    }
+
+    protected static String text(Element dom, int prefixLength, int suffixLength) {
+
+        final String text = dom.getTextContent().trim();
+        final int totalLength = prefixLength + suffixLength;
+        if (text.length() > totalLength) {
+            final String prefix = Strings.cleanTruncate(text, prefixLength) + "...";
+            if (suffixLength == 0) {
+                return prefix;
+            }
+            int suffixIndex = text.length() - suffixLength;
+            if (Character.isHighSurrogate(text.charAt(suffixIndex))) {
+                suffixIndex++;
+            }
+            return prefix + text.substring(suffixIndex);
+        } else {
+            return text;
+        }
+    }
+
+    public String describeCredentials(Collection<? extends Credential> credentials) {
+        return credentials.stream()
+            .map(c -> {
+                if (c == null) {
+                    return "<null>";
+                }
+                byte[] encoded;
+                if (c instanceof X509Credential) {
+                    X509Credential x = (X509Credential) c;
+                    try {
+                        encoded = x.getEntityCertificate().getEncoded();
+                    } catch (CertificateEncodingException e) {
+                        encoded = c.getPublicKey().getEncoded();
+                    }
+                } else {
+                    encoded = c.getPublicKey().getEncoded();
+                }
+                return Base64.getEncoder().encodeToString(encoded).substring(0, 64) + "...";
+            })
+            .collect(Collectors.joining(","));
+    }
+
+    public Element toDomElement(XMLObject object) {
+        try {
+            return XMLObjectSupport.marshall(object);
+        } catch (MarshallingException e) {
+            throw new ElasticsearchSecurityException("failed to marshall SAML object to DOM element", e);
+        }
+    }
+
+
+    @SuppressForbidden(reason = "This is the only allowed way to construct a Transformer")
+    public Transformer getHardenedXMLTransformer() throws TransformerConfigurationException {
+        final TransformerFactory tfactory = TransformerFactory.newInstance();
+        tfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        tfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        tfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+        tfactory.setAttribute("indent-number", 2);
+        Transformer transformer = tfactory.newTransformer();
+        transformer.setErrorListener(new SamlFactory.TransformerErrorListener());
+        return transformer;
+    }
+
+    /**
+     * Constructs a DocumentBuilder with all the necessary features for it to be secure
+     *
+     * @throws ParserConfigurationException if one of the features can't be set on the DocumentBuilderFactory
+     */
+    @SuppressForbidden(reason = "This is the only allowed way to construct a DocumentBuilder")
+    public static DocumentBuilder getHardenedBuilder(String[] schemaFiles) throws ParserConfigurationException {
+        final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        dbf.setNamespaceAware(true);
+        // Ensure that Schema Validation is enabled for the factory
+        dbf.setValidating(true);
+        // Disallow internal and external entity expansion
+        dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        dbf.setFeature("http://xml.org/sax/features/validation", true);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
+        dbf.setIgnoringComments(true);
+        // This is required, otherwise schema validation causes signature invalidation
+        dbf.setFeature("http://apache.org/xml/features/validation/schema/normalized-value", false);
+        // Make sure that URL schema namespaces are not resolved/downloaded from URLs we do not control
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "file,jar");
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "file,jar");
+        dbf.setFeature("http://apache.org/xml/features/honour-all-schemaLocations", true);
+        // Ensure we do not resolve XIncludes. Defaults to false, but set it explicitly to be future-proof
+        dbf.setXIncludeAware(false);
+        // Ensure we do not expand entity reference nodes
+        dbf.setExpandEntityReferences(false);
+        // Further limit danger from denial of service attacks
+        dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        dbf.setAttribute("http://apache.org/xml/features/validation/schema", true);
+        dbf.setAttribute("http://apache.org/xml/features/validation/schema-full-checking", true);
+        dbf.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaLanguage",
+            XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        // We ship our own xsd files for schema validation since we do not trust anyone else.
+        dbf.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaSource", resolveSchemaFilePaths(schemaFiles));
+        DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
+        documentBuilder.setErrorHandler(new SamlFactory.DocumentBuilderErrorHandler());
+        return documentBuilder;
+    }
+
+    public String getJavaAlorithmNameFromUri(String sigAlg) {
+        switch (sigAlg) {
+            case "http://www.w3.org/2000/09/xmldsig#dsa-sha1":
+                return "SHA1withDSA";
+            case "http://www.w3.org/2000/09/xmldsig#dsa-sha256":
+                return "SHA256withDSA";
+            case "http://www.w3.org/2000/09/xmldsig#rsa-sha1":
+                return "SHA1withRSA";
+            case "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256":
+                return "SHA256withRSA";
+            case "http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha256":
+                return "SHA256withECDSA";
+            default:
+                throw new IllegalArgumentException("Unsupported signing algorithm identifier: " + sigAlg);
+        }
+    }
+
+
+    private static String[] resolveSchemaFilePaths(String[] relativePaths) {
+
+        return Arrays.stream(relativePaths).
+            map(file -> {
+                try {
+                    return SamlFactory.class.getResource(file).toURI().toString();
+                } catch (URISyntaxException e) {
+                    LOGGER.warn("Error resolving schema file path", e);
+                    return null;
+                }
+            }).filter(Objects::nonNull).toArray(String[]::new);
+    }
+
+    private static class DocumentBuilderErrorHandler implements org.xml.sax.ErrorHandler {
+        /**
+         * Enabling schema validation with `setValidating(true)` in our
+         * DocumentBuilderFactory requires that we provide our own
+         * ErrorHandler implementation
+         *
+         * @throws SAXException If the document we attempt to parse is not valid according to the specified schema.
+         */
+        @Override
+        public void warning(SAXParseException e) throws SAXException {
+            LOGGER.debug("XML Parser error ", e);
+            throw e;
+        }
+
+        @Override
+        public void error(SAXParseException e) throws SAXException {
+            warning(e);
+        }
+
+        @Override
+        public void fatalError(SAXParseException e) throws SAXException {
+            warning(e);
+        }
+    }
+
+    private static class TransformerErrorListener implements javax.xml.transform.ErrorListener {
+
+        @Override
+        public void warning(TransformerException e) throws TransformerException {
+            LOGGER.debug("XML transformation error", e);
+            throw e;
+        }
+
+        @Override
+        public void error(TransformerException e) throws TransformerException {
+            warning(e);
+        }
+
+        @Override
+        public void fatalError(TransformerException e) throws TransformerException {
+            warning(e);
+        }
+    }
+
+}
+
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java
new file mode 100644
index 0000000000000..e6bfe1db8b219
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
+import org.opensaml.core.config.InitializationService;
+import org.slf4j.LoggerFactory;
+
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public final class SamlInit {
+
+    private static final AtomicBoolean INITIALISED = new AtomicBoolean(false);
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private SamlInit() { }
+
+    /**
+     * This is needed in order to initialize the underlying OpenSAML library.
+     * It must be called before doing anything that potentially interacts with OpenSAML (whether in server code, or in tests).
+     * The initialization happens within do privileged block as the underlying Apache XML security library has a permission check.
+     * The initialization happens with a specific context classloader as OpenSAML loads resources from its jar file.
+     */
+    public static void initialize() {
+        if (INITIALISED.compareAndSet(false, true)) {
+            // We want to force these classes to be loaded _before_ we fiddle with the context classloader
+            LoggerFactory.getLogger(InitializationService.class);
+            SpecialPermission.check();
+            try {
+                AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                    LOGGER.debug("Initializing OpenSAML");
+                    try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(InitializationService.class)) {
+                        InitializationService.initialize();
+                    }
+                    LOGGER.debug("Initialized OpenSAML");
+                    return null;
+                });
+            } catch (PrivilegedActionException e) {
+                throw new ElasticsearchSecurityException("failed to set context classloader for SAML IdP", e);
+            }
+        }
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java
new file mode 100644
index 0000000000000..d401e6bfc1b9d
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.opensaml.xmlsec.signature.SignableXMLObject;
+import org.opensaml.xmlsec.signature.Signature;
+import org.opensaml.xmlsec.signature.support.SignatureConstants;
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.xmlsec.signature.support.Signer;
+import org.opensaml.xmlsec.signature.support.SignerProvider;
+import org.w3c.dom.Element;
+
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Signs OpenSAML {@link SignableXMLObject} instances using {@link SamlIdentityProvider#getSigningCredential()}.
+ */
+public class SamlObjectSigner {
+
+    private final SamlIdentityProvider idp;
+    private final SamlFactory samlFactory;
+
+    public SamlObjectSigner(SamlFactory samlFactory, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.idp = idp;
+        this.samlFactory = samlFactory;
+    }
+
+    public Element sign(SignableXMLObject object) {
+        final Signature signature = samlFactory.buildObject(Signature.class, Signature.DEFAULT_ELEMENT_NAME);
+        signature.setSigningCredential(idp.getSigningCredential());
+        signature.setSignatureAlgorithm(SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256);
+        signature.setCanonicalizationAlgorithm(SignatureConstants.ALGO_ID_C14N_EXCL_OMIT_COMMENTS);
+        object.setSignature(signature);
+        Element element = samlFactory.toDomElement(object);
+        try {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignerProvider.class)) {
+                    Signer.signObject(signature);
+                } catch (SignatureException e) {
+                    throw new SecurityException("failed to sign SAML object " + object, e);
+                }
+                return null;
+            });
+        } catch (PrivilegedActionException e) {
+            throw new SecurityException("failed to sign SAML object " + object, e);
+        }
+        return element;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java
new file mode 100644
index 0000000000000..023c893dfb666
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.w3c.dom.bootstrap.DOMImplementationRegistry;
+import org.w3c.dom.ls.DOMImplementationLS;
+import org.w3c.dom.ls.LSInput;
+import org.w3c.dom.ls.LSResourceResolver;
+
+import javax.xml.XMLConstants;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Validates an XML stream against a specified schema.
+ */
+public class XmlValidator {
+
+    private final SchemaFactory schemaFactory;
+    private final String xsdName;
+
+    public XmlValidator(String xsdName) {
+        this.schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        this.xsdName = xsdName;
+    }
+
+    public void validate(String xml) throws Exception {
+        try(InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) {
+            validate(stream);
+        }
+    }
+
+    public void validate(InputStream xml) throws Exception {
+        try (InputStream xsdStream = loadSchema(xsdName);
+             ResourceResolver resolver = new ResourceResolver()) {
+            schemaFactory.setResourceResolver(resolver);
+            Schema schema = schemaFactory.newSchema(new StreamSource(xsdStream));
+            Validator validator = schema.newValidator();
+            validator.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+            validator.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+            validator.validate(new StreamSource(xml));
+        }
+    }
+
+    private InputStream loadSchema(String name) {
+        if (name.endsWith(".xsd") && name.indexOf('/') == -1 && name.indexOf('\\') == -1) {
+            return getClass().getResourceAsStream(name);
+        } else {
+            return null;
+        }
+    }
+
+    private class ResourceResolver implements LSResourceResolver, AutoCloseable {
+        private final DOMImplementationLS domLS;
+        private final List<InputStream> streams;
+
+        private ResourceResolver() throws InstantiationException, IllegalAccessException, ClassNotFoundException {
+            // Seriously, who thought this was a good idea for an API ???
+            DOMImplementationRegistry registry = DOMImplementationRegistry.newInstance();
+            domLS = (DOMImplementationLS) registry.getDOMImplementation("LS");
+            streams = new ArrayList<>();
+        }
+
+        @Override
+        public LSInput resolveResource(String type, String namespaceURI, String publicId, String systemId, String baseURI) {
+            InputStream stream = loadSchema(systemId);
+            if (stream == null) {
+                return null;
+            }
+            streams.add(stream);
+            final LSInput input = domLS.createLSInput();
+            input.setByteStream(stream);
+            return input;
+        }
+
+        @Override
+        public void close() throws IOException {
+            IOUtils.close(streams);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 0000000000000..65334a8db5760
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,29 @@
+grant {
+  permission java.lang.RuntimePermission "setFactory";
+
+  // ApacheXMLSecurityInitializer
+  permission java.util.PropertyPermission "org.apache.xml.security.ignoreLineBreaks", "read,write";
+
+  // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader)
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "setContextClassLoader";
+
+  // needed during initialization of OpenSAML library where xml security algorithms are registered
+  // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220
+  // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52
+  // which uses it in the opensaml-xmlsec-impl
+  permission java.security.SecurityPermission "org.apache.xml.security.register";
+
+  // needed for multiple server implementations used in tests
+  permission java.net.SocketPermission "*", "accept,connect";
+};
+
+grant codeBase "${codebase.elasticsearch-rest-client}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
+
+grant codeBase "${codebase.httpasyncclient}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd
new file mode 100644
index 0000000000000..759baf8993b3e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd
@@ -0,0 +1,283 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<schema
+    targetNamespace="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns="http://www.w3.org/2001/XMLSchema"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    xmlns:xenc="http://www.w3.org/2001/04/xmlenc#"
+    elementFormDefault="unqualified"
+    attributeFormDefault="unqualified"
+    blockDefault="substitution"
+    version="2.0">
+    <import namespace="http://www.w3.org/2000/09/xmldsig#"
+        schemaLocation="xmldsig-core-schema.xsd"/>
+    <import namespace="http://www.w3.org/2001/04/xmlenc#"
+        schemaLocation="xenc-schema.xsd"/>
+    <annotation>
+        <documentation>
+            Document identifier: saml-schema-assertion-2.0
+            Location: http://docs.oasis-open.org/security/saml/v2.0/
+            Revision history:
+            V1.0 (November, 2002):
+              Initial Standard Schema.
+            V1.1 (September, 2003):
+              Updates within the same V1.0 namespace.
+            V2.0 (March, 2005):
+              New assertion schema for SAML V2.0 namespace.
+        </documentation>
+    </annotation>
+    <attributeGroup name="IDNameQualifiers">
+        <attribute name="NameQualifier" type="string" use="optional"/>
+        <attribute name="SPNameQualifier" type="string" use="optional"/>
+    </attributeGroup>
+    <element name="BaseID" type="saml:BaseIDAbstractType"/>
+    <complexType name="BaseIDAbstractType" abstract="true">
+        <attributeGroup ref="saml:IDNameQualifiers"/>
+    </complexType>
+    <element name="NameID" type="saml:NameIDType"/>
+    <complexType name="NameIDType">
+        <simpleContent>
+            <extension base="string">
+                <attributeGroup ref="saml:IDNameQualifiers"/>
+                <attribute name="Format" type="anyURI" use="optional"/>
+                <attribute name="SPProvidedID" type="string" use="optional"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    <complexType name="EncryptedElementType">
+        <sequence>
+            <element ref="xenc:EncryptedData"/>
+            <element ref="xenc:EncryptedKey" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <element name="EncryptedID" type="saml:EncryptedElementType"/>
+    <element name="Issuer" type="saml:NameIDType"/>
+    <element name="AssertionIDRef" type="NCName"/>
+    <element name="AssertionURIRef" type="anyURI"/>
+    <element name="Assertion" type="saml:AssertionType"/>
+    <complexType name="AssertionType">
+        <sequence>
+            <element ref="saml:Issuer"/>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="saml:Subject" minOccurs="0"/>
+            <element ref="saml:Conditions" minOccurs="0"/>
+            <element ref="saml:Advice" minOccurs="0"/>
+            <choice minOccurs="0" maxOccurs="unbounded">
+                <element ref="saml:Statement"/>
+                <element ref="saml:AuthnStatement"/>
+                <element ref="saml:AuthzDecisionStatement"/>
+                <element ref="saml:AttributeStatement"/>
+            </choice>
+        </sequence>
+        <attribute name="Version" type="string" use="required"/>
+        <attribute name="ID" type="ID" use="required"/>
+        <attribute name="IssueInstant" type="dateTime" use="required"/>
+    </complexType>
+    <element name="Subject" type="saml:SubjectType"/>
+    <complexType name="SubjectType">
+        <choice>
+            <sequence>
+                <choice>
+                    <element ref="saml:BaseID"/>
+                    <element ref="saml:NameID"/>
+                    <element ref="saml:EncryptedID"/>
+                </choice>
+                <element ref="saml:SubjectConfirmation" minOccurs="0" maxOccurs="unbounded"/>
+            </sequence>
+            <element ref="saml:SubjectConfirmation" maxOccurs="unbounded"/>
+        </choice>
+    </complexType>
+    <element name="SubjectConfirmation" type="saml:SubjectConfirmationType"/>
+    <complexType name="SubjectConfirmationType">
+        <sequence>
+            <choice minOccurs="0">
+                <element ref="saml:BaseID"/>
+                <element ref="saml:NameID"/>
+                <element ref="saml:EncryptedID"/>
+            </choice>
+            <element ref="saml:SubjectConfirmationData" minOccurs="0"/>
+        </sequence>
+        <attribute name="Method" type="anyURI" use="required"/>
+    </complexType>
+    <element name="SubjectConfirmationData" type="saml:SubjectConfirmationDataType"/>
+    <complexType name="SubjectConfirmationDataType" mixed="true">
+        <complexContent>
+            <restriction base="anyType">
+                <sequence>
+                    <any namespace="##any" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="NotBefore" type="dateTime" use="optional"/>
+                <attribute name="NotOnOrAfter" type="dateTime" use="optional"/>
+                <attribute name="Recipient" type="anyURI" use="optional"/>
+                <attribute name="InResponseTo" type="NCName" use="optional"/>
+                <attribute name="Address" type="string" use="optional"/>
+                <anyAttribute namespace="##other" processContents="lax"/>
+            </restriction>
+        </complexContent>
+    </complexType>
+    <complexType name="KeyInfoConfirmationDataType" mixed="false">
+        <complexContent>
+            <restriction base="saml:SubjectConfirmationDataType">
+                <sequence>
+                    <element ref="ds:KeyInfo" maxOccurs="unbounded"/>
+                </sequence>
+            </restriction>
+        </complexContent>
+    </complexType>
+    <element name="Conditions" type="saml:ConditionsType"/>
+    <complexType name="ConditionsType">
+        <choice minOccurs="0" maxOccurs="unbounded">
+            <element ref="saml:Condition"/>
+            <element ref="saml:AudienceRestriction"/>
+            <element ref="saml:OneTimeUse"/>
+            <element ref="saml:ProxyRestriction"/>
+        </choice>
+        <attribute name="NotBefore" type="dateTime" use="optional"/>
+        <attribute name="NotOnOrAfter" type="dateTime" use="optional"/>
+    </complexType>
+    <element name="Condition" type="saml:ConditionAbstractType"/>
+    <complexType name="ConditionAbstractType" abstract="true"/>
+    <element name="AudienceRestriction" type="saml:AudienceRestrictionType"/>
+    <complexType name="AudienceRestrictionType">
+        <complexContent>
+            <extension base="saml:ConditionAbstractType">
+                <sequence>
+                    <element ref="saml:Audience" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="Audience" type="anyURI"/>
+    <element name="OneTimeUse" type="saml:OneTimeUseType" />
+    <complexType name="OneTimeUseType">
+        <complexContent>
+            <extension base="saml:ConditionAbstractType"/>
+        </complexContent>
+    </complexType>
+    <element name="ProxyRestriction" type="saml:ProxyRestrictionType"/>
+    <complexType name="ProxyRestrictionType">
+    <complexContent>
+        <extension base="saml:ConditionAbstractType">
+            <sequence>
+                <element ref="saml:Audience" minOccurs="0" maxOccurs="unbounded"/>
+            </sequence>
+            <attribute name="Count" type="nonNegativeInteger" use="optional"/>
+        </extension>
+    </complexContent>
+    </complexType>
+    <element name="Advice" type="saml:AdviceType"/>
+    <complexType name="AdviceType">
+        <choice minOccurs="0" maxOccurs="unbounded">
+            <element ref="saml:AssertionIDRef"/>
+            <element ref="saml:AssertionURIRef"/>
+            <element ref="saml:Assertion"/>
+            <element ref="saml:EncryptedAssertion"/>
+            <any namespace="##other" processContents="lax"/>
+        </choice>
+    </complexType>
+    <element name="EncryptedAssertion" type="saml:EncryptedElementType"/>
+    <element name="Statement" type="saml:StatementAbstractType"/>
+    <complexType name="StatementAbstractType" abstract="true"/>
+    <element name="AuthnStatement" type="saml:AuthnStatementType"/>
+    <complexType name="AuthnStatementType">
+        <complexContent>
+            <extension base="saml:StatementAbstractType">
+                <sequence>
+                    <element ref="saml:SubjectLocality" minOccurs="0"/>
+                    <element ref="saml:AuthnContext"/>
+                </sequence>
+                <attribute name="AuthnInstant" type="dateTime" use="required"/>
+                <attribute name="SessionIndex" type="string" use="optional"/>
+                <attribute name="SessionNotOnOrAfter" type="dateTime" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SubjectLocality" type="saml:SubjectLocalityType"/>
+    <complexType name="SubjectLocalityType">
+        <attribute name="Address" type="string" use="optional"/>
+        <attribute name="DNSName" type="string" use="optional"/>
+    </complexType>
+    <element name="AuthnContext" type="saml:AuthnContextType"/>
+    <complexType name="AuthnContextType">
+        <sequence>
+            <choice>
+                <sequence>
+                    <element ref="saml:AuthnContextClassRef"/>
+                    <choice minOccurs="0">
+                        <element ref="saml:AuthnContextDecl"/>
+                        <element ref="saml:AuthnContextDeclRef"/>
+                    </choice>
+                </sequence>
+                <choice>
+                    <element ref="saml:AuthnContextDecl"/>
+                    <element ref="saml:AuthnContextDeclRef"/>
+                </choice>
+            </choice>
+            <element ref="saml:AuthenticatingAuthority" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <element name="AuthnContextClassRef" type="anyURI"/>
+    <element name="AuthnContextDeclRef" type="anyURI"/>
+    <element name="AuthnContextDecl" type="anyType"/>
+    <element name="AuthenticatingAuthority" type="anyURI"/>
+    <element name="AuthzDecisionStatement" type="saml:AuthzDecisionStatementType"/>
+    <complexType name="AuthzDecisionStatementType">
+        <complexContent>
+            <extension base="saml:StatementAbstractType">
+                <sequence>
+                    <element ref="saml:Action" maxOccurs="unbounded"/>
+                    <element ref="saml:Evidence" minOccurs="0"/>
+                </sequence>
+                <attribute name="Resource" type="anyURI" use="required"/>
+                <attribute name="Decision" type="saml:DecisionType" use="required"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <simpleType name="DecisionType">
+        <restriction base="string">
+            <enumeration value="Permit"/>
+            <enumeration value="Deny"/>
+            <enumeration value="Indeterminate"/>
+        </restriction>
+    </simpleType>
+    <element name="Action" type="saml:ActionType"/>
+    <complexType name="ActionType">
+        <simpleContent>
+            <extension base="string">
+                <attribute name="Namespace" type="anyURI" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    <element name="Evidence" type="saml:EvidenceType"/>
+    <complexType name="EvidenceType">
+        <choice maxOccurs="unbounded">
+            <element ref="saml:AssertionIDRef"/>
+            <element ref="saml:AssertionURIRef"/>
+            <element ref="saml:Assertion"/>
+            <element ref="saml:EncryptedAssertion"/>
+        </choice>
+    </complexType>
+    <element name="AttributeStatement" type="saml:AttributeStatementType"/>
+    <complexType name="AttributeStatementType">
+        <complexContent>
+            <extension base="saml:StatementAbstractType">
+                <choice maxOccurs="unbounded">
+                    <element ref="saml:Attribute"/>
+                    <element ref="saml:EncryptedAttribute"/>
+                </choice>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="Attribute" type="saml:AttributeType"/>
+    <complexType name="AttributeType">
+        <sequence>
+            <element ref="saml:AttributeValue" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="Name" type="string" use="required"/>
+        <attribute name="NameFormat" type="anyURI" use="optional"/>
+        <attribute name="FriendlyName" type="string" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="AttributeValue" type="anyType" nillable="true"/>
+    <element name="EncryptedAttribute" type="saml:EncryptedElementType"/>
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd
new file mode 100644
index 0000000000000..9d5e4832a1ada
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema
+    targetNamespace="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    xmlns:xenc="http://www.w3.org/2001/04/xmlenc#"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns="http://www.w3.org/2001/XMLSchema"
+    elementFormDefault="unqualified"
+    attributeFormDefault="unqualified"
+    blockDefault="substitution"
+    version="2.0">
+    <import namespace="http://www.w3.org/2000/09/xmldsig#"
+        schemaLocation="xmldsig-core-schema.xsd"/>
+    <import namespace="http://www.w3.org/2001/04/xmlenc#"
+        schemaLocation="xenc-schema.xsd"/>
+    <import namespace="urn:oasis:names:tc:SAML:2.0:assertion"
+        schemaLocation="saml-schema-assertion-2.0.xsd"/>
+    <import namespace="http://www.w3.org/XML/1998/namespace"
+        schemaLocation="xml.xsd" />
+    <annotation>
+        <documentation>
+            Document identifier: saml-schema-metadata-2.0
+            Location: http://docs.oasis-open.org/security/saml/v2.0/
+            Revision history:
+              V2.0 (March, 2005):
+                Schema for SAML metadata, first published in SAML 2.0.
+        </documentation>
+    </annotation>
+
+    <simpleType name="entityIDType">
+        <restriction base="anyURI">
+            <maxLength value="1024"/>
+        </restriction>
+    </simpleType>
+    <complexType name="localizedNameType">
+        <simpleContent>
+            <extension base="string">
+                <attribute ref="xml:lang" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    <complexType name="localizedURIType">
+        <simpleContent>
+            <extension base="anyURI">
+                <attribute ref="xml:lang" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    
+    <element name="Extensions" type="md:ExtensionsType"/>
+    <complexType final="#all" name="ExtensionsType">
+        <sequence>
+            <any namespace="##other" processContents="lax" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    
+    <complexType name="EndpointType">
+        <sequence>
+            <any namespace="##other" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="Binding" type="anyURI" use="required"/>
+        <attribute name="Location" type="anyURI" use="required"/>
+        <attribute name="ResponseLocation" type="anyURI" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    
+    <complexType name="IndexedEndpointType">
+        <complexContent>
+            <extension base="md:EndpointType">
+                <attribute name="index" type="unsignedShort" use="required"/>
+                <attribute name="isDefault" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    
+    <element name="EntitiesDescriptor" type="md:EntitiesDescriptorType"/>
+    <complexType name="EntitiesDescriptorType">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <choice minOccurs="1" maxOccurs="unbounded">
+                <element ref="md:EntityDescriptor"/>
+                <element ref="md:EntitiesDescriptor"/>
+            </choice>
+        </sequence>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="ID" type="ID" use="optional"/>
+        <attribute name="Name" type="string" use="optional"/>
+    </complexType>
+
+    <element name="EntityDescriptor" type="md:EntityDescriptorType"/>
+    <complexType name="EntityDescriptorType">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <choice>
+                <choice maxOccurs="unbounded">
+                    <element ref="md:RoleDescriptor"/>
+                    <element ref="md:IDPSSODescriptor"/>
+                    <element ref="md:SPSSODescriptor"/>
+                    <element ref="md:AuthnAuthorityDescriptor"/>
+                    <element ref="md:AttributeAuthorityDescriptor"/>
+                    <element ref="md:PDPDescriptor"/>
+                </choice>
+                <element ref="md:AffiliationDescriptor"/>
+            </choice>
+            <element ref="md:Organization" minOccurs="0"/>
+            <element ref="md:ContactPerson" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:AdditionalMetadataLocation" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="entityID" type="md:entityIDType" use="required"/>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="ID" type="ID" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    
+    <element name="Organization" type="md:OrganizationType"/>
+    <complexType name="OrganizationType">
+        <sequence>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:OrganizationName" maxOccurs="unbounded"/>
+            <element ref="md:OrganizationDisplayName" maxOccurs="unbounded"/>
+            <element ref="md:OrganizationURL" maxOccurs="unbounded"/>
+        </sequence>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="OrganizationName" type="md:localizedNameType"/>
+    <element name="OrganizationDisplayName" type="md:localizedNameType"/>
+    <element name="OrganizationURL" type="md:localizedURIType"/>
+    <element name="ContactPerson" type="md:ContactType"/>
+    <complexType name="ContactType">
+        <sequence>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:Company" minOccurs="0"/>
+            <element ref="md:GivenName" minOccurs="0"/>
+            <element ref="md:SurName" minOccurs="0"/>
+            <element ref="md:EmailAddress" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:TelephoneNumber" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="contactType" type="md:ContactTypeType" use="required"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="Company" type="string"/>
+    <element name="GivenName" type="string"/>
+    <element name="SurName" type="string"/>
+    <element name="EmailAddress" type="anyURI"/>
+    <element name="TelephoneNumber" type="string"/>
+    <simpleType name="ContactTypeType">
+        <restriction base="string">
+            <enumeration value="technical"/>
+            <enumeration value="support"/>
+            <enumeration value="administrative"/>
+            <enumeration value="billing"/>
+            <enumeration value="other"/>
+        </restriction>
+    </simpleType>
+
+    <element name="AdditionalMetadataLocation" type="md:AdditionalMetadataLocationType"/>
+    <complexType name="AdditionalMetadataLocationType">
+        <simpleContent>
+            <extension base="anyURI">
+                <attribute name="namespace" type="anyURI" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+
+    <element name="RoleDescriptor" type="md:RoleDescriptorType"/>
+    <complexType name="RoleDescriptorType" abstract="true">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:KeyDescriptor" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:Organization" minOccurs="0"/>
+            <element ref="md:ContactPerson" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="ID" type="ID" use="optional"/>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="protocolSupportEnumeration" type="md:anyURIListType" use="required"/>
+        <attribute name="errorURL" type="anyURI" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <simpleType name="anyURIListType">
+        <list itemType="anyURI"/>
+    </simpleType>
+
+    <element name="KeyDescriptor" type="md:KeyDescriptorType"/>
+    <complexType name="KeyDescriptorType">
+        <sequence>
+            <element ref="ds:KeyInfo"/>
+            <element ref="md:EncryptionMethod" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="use" type="md:KeyTypes" use="optional"/>
+    </complexType>
+    <simpleType name="KeyTypes">
+        <restriction base="string">
+            <enumeration value="encryption"/>
+            <enumeration value="signing"/>
+        </restriction>
+    </simpleType>
+    <element name="EncryptionMethod" type="xenc:EncryptionMethodType"/>
+    
+    <complexType name="SSODescriptorType" abstract="true">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:ArtifactResolutionService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:SingleLogoutService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:ManageNameIDService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="ArtifactResolutionService" type="md:IndexedEndpointType"/>
+    <element name="SingleLogoutService" type="md:EndpointType"/>
+    <element name="ManageNameIDService" type="md:EndpointType"/>
+    <element name="NameIDFormat" type="anyURI"/>
+
+    <element name="IDPSSODescriptor" type="md:IDPSSODescriptorType"/>
+    <complexType name="IDPSSODescriptorType">
+        <complexContent>
+            <extension base="md:SSODescriptorType">
+                <sequence>
+                    <element ref="md:SingleSignOnService" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDMappingService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:AttributeProfile" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="saml:Attribute" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="WantAuthnRequestsSigned" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SingleSignOnService" type="md:EndpointType"/>
+    <element name="NameIDMappingService" type="md:EndpointType"/>
+    <element name="AssertionIDRequestService" type="md:EndpointType"/>
+    <element name="AttributeProfile" type="anyURI"/>
+    
+    <element name="SPSSODescriptor" type="md:SPSSODescriptorType"/>
+    <complexType name="SPSSODescriptorType">
+        <complexContent>
+            <extension base="md:SSODescriptorType">
+                <sequence>
+                    <element ref="md:AssertionConsumerService" maxOccurs="unbounded"/>
+                    <element ref="md:AttributeConsumingService" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="AuthnRequestsSigned" type="boolean" use="optional"/>
+                <attribute name="WantAssertionsSigned" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AssertionConsumerService" type="md:IndexedEndpointType"/>
+    <element name="AttributeConsumingService" type="md:AttributeConsumingServiceType"/>
+    <complexType name="AttributeConsumingServiceType">
+        <sequence>
+            <element ref="md:ServiceName" maxOccurs="unbounded"/>
+            <element ref="md:ServiceDescription" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:RequestedAttribute" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="index" type="unsignedShort" use="required"/>
+        <attribute name="isDefault" type="boolean" use="optional"/>
+    </complexType>
+    <element name="ServiceName" type="md:localizedNameType"/>
+    <element name="ServiceDescription" type="md:localizedNameType"/>
+    <element name="RequestedAttribute" type="md:RequestedAttributeType"/>
+    <complexType name="RequestedAttributeType">
+        <complexContent>
+            <extension base="saml:AttributeType">
+                <attribute name="isRequired" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+  
+    <element name="AuthnAuthorityDescriptor" type="md:AuthnAuthorityDescriptorType"/>
+    <complexType name="AuthnAuthorityDescriptorType">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:AuthnQueryService" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthnQueryService" type="md:EndpointType"/>
+
+    <element name="PDPDescriptor" type="md:PDPDescriptorType"/>
+    <complexType name="PDPDescriptorType">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:AuthzService" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthzService" type="md:EndpointType"/>
+
+    <element name="AttributeAuthorityDescriptor" type="md:AttributeAuthorityDescriptorType"/>
+    <complexType name="AttributeAuthorityDescriptorType">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:AttributeService" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:AttributeProfile" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="saml:Attribute" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AttributeService" type="md:EndpointType"/>
+   
+    <element name="AffiliationDescriptor" type="md:AffiliationDescriptorType"/>
+    <complexType name="AffiliationDescriptorType">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:AffiliateMember" maxOccurs="unbounded"/>
+            <element ref="md:KeyDescriptor" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="affiliationOwnerID" type="md:entityIDType" use="required"/>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="ID" type="ID" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="AffiliateMember" type="md:entityIDType"/>
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd
new file mode 100644
index 0000000000000..48ec69cbc967d
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd
@@ -0,0 +1,302 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema
+    targetNamespace="urn:oasis:names:tc:SAML:2.0:protocol"
+    xmlns="http://www.w3.org/2001/XMLSchema"
+    xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    elementFormDefault="unqualified"
+    attributeFormDefault="unqualified"
+    blockDefault="substitution"
+    version="2.0">
+    <import namespace="urn:oasis:names:tc:SAML:2.0:assertion"
+        schemaLocation="saml-schema-assertion-2.0.xsd"/>
+    <import namespace="http://www.w3.org/2000/09/xmldsig#"
+        schemaLocation="xmldsig-core-schema.xsd"/>
+    <annotation>
+        <documentation>
+            Document identifier: saml-schema-protocol-2.0
+            Location: http://docs.oasis-open.org/security/saml/v2.0/
+            Revision history:
+            V1.0 (November, 2002):
+              Initial Standard Schema.
+            V1.1 (September, 2003):
+              Updates within the same V1.0 namespace.
+            V2.0 (March, 2005):
+              New protocol schema based in a SAML V2.0 namespace.
+     </documentation>
+    </annotation>
+    <complexType name="RequestAbstractType" abstract="true">
+        <sequence>
+            <element ref="saml:Issuer" minOccurs="0"/>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="samlp:Extensions" minOccurs="0"/>
+        </sequence>
+        <attribute name="ID" type="ID" use="required"/>
+        <attribute name="Version" type="string" use="required"/>
+        <attribute name="IssueInstant" type="dateTime" use="required"/>
+        <attribute name="Destination" type="anyURI" use="optional"/>
+        <attribute name="Consent" type="anyURI" use="optional"/>
+    </complexType>
+    <element name="Extensions" type="samlp:ExtensionsType"/>
+    <complexType name="ExtensionsType">
+        <sequence>
+            <any namespace="##other" processContents="lax" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <complexType name="StatusResponseType">
+        <sequence>
+            <element ref="saml:Issuer" minOccurs="0"/>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="samlp:Extensions" minOccurs="0"/>
+            <element ref="samlp:Status"/>
+        </sequence>
+        <attribute name="ID" type="ID" use="required"/>
+        <attribute name="InResponseTo" type="NCName" use="optional"/>
+        <attribute name="Version" type="string" use="required"/>
+        <attribute name="IssueInstant" type="dateTime" use="required"/>
+        <attribute name="Destination" type="anyURI" use="optional"/>
+        <attribute name="Consent" type="anyURI" use="optional"/>
+    </complexType>
+    <element name="Status" type="samlp:StatusType"/>
+    <complexType name="StatusType">
+        <sequence>
+            <element ref="samlp:StatusCode"/>
+            <element ref="samlp:StatusMessage" minOccurs="0"/>
+            <element ref="samlp:StatusDetail" minOccurs="0"/>
+        </sequence>
+    </complexType>
+    <element name="StatusCode" type="samlp:StatusCodeType"/>
+    <complexType name="StatusCodeType">
+        <sequence>
+            <element ref="samlp:StatusCode" minOccurs="0"/>
+        </sequence>
+        <attribute name="Value" type="anyURI" use="required"/>
+    </complexType>
+    <element name="StatusMessage" type="string"/>
+    <element name="StatusDetail" type="samlp:StatusDetailType"/>
+    <complexType name="StatusDetailType">
+        <sequence>
+            <any namespace="##any" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <element name="AssertionIDRequest" type="samlp:AssertionIDRequestType"/>
+    <complexType name="AssertionIDRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="saml:AssertionIDRef" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SubjectQuery" type="samlp:SubjectQueryAbstractType"/>
+    <complexType name="SubjectQueryAbstractType" abstract="true">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="saml:Subject"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthnQuery" type="samlp:AuthnQueryType"/>
+    <complexType name="AuthnQueryType">
+        <complexContent>
+            <extension base="samlp:SubjectQueryAbstractType">
+                <sequence>
+                    <element ref="samlp:RequestedAuthnContext" minOccurs="0"/>
+                </sequence>
+                <attribute name="SessionIndex" type="string" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="RequestedAuthnContext" type="samlp:RequestedAuthnContextType"/>
+    <complexType name="RequestedAuthnContextType">
+        <choice>
+            <element ref="saml:AuthnContextClassRef" maxOccurs="unbounded"/>
+            <element ref="saml:AuthnContextDeclRef" maxOccurs="unbounded"/>
+        </choice>
+        <attribute name="Comparison" type="samlp:AuthnContextComparisonType" use="optional"/>
+    </complexType>
+    <simpleType name="AuthnContextComparisonType">
+        <restriction base="string">
+            <enumeration value="exact"/>
+            <enumeration value="minimum"/>
+            <enumeration value="maximum"/>
+            <enumeration value="better"/>
+        </restriction>
+    </simpleType>
+    <element name="AttributeQuery" type="samlp:AttributeQueryType"/>
+    <complexType name="AttributeQueryType">
+        <complexContent>
+            <extension base="samlp:SubjectQueryAbstractType">
+                <sequence>
+                    <element ref="saml:Attribute" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthzDecisionQuery" type="samlp:AuthzDecisionQueryType"/>
+    <complexType name="AuthzDecisionQueryType">
+        <complexContent>
+            <extension base="samlp:SubjectQueryAbstractType">
+                <sequence>
+                    <element ref="saml:Action" maxOccurs="unbounded"/>
+                    <element ref="saml:Evidence" minOccurs="0"/>
+                </sequence>
+                <attribute name="Resource" type="anyURI" use="required"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthnRequest" type="samlp:AuthnRequestType"/>
+    <complexType name="AuthnRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="saml:Subject" minOccurs="0"/>
+                    <element ref="samlp:NameIDPolicy" minOccurs="0"/>
+                    <element ref="saml:Conditions" minOccurs="0"/>
+                    <element ref="samlp:RequestedAuthnContext" minOccurs="0"/>
+                    <element ref="samlp:Scoping" minOccurs="0"/>
+                </sequence>
+                <attribute name="ForceAuthn" type="boolean" use="optional"/>
+                <attribute name="IsPassive" type="boolean" use="optional"/>
+                <attribute name="ProtocolBinding" type="anyURI" use="optional"/>
+                <attribute name="AssertionConsumerServiceIndex" type="unsignedShort" use="optional"/>
+                <attribute name="AssertionConsumerServiceURL" type="anyURI" use="optional"/>
+                <attribute name="AttributeConsumingServiceIndex" type="unsignedShort" use="optional"/>
+                <attribute name="ProviderName" type="string" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="NameIDPolicy" type="samlp:NameIDPolicyType"/>
+    <complexType name="NameIDPolicyType">
+        <attribute name="Format" type="anyURI" use="optional"/>
+        <attribute name="SPNameQualifier" type="string" use="optional"/>
+        <attribute name="AllowCreate" type="boolean" use="optional"/>
+    </complexType>
+    <element name="Scoping" type="samlp:ScopingType"/>
+    <complexType name="ScopingType">
+        <sequence>
+            <element ref="samlp:IDPList" minOccurs="0"/>
+            <element ref="samlp:RequesterID" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="ProxyCount" type="nonNegativeInteger" use="optional"/>
+    </complexType>
+    <element name="RequesterID" type="anyURI"/>
+    <element name="IDPList" type="samlp:IDPListType"/>
+    <complexType name="IDPListType">
+        <sequence>
+            <element ref="samlp:IDPEntry" maxOccurs="unbounded"/>
+            <element ref="samlp:GetComplete" minOccurs="0"/>
+        </sequence>
+    </complexType>
+    <element name="IDPEntry" type="samlp:IDPEntryType"/>
+    <complexType name="IDPEntryType">
+        <attribute name="ProviderID" type="anyURI" use="required"/>
+        <attribute name="Name" type="string" use="optional"/>
+        <attribute name="Loc" type="anyURI" use="optional"/>
+    </complexType>
+    <element name="GetComplete" type="anyURI"/>
+    <element name="Response" type="samlp:ResponseType"/>
+    <complexType name="ResponseType">
+        <complexContent>
+            <extension base="samlp:StatusResponseType">
+                <choice minOccurs="0" maxOccurs="unbounded">
+                    <element ref="saml:Assertion"/>
+                    <element ref="saml:EncryptedAssertion"/>
+                </choice>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="ArtifactResolve" type="samlp:ArtifactResolveType"/>
+    <complexType name="ArtifactResolveType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="samlp:Artifact"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="Artifact" type="string"/>
+    <element name="ArtifactResponse" type="samlp:ArtifactResponseType"/>
+    <complexType name="ArtifactResponseType">
+        <complexContent>
+            <extension base="samlp:StatusResponseType">
+                <sequence>
+                    <any namespace="##any" processContents="lax" minOccurs="0"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="ManageNameIDRequest" type="samlp:ManageNameIDRequestType"/>
+    <complexType name="ManageNameIDRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <choice>
+                        <element ref="saml:NameID"/>
+                        <element ref="saml:EncryptedID"/>
+                    </choice>
+                    <choice>
+                        <element ref="samlp:NewID"/>
+                        <element ref="samlp:NewEncryptedID"/>
+                        <element ref="samlp:Terminate"/>
+                    </choice>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="NewID" type="string"/>
+    <element name="NewEncryptedID" type="saml:EncryptedElementType"/>
+    <element name="Terminate" type="samlp:TerminateType"/>
+    <complexType name="TerminateType"/>
+    <element name="ManageNameIDResponse" type="samlp:StatusResponseType"/>
+    <element name="LogoutRequest" type="samlp:LogoutRequestType"/>
+    <complexType name="LogoutRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <choice>
+                        <element ref="saml:BaseID"/>
+                        <element ref="saml:NameID"/>
+                        <element ref="saml:EncryptedID"/>
+                    </choice>
+                    <element ref="samlp:SessionIndex" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="Reason" type="string" use="optional"/>
+                <attribute name="NotOnOrAfter" type="dateTime" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SessionIndex" type="string"/>
+    <element name="LogoutResponse" type="samlp:StatusResponseType"/>
+    <element name="NameIDMappingRequest" type="samlp:NameIDMappingRequestType"/>
+    <complexType name="NameIDMappingRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <choice>
+                        <element ref="saml:BaseID"/>
+                        <element ref="saml:NameID"/>
+                        <element ref="saml:EncryptedID"/>
+                    </choice>
+                    <element ref="samlp:NameIDPolicy"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="NameIDMappingResponse" type="samlp:NameIDMappingResponseType"/>
+    <complexType name="NameIDMappingResponseType">
+        <complexContent>
+            <extension base="samlp:StatusResponseType">
+                <choice>
+                    <element ref="saml:NameID"/>
+                    <element ref="saml:EncryptedID"/>
+                </choice>
+            </extension>
+        </complexContent>
+    </complexType>
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd
new file mode 100644
index 0000000000000..c902d4fc60772
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="utf-8"?>
+<schema xmlns='http://www.w3.org/2001/XMLSchema' version='1.0'
+        xmlns:xenc='http://www.w3.org/2001/04/xmlenc#'
+        xmlns:ds='http://www.w3.org/2000/09/xmldsig#'
+        targetNamespace='http://www.w3.org/2001/04/xmlenc#'
+        elementFormDefault='qualified'>
+
+  <import namespace='http://www.w3.org/2000/09/xmldsig#'
+          schemaLocation='xmldsig-core-schema.xsd'/>
+
+  <complexType name='EncryptedType' abstract='true'>
+    <sequence>
+      <element name='EncryptionMethod' type='xenc:EncryptionMethodType'
+       minOccurs='0'/>
+      <element ref='ds:KeyInfo' minOccurs='0'/>
+      <element ref='xenc:CipherData'/>
+      <element ref='xenc:EncryptionProperties' minOccurs='0'/>
+    </sequence>
+    <attribute name='Id' type='ID' use='optional'/>
+    <attribute name='Type' type='anyURI' use='optional'/>
+    <attribute name='MimeType' type='string' use='optional'/>
+    <attribute name='Encoding' type='anyURI' use='optional'/>
+  </complexType>
+  
+  <complexType name='EncryptionMethodType' mixed='true'>
+    <sequence>
+      <element name='KeySize' minOccurs='0' type='xenc:KeySizeType'/>
+      <element name='OAEPparams' minOccurs='0' type='base64Binary'/>
+      <any namespace='##other' minOccurs='0' maxOccurs='unbounded'/>
+    </sequence>
+    <attribute name='Algorithm' type='anyURI' use='required'/>
+  </complexType>
+
+    <simpleType name='KeySizeType'>
+      <restriction base="integer"/>
+    </simpleType>
+
+  <element name='CipherData' type='xenc:CipherDataType'/>
+  <complexType name='CipherDataType'>
+     <choice>
+       <element name='CipherValue' type='base64Binary'/>
+       <element ref='xenc:CipherReference'/>
+     </choice>
+    </complexType>
+
+   <element name='CipherReference' type='xenc:CipherReferenceType'/>
+   <complexType name='CipherReferenceType'>
+       <choice>
+         <element name='Transforms' type='xenc:TransformsType' minOccurs='0'/>
+       </choice>
+       <attribute name='URI' type='anyURI' use='required'/>
+   </complexType>
+
+     <complexType name='TransformsType'>
+       <sequence>
+         <element ref='ds:Transform' maxOccurs='unbounded'/>
+       </sequence>
+     </complexType>
+
+
+  <element name='EncryptedData' type='xenc:EncryptedDataType'/>
+  <complexType name='EncryptedDataType'>
+    <complexContent>
+      <extension base='xenc:EncryptedType'>
+       </extension>
+    </complexContent>
+  </complexType>
+
+  <!-- Children of ds:KeyInfo -->
+
+  <element name='EncryptedKey' type='xenc:EncryptedKeyType'/>
+  <complexType name='EncryptedKeyType'>
+    <complexContent>
+      <extension base='xenc:EncryptedType'>
+        <sequence>
+          <element ref='xenc:ReferenceList' minOccurs='0'/>
+          <element name='CarriedKeyName' type='string' minOccurs='0'/>
+        </sequence>
+        <attribute name='Recipient' type='string'
+         use='optional'/>
+      </extension>
+    </complexContent>
+  </complexType>
+
+    <element name="AgreementMethod" type="xenc:AgreementMethodType"/>
+    <complexType name="AgreementMethodType" mixed="true">
+      <sequence>
+        <element name="KA-Nonce" minOccurs="0" type="base64Binary"/>
+        <!-- <element ref="ds:DigestMethod" minOccurs="0"/> -->
+        <any namespace="##other" minOccurs="0" maxOccurs="unbounded"/>
+        <element name="OriginatorKeyInfo" minOccurs="0" type="ds:KeyInfoType"/>
+        <element name="RecipientKeyInfo" minOccurs="0" type="ds:KeyInfoType"/>
+      </sequence>
+      <attribute name="Algorithm" type="anyURI" use="required"/>
+    </complexType>
+
+  <!-- End Children of ds:KeyInfo -->
+
+  <element name='ReferenceList'>
+    <complexType>
+      <choice minOccurs='1' maxOccurs='unbounded'>
+        <element name='DataReference' type='xenc:ReferenceType'/>
+        <element name='KeyReference' type='xenc:ReferenceType'/>
+      </choice>
+    </complexType>
+  </element>
+
+  <complexType name='ReferenceType'>
+    <sequence>
+      <any namespace='##other' minOccurs='0' maxOccurs='unbounded'/>
+    </sequence>
+    <attribute name='URI' type='anyURI' use='required'/>
+  </complexType>
+
+
+  <element name='EncryptionProperties' type='xenc:EncryptionPropertiesType'/>
+  <complexType name='EncryptionPropertiesType'>
+    <sequence>
+      <element ref='xenc:EncryptionProperty' maxOccurs='unbounded'/>
+    </sequence>
+    <attribute name='Id' type='ID' use='optional'/>
+  </complexType>
+
+    <element name='EncryptionProperty' type='xenc:EncryptionPropertyType'/>
+    <complexType name='EncryptionPropertyType' mixed='true'>
+      <choice maxOccurs='unbounded'>
+        <any namespace='##other' processContents='lax'/>
+      </choice>
+      <attribute name='Target' type='anyURI' use='optional'/>
+      <attribute name='Id' type='ID' use='optional'/>
+      <anyAttribute namespace="http://www.w3.org/XML/1998/namespace"/>
+    </complexType>
+
+</schema>
+
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd
new file mode 100644
index 0000000000000..5a282019b6afe
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd
@@ -0,0 +1,286 @@
+<?xml version='1.0'?>
+<xs:schema targetNamespace="http://www.w3.org/XML/1998/namespace"
+  xmlns:xs="http://www.w3.org/2001/XMLSchema" 
+  xmlns   ="http://www.w3.org/1999/xhtml"
+  xml:lang="en">
+
+ <xs:annotation>
+  <xs:documentation>
+   <div>
+    <h1>About the XML namespace</h1>
+
+    <div class="bodytext">
+     <p>
+      This schema document describes the XML namespace, in a form
+      suitable for import by other schema documents.
+     </p>
+     <p>
+      See <a href="http://www.w3.org/XML/1998/namespace.html">
+      http://www.w3.org/XML/1998/namespace.html</a> and
+      <a href="http://www.w3.org/TR/REC-xml">
+      http://www.w3.org/TR/REC-xml</a> for information 
+      about this namespace.
+     </p>
+     <p>
+      Note that local names in this namespace are intended to be
+      defined only by the World Wide Web Consortium or its subgroups.
+      The names currently defined in this namespace are listed below.
+      They should not be used with conflicting semantics by any Working
+      Group, specification, or document instance.
+     </p>
+     <p>   
+      See further below in this document for more information about <a
+      href="#usage">how to refer to this schema document from your own
+      XSD schema documents</a> and about <a href="#nsversioning">the
+      namespace-versioning policy governing this schema document</a>.
+     </p>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:attribute name="lang">
+  <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>lang (as an attribute name)</h3>
+      <p>
+       denotes an attribute whose value
+       is a language code for the natural language of the content of
+       any element; its value is inherited.  This name is reserved
+       by virtue of its definition in the XML specification.</p>
+     
+    </div>
+    <div>
+     <h4>Notes</h4>
+     <p>
+      Attempting to install the relevant ISO 2- and 3-letter
+      codes as the enumerated possible values is probably never
+      going to be a realistic possibility.  
+     </p>
+     <p>
+      See BCP 47 at <a href="http://www.rfc-editor.org/rfc/bcp/bcp47.txt">
+       http://www.rfc-editor.org/rfc/bcp/bcp47.txt</a>
+      and the IANA language subtag registry at
+      <a href="http://www.iana.org/assignments/language-subtag-registry">
+       http://www.iana.org/assignments/language-subtag-registry</a>
+      for further information.
+     </p>
+     <p>
+      The union allows for the 'un-declaration' of xml:lang with
+      the empty string.
+     </p>
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+  <xs:simpleType>
+   <xs:union memberTypes="xs:language">
+    <xs:simpleType>    
+     <xs:restriction base="xs:string">
+      <xs:enumeration value=""/>
+     </xs:restriction>
+    </xs:simpleType>
+   </xs:union>
+  </xs:simpleType>
+ </xs:attribute>
+
+ <xs:attribute name="space">
+  <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>space (as an attribute name)</h3>
+      <p>
+       denotes an attribute whose
+       value is a keyword indicating what whitespace processing
+       discipline is intended for the content of the element; its
+       value is inherited.  This name is reserved by virtue of its
+       definition in the XML specification.</p>
+     
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+  <xs:simpleType>
+   <xs:restriction base="xs:NCName">
+    <xs:enumeration value="default"/>
+    <xs:enumeration value="preserve"/>
+   </xs:restriction>
+  </xs:simpleType>
+ </xs:attribute>
+ 
+ <xs:attribute name="base" type="xs:anyURI"> <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>base (as an attribute name)</h3>
+      <p>
+       denotes an attribute whose value
+       provides a URI to be used as the base for interpreting any
+       relative URIs in the scope of the element on which it
+       appears; its value is inherited.  This name is reserved
+       by virtue of its definition in the XML Base specification.</p>
+     
+     <p>
+      See <a
+      href="http://www.w3.org/TR/xmlbase/">http://www.w3.org/TR/xmlbase/</a>
+      for information about this attribute.
+     </p>
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+ </xs:attribute>
+ 
+ <xs:attribute name="id" type="xs:ID">
+  <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>id (as an attribute name)</h3> 
+      <p>
+       denotes an attribute whose value
+       should be interpreted as if declared to be of type ID.
+       This name is reserved by virtue of its definition in the
+       xml:id specification.</p>
+     
+     <p>
+      See <a
+      href="http://www.w3.org/TR/xml-id/">http://www.w3.org/TR/xml-id/</a>
+      for information about this attribute.
+     </p>
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+ </xs:attribute>
+
+ <xs:attributeGroup name="specialAttrs">
+  <xs:attribute ref="xml:base"/>
+  <xs:attribute ref="xml:lang"/>
+  <xs:attribute ref="xml:space"/>
+  <xs:attribute ref="xml:id"/>
+ </xs:attributeGroup>
+
+ <xs:annotation>
+  <xs:documentation>
+   <div>
+   
+    <h3>Father (in any context at all)</h3> 
+
+    <div class="bodytext">
+     <p>
+      denotes Jon Bosak, the chair of 
+      the original XML Working Group.  This name is reserved by 
+      the following decision of the W3C XML Plenary and 
+      XML Coordination groups:
+     </p>
+     <blockquote>
+       <p>
+        In appreciation for his vision, leadership and
+        dedication the W3C XML Plenary on this 10th day of
+        February, 2000, reserves for Jon Bosak in perpetuity
+        the XML name "xml:Father".
+       </p>
+     </blockquote>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:annotation>
+  <xs:documentation>
+   <div xml:id="usage" id="usage">
+    <h2><a name="usage">About this schema document</a></h2>
+
+    <div class="bodytext">
+     <p>
+      This schema defines attributes and an attribute group suitable
+      for use by schemas wishing to allow <code>xml:base</code>,
+      <code>xml:lang</code>, <code>xml:space</code> or
+      <code>xml:id</code> attributes on elements they define.
+     </p>
+     <p>
+      To enable this, such a schema must import this schema for
+      the XML namespace, e.g. as follows:
+     </p>
+     <pre>
+          &lt;schema . . .>
+           . . .
+           &lt;import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
+     </pre>
+     <p>
+      or
+     </p>
+     <pre>
+           &lt;import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
+     </pre>
+     <p>
+      Subsequently, qualified reference to any of the attributes or the
+      group defined below will have the desired effect, e.g.
+     </p>
+     <pre>
+          &lt;type . . .>
+           . . .
+           &lt;attributeGroup ref="xml:specialAttrs"/>
+     </pre>
+     <p>
+      will define a type which will schema-validate an instance element
+      with any of those attributes.
+     </p>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:annotation>
+  <xs:documentation>
+   <div id="nsversioning" xml:id="nsversioning">
+    <h2><a name="nsversioning">Versioning policy for this schema document</a></h2>
+    <div class="bodytext">
+     <p>
+      In keeping with the XML Schema WG's standard versioning
+      policy, this schema document will persist at
+      <a href="http://www.w3.org/2009/01/xml.xsd">
+       http://www.w3.org/2009/01/xml.xsd</a>.
+     </p>
+     <p>
+      At the date of issue it can also be found at
+      <a href="http://www.w3.org/2001/xml.xsd">
+       http://www.w3.org/2001/xml.xsd</a>.
+     </p>
+     <p>
+      The schema document at that URI may however change in the future,
+      in order to remain compatible with the latest version of XML
+      Schema itself, or with the XML namespace itself.  In other words,
+      if the XML Schema or XML namespaces change, the version of this
+      document at <a href="http://www.w3.org/2001/xml.xsd">
+       http://www.w3.org/2001/xml.xsd 
+      </a> 
+      will change accordingly; the version at 
+      <a href="http://www.w3.org/2009/01/xml.xsd">
+       http://www.w3.org/2009/01/xml.xsd 
+      </a> 
+      will not change.
+     </p>
+     <p>
+      Previous dated (and unchanging) versions of this schema 
+      document are at:
+     </p>
+     <ul>
+      <li><a href="http://www.w3.org/2009/01/xml.xsd">
+        http://www.w3.org/2009/01/xml.xsd</a></li>
+      <li><a href="http://www.w3.org/2007/08/xml.xsd">
+        http://www.w3.org/2007/08/xml.xsd</a></li>
+      <li><a href="http://www.w3.org/2004/10/xml.xsd">
+        http://www.w3.org/2004/10/xml.xsd</a></li>
+      <li><a href="http://www.w3.org/2001/03/xml.xsd">
+        http://www.w3.org/2001/03/xml.xsd</a></li>
+     </ul>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+</xs:schema>
+
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd
new file mode 100644
index 0000000000000..8422fdfaaf9d2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd
@@ -0,0 +1,308 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Schema for XML Signatures
+    http://www.w3.org/2000/09/xmldsig#
+    $Revision: 1.1 $ on $Date: 2002/02/08 20:32:26 $ by $Author: reagle $
+
+    Copyright 2001 The Internet Society and W3C (Massachusetts Institute
+    of Technology, Institut National de Recherche en Informatique et en
+    Automatique, Keio University). All Rights Reserved.
+    http://www.w3.org/Consortium/Legal/
+
+    This document is governed by the W3C Software License [1] as described
+    in the FAQ [2].
+
+    [1] http://www.w3.org/Consortium/Legal/copyright-software-19980720
+    [2] http://www.w3.org/Consortium/Legal/IPR-FAQ-20000620.html#DTD
+-->
+
+
+<schema xmlns="http://www.w3.org/2001/XMLSchema"
+        xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+        targetNamespace="http://www.w3.org/2000/09/xmldsig#"
+        version="0.1" elementFormDefault="qualified"> 
+
+<!-- Basic Types Defined for Signatures -->
+
+<simpleType name="CryptoBinary">
+  <restriction base="base64Binary">
+  </restriction>
+</simpleType>
+
+<!-- Start Signature -->
+
+<element name="Signature" type="ds:SignatureType"/>
+<complexType name="SignatureType">
+  <sequence> 
+    <element ref="ds:SignedInfo"/> 
+    <element ref="ds:SignatureValue"/> 
+    <element ref="ds:KeyInfo" minOccurs="0"/> 
+    <element ref="ds:Object" minOccurs="0" maxOccurs="unbounded"/> 
+  </sequence>  
+  <attribute name="Id" type="ID" use="optional"/>
+</complexType>
+
+  <element name="SignatureValue" type="ds:SignatureValueType"/> 
+  <complexType name="SignatureValueType">
+    <simpleContent>
+      <extension base="base64Binary">
+        <attribute name="Id" type="ID" use="optional"/>
+      </extension>
+    </simpleContent>
+  </complexType>
+
+<!-- Start SignedInfo -->
+
+<element name="SignedInfo" type="ds:SignedInfoType"/>
+<complexType name="SignedInfoType">
+  <sequence> 
+    <element ref="ds:CanonicalizationMethod"/> 
+    <element ref="ds:SignatureMethod"/> 
+    <element ref="ds:Reference" maxOccurs="unbounded"/> 
+  </sequence>  
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+  <element name="CanonicalizationMethod" type="ds:CanonicalizationMethodType"/> 
+  <complexType name="CanonicalizationMethodType" mixed="true">
+    <sequence>
+      <any namespace="##any" minOccurs="0" maxOccurs="unbounded"/>
+      <!-- (0,unbounded) elements from (1,1) namespace -->
+    </sequence>
+    <attribute name="Algorithm" type="anyURI" use="required"/> 
+  </complexType>
+
+  <element name="SignatureMethod" type="ds:SignatureMethodType"/>
+  <complexType name="SignatureMethodType" mixed="true">
+    <sequence>
+      <element name="HMACOutputLength" minOccurs="0" type="ds:HMACOutputLengthType"/>
+      <any namespace="##other" minOccurs="0" maxOccurs="unbounded"/>
+      <!-- (0,unbounded) elements from (1,1) external namespace -->
+    </sequence>
+    <attribute name="Algorithm" type="anyURI" use="required"/> 
+  </complexType>
+
+<!-- Start Reference -->
+
+<element name="Reference" type="ds:ReferenceType"/>
+<complexType name="ReferenceType">
+  <sequence> 
+    <element ref="ds:Transforms" minOccurs="0"/> 
+    <element ref="ds:DigestMethod"/> 
+    <element ref="ds:DigestValue"/> 
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+  <attribute name="URI" type="anyURI" use="optional"/> 
+  <attribute name="Type" type="anyURI" use="optional"/> 
+</complexType>
+
+  <element name="Transforms" type="ds:TransformsType"/>
+  <complexType name="TransformsType">
+    <sequence>
+      <element ref="ds:Transform" maxOccurs="unbounded"/>  
+    </sequence>
+  </complexType>
+
+  <element name="Transform" type="ds:TransformType"/>
+  <complexType name="TransformType" mixed="true">
+    <choice minOccurs="0" maxOccurs="unbounded"> 
+      <any namespace="##other" processContents="lax"/>
+      <!-- (1,1) elements from (0,unbounded) namespaces -->
+      <element name="XPath" type="string"/> 
+    </choice>
+    <attribute name="Algorithm" type="anyURI" use="required"/> 
+  </complexType>
+
+<!-- End Reference -->
+
+<element name="DigestMethod" type="ds:DigestMethodType"/>
+<complexType name="DigestMethodType" mixed="true"> 
+  <sequence>
+    <any namespace="##other" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+  </sequence>    
+  <attribute name="Algorithm" type="anyURI" use="required"/> 
+</complexType>
+
+<element name="DigestValue" type="ds:DigestValueType"/>
+<simpleType name="DigestValueType">
+  <restriction base="base64Binary"/>
+</simpleType>
+
+<!-- End SignedInfo -->
+
+<!-- Start KeyInfo -->
+
+<element name="KeyInfo" type="ds:KeyInfoType"/> 
+<complexType name="KeyInfoType" mixed="true">
+  <choice maxOccurs="unbounded">     
+    <element ref="ds:KeyName"/> 
+    <element ref="ds:KeyValue"/> 
+    <element ref="ds:RetrievalMethod"/> 
+    <element ref="ds:X509Data"/> 
+    <element ref="ds:PGPData"/> 
+    <element ref="ds:SPKIData"/>
+    <element ref="ds:MgmtData"/>
+    <any processContents="lax" namespace="##other"/>
+    <!-- (1,1) elements from (0,unbounded) namespaces -->
+  </choice>
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+  <element name="KeyName" type="string"/>
+  <element name="MgmtData" type="string"/>
+
+  <element name="KeyValue" type="ds:KeyValueType"/> 
+  <complexType name="KeyValueType" mixed="true">
+   <choice>
+     <element ref="ds:DSAKeyValue"/>
+     <element ref="ds:RSAKeyValue"/>
+     <any namespace="##other" processContents="lax"/>
+   </choice>
+  </complexType>
+
+  <element name="RetrievalMethod" type="ds:RetrievalMethodType"/> 
+  <complexType name="RetrievalMethodType">
+    <sequence>
+      <element ref="ds:Transforms" minOccurs="0"/> 
+    </sequence>  
+    <attribute name="URI" type="anyURI"/>
+    <attribute name="Type" type="anyURI" use="optional"/>
+  </complexType>
+
+<!-- Start X509Data -->
+
+<element name="X509Data" type="ds:X509DataType"/> 
+<complexType name="X509DataType">
+  <sequence maxOccurs="unbounded">
+    <choice>
+      <element name="X509IssuerSerial" type="ds:X509IssuerSerialType"/>
+      <element name="X509SKI" type="base64Binary"/>
+      <element name="X509SubjectName" type="string"/>
+      <element name="X509Certificate" type="base64Binary"/>
+      <element name="X509CRL" type="base64Binary"/>
+      <any namespace="##other" processContents="lax"/>
+    </choice>
+  </sequence>
+</complexType>
+
+<complexType name="X509IssuerSerialType"> 
+  <sequence> 
+    <element name="X509IssuerName" type="string"/> 
+    <element name="X509SerialNumber" type="integer"/> 
+  </sequence>
+</complexType>
+
+<!-- End X509Data -->
+
+<!-- Begin PGPData -->
+
+<element name="PGPData" type="ds:PGPDataType"/> 
+<complexType name="PGPDataType"> 
+  <choice>
+    <sequence>
+      <element name="PGPKeyID" type="base64Binary"/> 
+      <element name="PGPKeyPacket" type="base64Binary" minOccurs="0"/> 
+      <any namespace="##other" processContents="lax" minOccurs="0"
+       maxOccurs="unbounded"/>
+    </sequence>
+    <sequence>
+      <element name="PGPKeyPacket" type="base64Binary"/> 
+      <any namespace="##other" processContents="lax" minOccurs="0"
+       maxOccurs="unbounded"/>
+    </sequence>
+  </choice>
+</complexType>
+
+<!-- End PGPData -->
+
+<!-- Begin SPKIData -->
+
+<element name="SPKIData" type="ds:SPKIDataType"/> 
+<complexType name="SPKIDataType">
+  <sequence maxOccurs="unbounded">
+    <element name="SPKISexp" type="base64Binary"/>
+    <any namespace="##other" processContents="lax" minOccurs="0"/>
+  </sequence>
+</complexType> 
+
+<!-- End SPKIData -->
+
+<!-- End KeyInfo -->
+
+<!-- Start Object (Manifest, SignatureProperty) -->
+
+<element name="Object" type="ds:ObjectType"/> 
+<complexType name="ObjectType" mixed="true">
+  <sequence minOccurs="0" maxOccurs="unbounded">
+    <any namespace="##any" processContents="lax"/>
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+  <attribute name="MimeType" type="string" use="optional"/> <!-- add a grep facet -->
+  <attribute name="Encoding" type="anyURI" use="optional"/> 
+</complexType>
+
+<element name="Manifest" type="ds:ManifestType"/> 
+<complexType name="ManifestType">
+  <sequence>
+    <element ref="ds:Reference" maxOccurs="unbounded"/> 
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+<element name="SignatureProperties" type="ds:SignaturePropertiesType"/> 
+<complexType name="SignaturePropertiesType">
+  <sequence>
+    <element ref="ds:SignatureProperty" maxOccurs="unbounded"/> 
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+   <element name="SignatureProperty" type="ds:SignaturePropertyType"/> 
+   <complexType name="SignaturePropertyType" mixed="true">
+     <choice maxOccurs="unbounded">
+       <any namespace="##other" processContents="lax"/>
+       <!-- (1,1) elements from (1,unbounded) namespaces -->
+     </choice>
+     <attribute name="Target" type="anyURI" use="required"/> 
+     <attribute name="Id" type="ID" use="optional"/> 
+   </complexType>
+
+<!-- End Object (Manifest, SignatureProperty) -->
+
+<!-- Start Algorithm Parameters -->
+
+<simpleType name="HMACOutputLengthType">
+  <restriction base="integer"/>
+</simpleType>
+
+<!-- Start KeyValue Element-types -->
+
+<element name="DSAKeyValue" type="ds:DSAKeyValueType"/>
+<complexType name="DSAKeyValueType">
+  <sequence>
+    <sequence minOccurs="0">
+      <element name="P" type="ds:CryptoBinary"/>
+      <element name="Q" type="ds:CryptoBinary"/>
+    </sequence>
+    <element name="G" type="ds:CryptoBinary" minOccurs="0"/>
+    <element name="Y" type="ds:CryptoBinary"/>
+    <element name="J" type="ds:CryptoBinary" minOccurs="0"/>
+    <sequence minOccurs="0">
+      <element name="Seed" type="ds:CryptoBinary"/>
+      <element name="PgenCounter" type="ds:CryptoBinary"/>
+    </sequence>
+  </sequence>
+</complexType>
+
+<element name="RSAKeyValue" type="ds:RSAKeyValueType"/>
+<complexType name="RSAKeyValueType">
+  <sequence>
+    <element name="Modulus" type="ds:CryptoBinary"/> 
+    <element name="Exponent" type="ds:CryptoBinary"/> 
+  </sequence>
+</complexType> 
+
+<!-- End KeyValue Element-types -->
+
+<!-- End Signature -->
+
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java
new file mode 100644
index 0000000000000..2bd810a0a9a93
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.Security;
+
+import java.nio.file.Path;
+
+public class LocalStateIdentityProviderPlugin extends LocalStateCompositeXPackPlugin {
+
+    public LocalStateIdentityProviderPlugin(Settings settings, Path configPath) throws Exception {
+        super(settings, configPath);
+        LocalStateIdentityProviderPlugin thisVar = this;
+        plugins.add(new Security(settings, configPath) {
+            @Override
+            protected SSLService getSslService() {
+                return thisVar.getSslService();
+            }
+
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+        });
+
+        plugins.add(new IdentityProviderPlugin() {
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+        });
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java
new file mode 100644
index 0000000000000..0c1c53ac3e610
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DeleteSamlServiceProviderRequestTests extends IdpSamlTestCase {
+
+    public void testSerialization() throws IOException {
+        final DeleteSamlServiceProviderRequest request = new DeleteSamlServiceProviderRequest(randomAlphaOfLengthBetween(1, 100),
+            randomFrom(WriteRequest.RefreshPolicy.values()));
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final DeleteSamlServiceProviderRequest read = copyWriteable(request, new NamedWriteableRegistry(List.of()),
+            DeleteSamlServiceProviderRequest::new, version);
+        MatcherAssert.assertThat("Serialized request with version [" + version + "] does not match original object",
+            read, equalTo(request));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java
new file mode 100644
index 0000000000000..785df0a8a58ae
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class DeleteSamlServiceProviderResponseTests extends IdpSamlTestCase {
+
+    public void testToXContent() throws Exception {
+        final String docId = randomAlphaOfLength(24);
+        final long seqNo = randomNonNegativeLong();
+        final long primaryTerm = randomNonNegativeLong();
+        final String entityId = randomAlphaOfLengthBetween(4, 128);
+        final DeleteSamlServiceProviderResponse response = new DeleteSamlServiceProviderResponse(docId, seqNo, primaryTerm, entityId);
+        final String xContent = Strings.toString(response);
+        final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), xContent, randomBoolean());
+        assertThat(map.keySet(), containsInAnyOrder("document", "service_provider"));
+
+        assertThat(map.get("document"), instanceOf(Map.class));
+        assertThat(((Map<?, ?>) map.get("document")).keySet(), containsInAnyOrder("_id", "_seq_no", "_primary_term"));
+
+        assertThat(map.get("service_provider"), instanceOf(Map.class));
+        assertThat(((Map<?, ?>) map.get("service_provider")).keySet(), containsInAnyOrder("entity_id"));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java
new file mode 100644
index 0000000000000..f81eaf081019c
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestMatchers;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndexTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
+
+public class PutSamlServiceProviderRequestTests extends ESTestCase {
+
+    public void testValidateSuccessfully() {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, randomFrom(RefreshPolicy.values()));
+        assertThat(request.validate(), nullValue());
+    }
+
+    public void testValidateAcs() {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        doc.acs = "this is not a URL";
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, randomFrom(RefreshPolicy.values()));
+        final ActionRequestValidationException validationException = request.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), hasSize(1));
+        assertThat(validationException.validationErrors().get(0), containsString("[acs]"));
+        assertThat(validationException.validationErrors().get(0), containsString("valid URL"));
+    }
+
+    public void testValidateRequiredFields() {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        doc.name = null;
+        doc.entityId = null;
+        doc.acs = null;
+        doc.created = null;
+        doc.lastModified = null;
+        doc.attributeNames.principal = null;
+        doc.privileges.resource = null;
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, randomFrom(RefreshPolicy.values()));
+        final ActionRequestValidationException validationException = request.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), hasSize(7));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[name]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[entity_id]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[acs]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[created]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[last_modified]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[attributes.principal]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[privileges.resource]")));
+    }
+
+    public void testSerialization() throws IOException {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, RefreshPolicy.NONE);
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final PutSamlServiceProviderRequest read = copyWriteable(request, new NamedWriteableRegistry(List.of()),
+            PutSamlServiceProviderRequest::new, version);
+        MatcherAssert.assertThat("Serialized request with version [" + version + "] does not match original object",
+            read, equalTo(request));
+    }
+
+    public void testParseRequestBodySuccessfully() throws Exception {
+        final Map<String, Object> fields = new HashMap<>();
+        fields.put("name", randomAlphaOfLengthBetween(3, 30));
+        fields.put("acs", "https://www." + randomAlphaOfLengthBetween(3, 30) + ".fake/saml/acs");
+        fields.put("enabled", randomBoolean());
+        fields.put("attributes", Map.of(
+            "principal", "urn:oid:0.1." + randomLongBetween(1, 1000),
+            "email", "urn:oid:0.2." + randomLongBetween(1001, 2000),
+            "name", "urn:oid:0.3." + randomLongBetween(2001, 3000),
+            "roles", "urn:oid:0.4." + randomLongBetween(3001, 4000)
+        ));
+        fields.put("privileges", Map.of(
+            "resource", "ece:deployment:" + randomLongBetween(1_000_000, 999_999_999),
+            "roles", Map.of("role_name", "role:" + randomAlphaOfLengthBetween(4, 8))
+        ));
+        fields.put("certificates", Map.of());
+        final String entityId = "https://www." + randomAlphaOfLengthBetween(5, 12) + ".app/";
+        final PutSamlServiceProviderRequest request = parseRequest(entityId, fields);
+        assertThat(request.getDocument().docId, nullValue());
+        assertThat(request.getDocument().entityId, equalTo(entityId));
+        assertThat(request.getDocument().acs, equalTo(fields.get("acs")));
+        assertThat(request.getDocument().enabled, equalTo(fields.get("enabled")));
+        assertThat(request.getDocument().privileges.resource, notNullValue());
+        assertThat(request.getDocument().privileges.roleActions, aMapWithSize(1));
+        assertThat(request.getDocument().privileges.roleActions.keySet(), contains("role_name"));
+        assertThat(request.getDocument().attributeNames.principal, startsWith("urn:oid:0.1"));
+        assertThat(request.getDocument().attributeNames.email, startsWith("urn:oid:0.2"));
+        assertThat(request.getDocument().attributeNames.name, startsWith("urn:oid:0.3"));
+        assertThat(request.getDocument().attributeNames.roles, startsWith("urn:oid:0.4"));
+        assertThat(request.getDocument().certificates.serviceProviderSigning, emptyIterable());
+        assertThat(request.getDocument().certificates.identityProviderSigning, emptyIterable());
+        assertThat(request.getDocument().certificates.identityProviderMetadataSigning, emptyIterable());
+
+        assertThat(request.validate(), nullValue());
+    }
+
+    public void testParseRequestBodyFailsIfTimestampsAreIncluded() throws Exception {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        final Map<String, Object> fields = convertToMap(XContentType.JSON.xContent(), Strings.toString(doc), randomBoolean());
+
+        fields.remove("entity_id");
+        fields.remove("created");
+        fields.remove("last_modified");
+
+        final String field = randomBoolean() ? "created" : "last_modified";
+        fields.put(field, System.currentTimeMillis() + randomLongBetween(-100_000, +100_000));
+
+        ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> parseRequest(doc.entityId, fields));
+        assertThat(exception, TestMatchers.throwableWithMessage("Field [" + field + "] may not be specified in a request"));
+    }
+
+    private PutSamlServiceProviderRequest parseRequest(String entityId, Map<String, Object> fields) throws IOException {
+        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+            builder.map(fields);
+            try (XContentParser parser = createParser(shuffleXContent(builder))) {
+                return PutSamlServiceProviderRequest.fromXContent(entityId, randomFrom(RefreshPolicy.values()), parser);
+            }
+        }
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
new file mode 100644
index 0000000000000..925fc04695a38
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
@@ -0,0 +1,349 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.test.IdentityProviderIntegTestCase;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.saml.common.SAMLObject;
+import org.opensaml.saml.saml2.core.AuthnRequest;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameIDPolicy;
+import org.opensaml.security.SecurityException;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.crypto.XMLSigningUtil;
+import org.opensaml.xmlsec.signature.support.SignatureConstants;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.joda.time.DateTime.now;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numClientNodes = 0, numDataNodes = 0)
+public class SamlIdentityProviderTests extends IdentityProviderIntegTestCase {
+
+    private final SamlFactory samlFactory = new SamlFactory();
+
+    public void testIdpInitiatedSso() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+
+        // User login a.k.a exchange the user credentials for an API Key
+        final String apiKeyCredentials = getApiKeyFromCredentials(SAMPLE_IDPUSER_NAME,
+            new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray()));
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request request = new Request("POST", "/_idp/saml/init");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+                new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+            .addHeader("es-secondary-authorization", "ApiKey " + apiKeyCredentials)
+            .build());
+        request.setJsonEntity("{ \"entity_id\": \"" + entityId + "\"}");
+        Response initResponse = getRestClient().performRequest(request);
+        ObjectPath objectPath = ObjectPath.createFromResponse(initResponse);
+        assertThat(objectPath.evaluate("post_url").toString(), equalTo(acsUrl));
+        final String body = objectPath.evaluate("saml_response").toString();
+        assertThat(body, containsString("Destination=\"" + acsUrl + "\""));
+        assertThat(body, containsString("<saml2:Audience>" + entityId + "</saml2:Audience>"));
+        assertThat(body, containsString("<saml2:NameID Format=\"" + TRANSIENT + "\">"));
+        Map<String, String> serviceProvider = objectPath.evaluate("service_provider");
+        assertThat(serviceProvider, hasKey("entity_id"));
+        assertThat(serviceProvider.get("entity_id"), equalTo(entityId));
+        assertContainsAttributeWithValue(body, "principal", SAMPLE_IDPUSER_NAME);
+    }
+
+    public void testIdPInitiatedSsoFailsForUnknownSP() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // User login a.k.a exchange the user credentials for an API Key
+        final String apiKeyCredentials = getApiKeyFromCredentials(SAMPLE_IDPUSER_NAME,
+            new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray()));
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request request = new Request("POST", "/_idp/saml/init");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+                new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+            .addHeader("es-secondary-authorization", "ApiKey " + apiKeyCredentials)
+            .build());
+        request.setJsonEntity("{ \"entity_id\": \"" + entityId + randomAlphaOfLength(3) + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("is not registered with this Identity Provider"));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+    }
+
+    public void testIdPInitiatedSsoFailsWithoutSecondaryAuthentication() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request request = new Request("POST", "/_idp/saml/init");
+        request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        request.setJsonEntity("{ \"entity_id\": \"" + entityId + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("Request is missing secondary authentication"));
+    }
+
+    public void testSpInitiatedSso() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Validate incoming authentication request
+        Request validateRequest = new Request("POST", "/_idp/saml/validate");
+        validateRequest.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final String nameIdFormat = TRANSIENT;
+        final String relayString = randomBoolean() ? randomAlphaOfLength(8) : null;
+        final boolean forceAuthn = true;
+        final AuthnRequest authnRequest = buildAuthnRequest(entityId, new URL(acsUrl),
+            new URL("https://idp.org/sso/redirect"), nameIdFormat, forceAuthn);
+        final String query = getQueryString(authnRequest, relayString, false, null);
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + query + "\"}");
+        Response validateResponse = getRestClient().performRequest(validateRequest);
+        ObjectPath validateResponseObject = ObjectPath.createFromResponse(validateResponse);
+        Map<String, String> serviceProvider = validateResponseObject.evaluate("service_provider");
+        assertThat(serviceProvider, hasKey("entity_id"));
+        assertThat(serviceProvider.get("entity_id"), equalTo(entityId));
+        assertThat(validateResponseObject.evaluate("force_authn"), equalTo(forceAuthn));
+        Map<String, String> authnState = validateResponseObject.evaluate("authn_state");
+        assertThat(authnState, hasKey("nameid_format"));
+        assertThat(authnState.get("nameid_format"), equalTo(nameIdFormat));
+        assertThat(authnState, hasKey("entity_id"));
+        assertThat(authnState.get("entity_id"), equalTo(entityId));
+        assertThat(authnState, hasKey("acs_url"));
+        assertThat(authnState.get("acs_url"), equalTo(acsUrl));
+        assertThat(authnState, hasKey("authn_request_id"));
+        final String expectedInResponeTo = authnState.get("authn_request_id");
+
+        // User login a.k.a exchange the user credentials for an API Key
+        final String apiKeyCredentials = getApiKeyFromCredentials(SAMPLE_IDPUSER_NAME,
+            new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray()));
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request initRequest = new Request("POST", "/_idp/saml/init");
+        initRequest.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+                new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+            .addHeader("es-secondary-authorization", "ApiKey " + apiKeyCredentials)
+            .build());
+        XContentBuilder authnStateBuilder = jsonBuilder();
+        authnStateBuilder.map(authnState);
+        initRequest.setJsonEntity("{ \"entity_id\":\"" + entityId + "\", \"authn_state\":" + Strings.toString(authnStateBuilder) + "}");
+        Response initResponse = getRestClient().performRequest(initRequest);
+        ObjectPath initResponseObject = ObjectPath.createFromResponse(initResponse);
+        assertThat(initResponseObject.evaluate("post_url").toString(), equalTo(acsUrl));
+        final String body = initResponseObject.evaluate("saml_response").toString();
+        assertThat(body, containsString("<saml2p:StatusCode Value=\"urn:oasis:names:tc:SAML:2.0:status:Success\"/>"));
+        assertThat(body, containsString("Destination=\"" + acsUrl + "\""));
+        assertThat(body, containsString("<saml2:Audience>" + entityId + "</saml2:Audience>"));
+        assertThat(body, containsString("<saml2:NameID Format=\"" + nameIdFormat + "\">"));
+        assertThat(body, containsString("InResponseTo=\"" + expectedInResponeTo + "\""));
+        Map<String, String> sp = initResponseObject.evaluate("service_provider");
+        assertThat(sp, hasKey("entity_id"));
+        assertThat(sp.get("entity_id"), equalTo(entityId));
+        assertContainsAttributeWithValue(body, "principal", SAMPLE_IDPUSER_NAME);
+    }
+
+    public void testSpInitiatedSsoFailsForUnknownSp() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Validate incoming authentication request
+        Request validateRequest = new Request("POST", "/_idp/saml/validate");
+        validateRequest.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final String nameIdFormat = TRANSIENT;
+        final String relayString = null;
+        final boolean forceAuthn = randomBoolean();
+        final AuthnRequest authnRequest = buildAuthnRequest(entityId + randomAlphaOfLength(4), new URL(acsUrl),
+            new URL("https://idp.org/sso/redirect"), nameIdFormat, forceAuthn);
+        final String query = getQueryString(authnRequest, relayString, false, null);
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + query + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(validateRequest));
+        assertThat(e.getMessage(), containsString("is not registered with this Identity Provider"));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+    }
+
+    public void testSpInitiatedSsoFailsForMalformedRequest() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Validate incoming authentication request
+        Request validateRequest = new Request("POST", "/_idp/saml/validate");
+        validateRequest.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final String nameIdFormat = TRANSIENT;
+        final String relayString = null;
+        final boolean forceAuthn = randomBoolean();
+        final AuthnRequest authnRequest = buildAuthnRequest(entityId + randomAlphaOfLength(4), new URL(acsUrl),
+            new URL("https://idp.org/sso/redirect"), nameIdFormat, forceAuthn);
+        final String query = getQueryString(authnRequest, relayString, false, null);
+        // Skip http parameter name
+        final String queryWithoutParam = query.substring(12);
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + queryWithoutParam + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(validateRequest));
+        assertThat(e.getMessage(), containsString("does not contain a SAMLRequest parameter"));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+        // arbitrarily trim the request
+        final String malformedRequestQuery = query.substring(0, query.length() - randomIntBetween(10, 15));
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + malformedRequestQuery + "\"}");
+        ResponseException e1 = expectThrows(ResponseException.class, () -> getRestClient().performRequest(validateRequest));
+        assertThat(e1.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+    }
+
+    private void registerServiceProvider(String entityId, String acsUrl) throws Exception {
+        Map<String, Object> spFields = new HashMap<>();
+        spFields.put(SamlServiceProviderDocument.Fields.ACS.getPreferredName(), acsUrl);
+        spFields.put(SamlServiceProviderDocument.Fields.ENTITY_ID.getPreferredName(), entityId);
+        spFields.put(SamlServiceProviderDocument.Fields.NAME_ID.getPreferredName(), TRANSIENT);
+        spFields.put(SamlServiceProviderDocument.Fields.NAME.getPreferredName(), "Dummy SP");
+        spFields.put("attributes", Map.of(
+            "principal", "https://saml.elasticsearch.org/attributes/principal"));
+        spFields.put("privileges", Map.of(
+            "resource", entityId,
+            "roles", Map.of("superuser", "sso:superuser")
+        ));
+        Request request =
+            new Request("PUT", "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue());
+        request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.map(spFields);
+        request.setJsonEntity(Strings.toString(builder));
+        Response registerResponse = getRestClient().performRequest(request);
+        assertThat(registerResponse.getStatusLine().getStatusCode(), equalTo(200));
+        ObjectPath registerResponseObject = ObjectPath.createFromResponse(registerResponse);
+        Map<String, Object> document = registerResponseObject.evaluate("document");
+        assertThat(document, hasKey("_created"));
+        assertThat(document.get("_created"), equalTo(true));
+        Map<String, Object> serviceProvider = registerResponseObject.evaluate("service_provider");
+        assertThat(serviceProvider, hasKey("entity_id"));
+        assertThat(serviceProvider.get("entity_id"), equalTo(entityId));
+        assertThat(serviceProvider, hasKey("enabled"));
+        assertThat(serviceProvider.get("enabled"), equalTo(true));
+    }
+
+    private String getApiKeyFromCredentials(String username, SecureString password) {
+        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(username, password)));
+        final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client)
+            .setName("test key")
+            .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
+            .get();
+        assertNotNull(response);
+        return Base64.getEncoder().encodeToString(
+            (response.getId() + ":" + response.getKey().toString()).getBytes(StandardCharsets.UTF_8));
+    }
+
+    private AuthnRequest buildAuthnRequest(String entityId, URL acs, URL destination, String nameIdFormat, boolean forceAuthn) {
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        final NameIDPolicy nameIDPolicy = samlFactory.buildObject(NameIDPolicy.class, NameIDPolicy.DEFAULT_ELEMENT_NAME);
+        nameIDPolicy.setFormat(nameIdFormat);
+        final AuthnRequest authnRequest = samlFactory.buildObject(AuthnRequest.class, AuthnRequest.DEFAULT_ELEMENT_NAME);
+        authnRequest.setID(samlFactory.secureIdentifier());
+        authnRequest.setIssuer(issuer);
+        authnRequest.setIssueInstant(now());
+        authnRequest.setAssertionConsumerServiceURL(acs.toString());
+        authnRequest.setDestination(destination.toString());
+        authnRequest.setNameIDPolicy(nameIDPolicy);
+        authnRequest.setForceAuthn(forceAuthn);
+        return authnRequest;
+    }
+
+    private String getQueryString(AuthnRequest authnRequest, String relayState, boolean sign, @Nullable X509Credential credential) {
+        try {
+            final String request = deflateAndBase64Encode(authnRequest);
+            String queryParam = "SAMLRequest=" + urlEncode(request);
+            if (relayState != null) {
+                queryParam += "&RelayState=" + urlEncode(relayState);
+            }
+            if (sign) {
+                final String algo = SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256;
+                queryParam += "&SigAlg=" + urlEncode(algo);
+                final byte[] sig = sign(queryParam, algo, credential);
+                queryParam += "&Signature=" + urlEncode(base64Encode(sig));
+            }
+            return queryParam;
+        } catch (Exception e) {
+            throw new ElasticsearchException("Cannot construct SAML redirect", e);
+        }
+    }
+
+    private String base64Encode(byte[] bytes) {
+        return Base64.getEncoder().encodeToString(bytes);
+    }
+
+    private static String urlEncode(String param) throws UnsupportedEncodingException {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8.name());
+    }
+
+    private String deflateAndBase64Encode(SAMLObject message) throws Exception {
+        Deflater deflater = new Deflater(Deflater.DEFLATED, true);
+        try (ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+             DeflaterOutputStream deflaterStream = new DeflaterOutputStream(bytesOut, deflater)) {
+            String messageStr = samlFactory.toString(XMLObjectSupport.marshall(message), false);
+            deflaterStream.write(messageStr.getBytes(StandardCharsets.UTF_8));
+            deflaterStream.finish();
+            return base64Encode(bytesOut.toByteArray());
+        }
+    }
+
+    private byte[] sign(String text, String algo, X509Credential credential) throws SecurityException {
+        return sign(text.getBytes(StandardCharsets.UTF_8), algo, credential);
+    }
+
+    private byte[] sign(byte[] content, String algo, X509Credential credential) throws SecurityException {
+        return XMLSigningUtil.signWithURI(credential, algo, content);
+    }
+
+    private void assertContainsAttributeWithValue(String message, String attribute, String value) {
+        assertThat(message, containsString("<saml2:Attribute FriendlyName=\"" + attribute + "\" Name=\"https://saml.elasticsearch" +
+            ".org/attributes/" + attribute + "\" NameFormat=\"urn:oasis:names:tc:SAML:2.0:attrname-format:uri\"><saml2:AttributeValue " +
+            "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"xsd:string\">" + value + "</saml2:AttributeValue></saml2" +
+            ":Attribute>"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java
new file mode 100644
index 0000000000000..4969067eb9055
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class SamlInitiateSingleSignOnRequestTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://kibana_url");
+        final BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+
+        final SamlInitiateSingleSignOnRequest request1 = new SamlInitiateSingleSignOnRequest(out.bytes().streamInput());
+        assertThat(request1.getSpEntityId(), equalTo(request.getSpEntityId()));
+        final ActionRequestValidationException validationException = request1.validate();
+        assertNull(validationException);
+    }
+
+    public void testValidation() {
+
+        final SamlInitiateSingleSignOnRequest request1 = new SamlInitiateSingleSignOnRequest();
+        final ActionRequestValidationException validationException = request1.validate();
+        assertNotNull(validationException);
+        assertThat(validationException.validationErrors().size(), equalTo(1));
+        assertThat(validationException.validationErrors().get(0), containsString("entity_id is missing"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java
new file mode 100644
index 0000000000000..7e7522103bee3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class SamlValidateAuthnRequestRequestTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        final SamlValidateAuthnRequestRequest request = new SamlValidateAuthnRequestRequest();
+        request.setQueryString("?SAMLRequest=x&RelayState=y&SigAlg=z&Signature=sig");
+        final BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+
+        final SamlValidateAuthnRequestRequest request1 = new SamlValidateAuthnRequestRequest(out.bytes().streamInput());
+        assertThat(request.getQueryString(), equalTo(request1.getQueryString()));
+        final ActionRequestValidationException exception = request1.validate();
+        assertNull(exception);
+    }
+
+    public void testValidation() {
+        final SamlValidateAuthnRequestRequest request = new SamlValidateAuthnRequestRequest();
+        final ActionRequestValidationException exception = request.validate();
+        assertNotNull(exception);
+        assertThat(exception.validationErrors().size(), equalTo(1));
+        assertThat(exception.validationErrors().get(0), containsString("Authentication request query string must be provided"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java
new file mode 100644
index 0000000000000..dccb848e78210
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndexTests;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.saml2.core.NameIDType.EMAIL;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class TransportPutSamlServiceProviderActionTests extends ESTestCase {
+
+    private SamlServiceProviderIndex index;
+    private TransportPutSamlServiceProviderAction action;
+    private SamlIdentityProvider idp;
+    private Instant now;
+
+    @Before
+    public void setupMocks() {
+        index = mock(SamlServiceProviderIndex.class);
+        idp = mock(SamlIdentityProvider.class);
+        when(idp.getAllowedNameIdFormats()).thenReturn(Set.of(TRANSIENT));
+
+        now = Instant.ofEpochMilli(System.currentTimeMillis() + randomLongBetween(-500_000, 500_000));
+        final Clock clock = Clock.fixed(now, randomZone());
+        action = new TransportPutSamlServiceProviderAction(
+            mock(TransportService.class), mock(ActionFilters.class), index, idp, clock);
+    }
+
+    public void testRegisterNewServiceProvider() throws Exception {
+        final SamlServiceProviderDocument document = SamlServiceProviderIndexTests.randomDocument();
+
+        mockExistingDocuments(document.entityId, Set.of());
+
+        final AtomicReference<DocWriteResponse> writeResponse = mockWriteResponse(document, true);
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(document, randomFrom(RefreshPolicy.values()));
+        final PlainActionFuture<PutSamlServiceProviderResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        final PutSamlServiceProviderResponse response = future.actionGet();
+        assertThat(response, notNullValue());
+        assertThat(response.getDocId(), equalTo(writeResponse.get().getId()));
+        assertThat(response.getEntityId(), equalTo(document.entityId));
+        assertThat(response.isCreated(), equalTo(true));
+        assertThat(response.isEnabled(), equalTo(document.enabled));
+        assertThat(response.getPrimaryTerm(), equalTo(writeResponse.get().getPrimaryTerm()));
+        assertThat(response.getSeqNo(), equalTo(writeResponse.get().getSeqNo()));
+
+        assertThat(document.created, equalTo(now));
+        assertThat(document.lastModified, equalTo(now));
+    }
+
+    public void testUpdateExistingServiceProvider() throws Exception {
+        final SamlServiceProviderDocument document = SamlServiceProviderIndexTests.randomDocument();
+
+        final SamlServiceProviderDocument existingDocument = SamlServiceProviderIndexTests.randomDocument();
+        existingDocument.entityId = document.entityId;
+        existingDocument.docId = randomAlphaOfLength(42);
+        mockExistingDocuments(document.entityId, Set.of(existingDocument));
+
+        final AtomicReference<DocWriteResponse> writeResponse = mockWriteResponse(document, false);
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(document, randomFrom(RefreshPolicy.values()));
+        final PlainActionFuture<PutSamlServiceProviderResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        final PutSamlServiceProviderResponse response = future.actionGet();
+        assertThat(response, notNullValue());
+        assertThat(response.getDocId(), equalTo(existingDocument.docId));
+        assertThat(response.getDocId(), equalTo(writeResponse.get().getId()));
+        assertThat(response.getEntityId(), equalTo(document.entityId));
+        assertThat(response.isCreated(), equalTo(false));
+        assertThat(response.isEnabled(), equalTo(document.enabled));
+        assertThat(response.getPrimaryTerm(), equalTo(writeResponse.get().getPrimaryTerm()));
+        assertThat(response.getSeqNo(), equalTo(writeResponse.get().getSeqNo()));
+
+        assertThat(document.created, equalTo(existingDocument.created));
+        assertThat(document.lastModified, equalTo(now));
+    }
+
+    public void testUnsupportedNameIDFormat() throws Exception {
+        final SamlServiceProviderDocument document = SamlServiceProviderIndexTests.randomDocument();
+        final String invalidFormat = randomFrom(PERSISTENT, EMAIL, randomAlphaOfLength(12));
+        document.setNameIdFormat(invalidFormat);
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(document, randomFrom(RefreshPolicy.values()));
+        final PlainActionFuture<PutSamlServiceProviderResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, future::actionGet);
+        assertThat(e.getMessage(), containsString("NameID format [" + invalidFormat + "] is not supported"));
+    }
+
+    public AtomicReference<DocWriteResponse> mockWriteResponse(SamlServiceProviderDocument document, boolean created) {
+        final AtomicReference<DocWriteResponse> writeResponse = new AtomicReference<>();
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(4));
+
+            SamlServiceProviderDocument doc = (SamlServiceProviderDocument) args[0];
+            assertThat(doc.getDocId(), notNullValue());
+            assertThat(doc, sameInstance(document));
+
+            final DocWriteResponse docWriteResponse = new IndexResponse(
+                new ShardId(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLength(24), randomIntBetween(1, 10)),
+                doc.docId, randomLong(), randomLong(), randomLong(), created);
+            writeResponse.set(docWriteResponse);
+
+            ActionListener listener = (ActionListener) args[args.length - 1];
+            listener.onResponse(docWriteResponse);
+
+            return null;
+        }).when(index).writeDocument(any(SamlServiceProviderDocument.class), any(DocWriteRequest.OpType.class),
+            any(WriteRequest.RefreshPolicy.class), any(ActionListener.class));
+
+        return writeResponse;
+    }
+
+    public void mockExistingDocuments(String expectedEntityId, Set<SamlServiceProviderDocument> documents) {
+        final Set<SamlServiceProviderIndex.DocumentSupplier> documentSuppliers = documents.stream()
+            .map(doc -> new SamlServiceProviderIndex.DocumentSupplier(
+                new DocumentVersion(randomAlphaOfLength(24), randomLong(), randomLong()),
+                () -> doc))
+            .collect(Collectors.toUnmodifiableSet());
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(2));
+
+            String entityId = (String) args[0];
+            assertThat(entityId, equalTo(expectedEntityId));
+
+            ActionListener<Set<SamlServiceProviderIndex.DocumentSupplier>> listener = (ActionListener) args[args.length - 1];
+            listener.onResponse(documentSuppliers);
+
+            return null;
+        }).when(index).findByEntityId(anyString(), any(ActionListener.class));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java
new file mode 100644
index 0000000000000..4b83d765b062d
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.support.SecondaryAuthentication;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.elasticsearch.xpack.idp.privileges.UserPrivilegeResolver;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.CloudServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.joda.time.Duration;
+import org.mockito.Mockito;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class TransportSamlInitiateSingleSignOnRequestTests extends IdpSamlTestCase {
+
+    public void testGetResponseForRegisteredSp() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://sp.some.org");
+
+        final PlainActionFuture<SamlInitiateSingleSignOnResponse> future = new PlainActionFuture<>();
+        final TransportSamlInitiateSingleSignOnAction action = setupTransportAction(true);
+        action.doExecute(mock(Task.class), request, future);
+
+        final SamlInitiateSingleSignOnResponse response = future.get();
+        assertThat(response.getEntityId(), equalTo("https://sp.some.org"));
+        assertThat(response.getPostUrl(), equalTo("https://sp.some.org/api/security/v1/saml"));
+        assertThat(response.getSamlResponse(), containsString(TRANSIENT));
+        assertContainsAttributeWithValue(response.getSamlResponse(), "email", "samlenduser@elastic.co");
+        assertContainsAttributeWithValue(response.getSamlResponse(), "name", "Saml Enduser");
+        assertContainsAttributeWithValue(response.getSamlResponse(), "principal", "saml_enduser");
+        assertThat(response.getSamlResponse(), containsString("https://saml.elasticsearch.org/attributes/roles"));
+    }
+
+    public void testGetResponseWithoutSecondaryAuthentication() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://sp.some.org");
+
+        final PlainActionFuture<SamlInitiateSingleSignOnResponse> future = new PlainActionFuture<>();
+        final TransportSamlInitiateSingleSignOnAction action = setupTransportAction(false);
+        action.doExecute(mock(Task.class), request, future);
+
+        Exception e = expectThrows(Exception.class, () -> future.get());
+        assertThat(e.getCause().getMessage(), containsString("Request is missing secondary authentication"));
+    }
+
+    public void testGetResponseForNotRegisteredSp() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://sp2.other.org");
+
+        final PlainActionFuture<SamlInitiateSingleSignOnResponse> future = new PlainActionFuture<>();
+        final TransportSamlInitiateSingleSignOnAction action = setupTransportAction(true);
+        action.doExecute(mock(Task.class), request, future);
+
+        Exception e = expectThrows(Exception.class, () -> future.get());
+        assertThat(e.getCause().getMessage(), containsString("https://sp2.other.org"));
+        assertThat(e.getCause().getMessage(), containsString("is not registered with this Identity Provider"));
+    }
+
+    private TransportSamlInitiateSingleSignOnAction setupTransportAction(boolean withSecondaryAuth) throws Exception {
+        final Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.idp.enabled", true)
+            .put("xpack.idp.entity_id", "https://idp.cloud.elastic.co")
+            .put("xpack.idp.sso_endpoint.redirect", "https://idp.cloud.elastic.co/saml/init")
+            .put("xpack.idp.organization.url", "https://cloud.elastic.co")
+            .put("xpack.idp.organization.name", "Cloud Elastic")
+            .put("xpack.idp.contact.email", "some@other.org")
+            .build();
+        final ThreadContext threadContext = new ThreadContext(settings);
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        final SecurityContext securityContext = new SecurityContext(settings, threadContext);
+        final TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
+        final ActionFilters actionFilters = mock(ActionFilters.class);
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+        new Authentication(new User("saml_service_account", "saml_service_role"),
+            new Authentication.RealmRef("default_native", "native", "node_name"),
+            new Authentication.RealmRef("default_native", "native", "node_name"))
+            .writeToContext(threadContext);
+        if (withSecondaryAuth) {
+            new SecondaryAuthentication(securityContext,
+                new Authentication(
+                    new User("saml_enduser", new String[]{"saml_enduser_role"}, "Saml Enduser", "samlenduser@elastic.co",
+                        new HashMap<>(), true),
+                    new Authentication.RealmRef("_es_api_key", "_es_api_key", "node_name"),
+                    new Authentication.RealmRef("_es_api_key", "_es_api_key", "node_name")))
+                .writeToContext(threadContext);
+        }
+
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final CloudServiceProvider serviceProvider = new CloudServiceProvider("https://sp.some.org",
+            "test sp",
+            true,
+            new URL("https://sp.some.org/api/security/v1/saml"),
+            TRANSIENT,
+            Duration.standardMinutes(5),
+            null,
+            new SamlServiceProvider.AttributeNames(
+                "https://saml.elasticsearch.org/attributes/principal",
+                "https://saml.elasticsearch.org/attributes/name",
+                "https://saml.elasticsearch.org/attributes/email",
+                "https://saml.elasticsearch.org/attributes/roles"),
+            null, false, false);
+        mockRegisteredServiceProvider(resolver, "https://sp.some.org", serviceProvider);
+        mockRegisteredServiceProvider(resolver, "https://sp2.other.org", null);
+        final ServiceProviderDefaults defaults = new ServiceProviderDefaults(
+            "elastic-cloud", TRANSIENT, Duration.standardMinutes(15));
+        final X509Credential signingCredential = readCredentials("RSA", randomFrom(1024, 2048, 4096));
+        final SamlIdentityProvider idp = SamlIdentityProvider
+            .builder(resolver)
+            .fromSettings(env)
+            .signingCredential(signingCredential)
+            .serviceProviderDefaults(defaults)
+            .build();
+        final SamlFactory factory = new SamlFactory();
+        final UserPrivilegeResolver privilegeResolver = Mockito.mock(UserPrivilegeResolver.class);
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(2));
+            ActionListener<UserPrivilegeResolver.UserPrivileges> listener
+                = (ActionListener<UserPrivilegeResolver.UserPrivileges>) args[args.length - 1];
+            final UserPrivilegeResolver.UserPrivileges privileges = new UserPrivilegeResolver.UserPrivileges(
+                "saml_enduser", true, Set.of(generateRandomStringArray(5, 8, false, false))
+            );
+            listener.onResponse(privileges);
+            return null;
+        }).when(privilegeResolver).resolve(any(ServiceProviderPrivileges.class), any(ActionListener.class));
+        return new TransportSamlInitiateSingleSignOnAction(transportService, actionFilters, securityContext,
+            idp, factory, privilegeResolver);
+    }
+
+    private void assertContainsAttributeWithValue(String message, String attribute, String value) {
+        assertThat(message, containsString("<saml2:Attribute FriendlyName=\"" + attribute + "\" Name=\"https://saml.elasticsearch" +
+            ".org/attributes/" + attribute + "\" NameFormat=\"urn:oasis:names:tc:SAML:2.0:attrname-format:uri\"><saml2:AttributeValue " +
+            "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"xsd:string\">" + value + "</saml2:AttributeValue></saml2" +
+            ":Attribute>"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java
new file mode 100644
index 0000000000000..802db6f0f3b9b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.privileges;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.same;
+
+public class UserPrivilegeResolverTests extends ESTestCase {
+
+    private Client client;
+    private SecurityContext securityContext;
+    private UserPrivilegeResolver resolver;
+
+    @Before
+    public void setupTest() {
+        client = Mockito.mock(Client.class);
+        securityContext = new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY));
+        resolver = new UserPrivilegeResolver(client, securityContext);
+    }
+
+    public void testResolveZeroAccess() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        setupUser(username);
+        setupHasPrivileges(username, app);
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, "cluster:" + randomLong(),
+            Map.of("viewer", "role:cluster:view", "admin", "role:cluster:admin")), future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(false));
+        assertThat(privileges.roles, emptyIterable());
+    }
+
+    public void testResolveSsoWithNoRoleAccess() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        final String resource = "cluster:" + MessageDigests.toHexString(randomByteArrayOfLength(16));
+        final String viewerAction = "role:cluster:view";
+        final String adminAction = "role:cluster:admin";
+
+        setupUser(username);
+        setupHasPrivileges(username, app, access(resource, viewerAction, false), access(resource, adminAction, false));
+
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, resource, Map.of("viewer", viewerAction, "admin", adminAction)), future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(false));
+        assertThat(privileges.roles, emptyIterable());
+    }
+
+    public void testResolveSsoWithSingleRole() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        final String resource = "cluster:" + MessageDigests.toHexString(randomByteArrayOfLength(16));
+        final String viewerAction = "role:cluster:view";
+        final String adminAction = "role:cluster:admin";
+
+        setupUser(username);
+        setupHasPrivileges(username, app, access(resource, viewerAction, true), access(resource, adminAction, false));
+
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, resource, Map.of("viewer", viewerAction, "admin", adminAction)), future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(true));
+        assertThat(privileges.roles, containsInAnyOrder("viewer"));
+    }
+
+    public void testResolveSsoWithMultipleRoles() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        final String resource = "cluster:" + MessageDigests.toHexString(randomByteArrayOfLength(16));
+        final String viewerAction = "role:cluster:view";
+        final String adminAction = "role:cluster:admin";
+        final String operatorAction = "role:cluster:operator";
+        final String monitorAction = "role:cluster:monitor";
+
+        setupUser(username);
+        setupHasPrivileges(username, app,
+            access(resource, viewerAction, false),
+            access(resource, adminAction, false),
+            access(resource, operatorAction, true),
+            access(resource, monitorAction, true)
+        );
+
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, resource,
+            Map.of("viewer", viewerAction, "admin", adminAction, "operator", operatorAction, "monitor", monitorAction)),
+            future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(true));
+        assertThat(privileges.roles, containsInAnyOrder("operator", "monitor"));
+    }
+
+    private ServiceProviderPrivileges service(String appName, String resource, Map<String, String> roles) {
+        return new ServiceProviderPrivileges(appName, resource, roles);
+    }
+
+    private HasPrivilegesResponse setupHasPrivileges(String username, String appName,
+                                                     Tuple<String, Tuple<String, Boolean>>... resourceActionAccess) {
+        final boolean isCompleteMatch = randomBoolean();
+        final Map<String, Map<String, Boolean>> resourcePrivilegeMap = new HashMap<>(resourceActionAccess.length);
+        for (Tuple<String, Tuple<String, Boolean>> t : resourceActionAccess) {
+            final String resource = t.v1();
+            final String action = t.v2().v1();
+            final Boolean access = t.v2().v2();
+            resourcePrivilegeMap.computeIfAbsent(resource, ignore -> new HashMap<>()).put(action, access);
+        }
+        final Collection<ResourcePrivileges> privileges = resourcePrivilegeMap.entrySet().stream()
+            .map(e -> ResourcePrivileges.builder(e.getKey()).addPrivileges(e.getValue()).build())
+            .collect(Collectors.toList());
+        final Map<String, Collection<ResourcePrivileges>> appPrivs = Map.of(appName, privileges);
+        final HasPrivilegesResponse response = new HasPrivilegesResponse(username, isCompleteMatch, Map.of(), Set.of(), appPrivs);
+
+        Mockito.doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args.length, equalTo(3));
+            ActionListener<HasPrivilegesResponse> listener = (ActionListener<HasPrivilegesResponse>) args[args.length - 1];
+            listener.onResponse(response);
+            return null;
+        }).when(client).execute(same(HasPrivilegesAction.INSTANCE), any(HasPrivilegesRequest.class), any(ActionListener.class));
+        return response;
+    }
+
+    private Tuple<String, Tuple<String, Boolean>> access(String resource, String action, boolean access) {
+        return new Tuple<>(resource, new Tuple<>(action, access));
+    }
+
+    private void setupUser(String principal) {
+        final User user = new User(principal, randomAlphaOfLengthBetween(6, 12));
+        securityContext.setUser(user, Version.CURRENT);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java
new file mode 100644
index 0000000000000..a52eb2fcfd084
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.XmlValidator;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.junit.Before;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.StatusCode;
+
+import java.time.Clock;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FailedAuthenticationResponseBuilderTests extends IdpSamlTestCase {
+
+    private SamlIdentityProvider idp;
+    private XmlValidator validator;
+    private SamlFactory samlFactory;
+
+    @Before
+    public void setupSaml() throws Exception {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+        validator = new XmlValidator("saml-schema-protocol-2.0.xsd");
+
+        idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://cloud.elastic.co/saml/idp");
+    }
+
+    public void testSimpleErrorResponseIsValid() throws Exception {
+        final Clock clock = Clock.systemUTC();
+        final FailedAuthenticationResponseMessageBuilder builder = new FailedAuthenticationResponseMessageBuilder(samlFactory, clock, idp);
+        final Response response = builder
+            .setAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs")
+            .setPrimaryStatusCode(StatusCode.REQUESTER)
+            .setInResponseTo(randomAlphaOfLength(12))
+            .build();
+        final String xml = super.toString(response);
+        validator.validate(xml);
+    }
+
+    public void testErrorResponseWithCodeIsValid() throws Exception {
+        final Clock clock = Clock.systemUTC();
+        final FailedAuthenticationResponseMessageBuilder builder = new FailedAuthenticationResponseMessageBuilder(samlFactory, clock, idp);
+        final Response response = builder
+            .setAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs")
+            .setPrimaryStatusCode(StatusCode.REQUESTER)
+            .setInResponseTo(randomAlphaOfLength(12))
+            .setSecondaryStatusCode(StatusCode.INVALID_NAMEID_POLICY)
+            .build();
+        final String xml = super.toString(response);
+        validator.validate(xml);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java
new file mode 100644
index 0000000000000..62a42340aca99
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java
@@ -0,0 +1,312 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestResponse;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.saml.common.SAMLObject;
+import org.opensaml.saml.saml2.core.AuthnRequest;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameIDPolicy;
+import org.opensaml.security.SecurityException;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.crypto.XMLSigningUtil;
+import org.opensaml.xmlsec.signature.support.SignatureConstants;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Set;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.joda.time.DateTime.now;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlAuthnRequestValidatorTests extends IdpSamlTestCase {
+
+    private SamlAuthnRequestValidator validator;
+    private SamlIdentityProvider idp;
+    private SamlFactory samlFactory = new SamlFactory();
+
+    @Before
+    public void setupValidator() throws Exception {
+        SamlInit.initialize();
+        idp = Mockito.mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://cloud.elastic.co/saml/idp");
+        when(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI)).thenReturn(new URL("https://cloud.elastic.co/saml/init"));
+        final SamlServiceProvider sp1 = Mockito.mock(SamlServiceProvider.class);
+        when(sp1.getEntityId()).thenReturn("https://sp1.kibana.org");
+        when(sp1.getAssertionConsumerService()).thenReturn(new URL("https://sp1.kibana.org/saml/acs"));
+        when(sp1.getAllowedNameIdFormat()).thenReturn(TRANSIENT);
+        when(sp1.shouldSignAuthnRequests()).thenReturn(false);
+        final SamlServiceProvider sp2 = Mockito.mock(SamlServiceProvider.class);
+        when(sp2.getEntityId()).thenReturn("https://sp2.kibana.org");
+        when(sp2.getAssertionConsumerService()).thenReturn(new URL("https://sp2.kibana.org/saml/acs"));
+        when(sp2.getAllowedNameIdFormat()).thenReturn(PERSISTENT);
+        when(sp2.getSpSigningCredentials()).thenReturn(Set.of(readCredentials("RSA", 4096)));
+        when(sp2.shouldSignAuthnRequests()).thenReturn(true);
+        mockRegisteredServiceProvider(idp, "https://sp1.kibana.org", sp1);
+        mockRegisteredServiceProvider(idp, "https://sp2.kibana.org", sp2);
+        validator = new SamlAuthnRequestValidator(samlFactory, idp);
+    }
+
+    public void testValidAuthnRequest() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp1.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(TRANSIENT));
+    }
+
+    public void testValidSignedAuthnRequest() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), PERSISTENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true,
+            readCredentials("RSA", 4096)), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp2.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(PERSISTENT));
+    }
+
+    public void testValidSignedAuthnRequestWithoutRelayState() throws Exception {
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), PERSISTENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, null, true,
+            readCredentials("RSA", 4096)), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp2.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(PERSISTENT));
+    }
+
+    public void testValidSignedAuthnRequestWhenServiceProviderShouldNotSign() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true, readCredentials("RSA", 4096)), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp1.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(TRANSIENT));
+    }
+
+    public void testValidUnSignedAuthnRequestWhenServiceProviderShouldSign() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("must sign authentication requests but no signature was found"));
+    }
+
+    public void testSignedAuthnRequestWithWrongKey() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true, readCredentials("RSA2", 4096)), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("Unable to validate signature of authentication request"));
+    }
+
+    public void testSignedAuthnRequestWithWrongSizeKey() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true, readCredentials("RSA", 2048)), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("Unable to validate signature of authentication request"));
+    }
+
+    public void testWrongDestination() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            new URL("https://wrong.destination.org"), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("but the SSO endpoint of this Identity Provider is"));
+        assertThat(e.getMessage(), containsString("wrong.destination.org"));
+    }
+
+    public void testUnregisteredAcsForSp() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://malicious.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("The registered ACS URL for this Service Provider is"));
+        assertThat(e.getMessage(), containsString("https://malicious.kibana.org/saml/acs"));
+    }
+
+    public void testUnregisteredSp()throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://unknown.kibana.org", new URL("https://unknown.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        mockRegisteredServiceProvider(idp, "https://unknown.kibana.org", null);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
+        assertThat(e.getMessage(), containsString("is not registered with this Identity Provider"));
+        assertThat(e.getMessage(), containsString("https://unknown.kibana.org"));
+    }
+
+    public void testAuthnRequestWithoutAcsUrl() throws Exception{
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        // remove ACS
+        authnRequest.setAssertionConsumerServiceURL(null);
+        final boolean containsIndex = randomBoolean();
+        if (containsIndex) {
+            authnRequest.setAssertionConsumerServiceIndex(randomInt(10));
+        }
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("SAML authentication does not contain an AssertionConsumerService URL"));
+        if (containsIndex) {
+            assertThat(e.getMessage(), containsString("It contains an Assertion Consumer Service Index "));
+        }
+    }
+
+    public void testAuthnRequestWithoutIssuer() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        // remove issuer
+        authnRequest.setIssuer(null);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("SAML authentication request has no issuer"));
+    }
+
+    public void testInvalidNameIDPolicy() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), PERSISTENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("doesn't match the allowed NameID format"));
+    }
+
+    private AuthnRequest buildAuthnRequest(String entityId, URL acs, URL destination, String nameIdFormat) {
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        final NameIDPolicy nameIDPolicy = samlFactory.buildObject(NameIDPolicy.class, NameIDPolicy.DEFAULT_ELEMENT_NAME);
+        nameIDPolicy.setFormat(nameIdFormat);
+        final AuthnRequest authnRequest = samlFactory.buildObject(AuthnRequest.class, AuthnRequest.DEFAULT_ELEMENT_NAME);
+        authnRequest.setID(samlFactory.secureIdentifier());
+        authnRequest.setIssuer(issuer);
+        authnRequest.setIssueInstant(now());
+        authnRequest.setAssertionConsumerServiceURL(acs.toString());
+        authnRequest.setDestination(destination.toString());
+        authnRequest.setNameIDPolicy(nameIDPolicy);
+        return authnRequest;
+    }
+
+    private String getQueryString(AuthnRequest authnRequest, String relayState) {
+        return getQueryString(authnRequest, relayState, false, null);
+    }
+
+    private String getQueryString(AuthnRequest authnRequest, String relayState, boolean sign, @Nullable X509Credential credential) {
+        try {
+            final String request = deflateAndBase64Encode(authnRequest);
+            String queryParam = "SAMLRequest=" + urlEncode(request);
+            if (relayState != null) {
+                queryParam += "&RelayState=" + urlEncode(relayState);
+            }
+            if (sign) {
+                final String algo = SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256;
+                queryParam += "&SigAlg=" + urlEncode(algo);
+                final byte[] sig = sign(queryParam, algo, credential);
+                queryParam += "&Signature=" + urlEncode(base64Encode(sig));
+            }
+            return queryParam;
+        } catch (Exception e) {
+            throw new ElasticsearchException("Cannot construct SAML redirect", e);
+        }
+    }
+
+    private String base64Encode(byte[] bytes) {
+        return Base64.getEncoder().encodeToString(bytes);
+    }
+
+    private String urlEncode(String param) throws UnsupportedEncodingException {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8.name());
+    }
+
+    private String deflateAndBase64Encode(SAMLObject message)
+        throws Exception {
+        Deflater deflater = new Deflater(Deflater.DEFLATED, true);
+        try (ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+             DeflaterOutputStream deflaterStream = new DeflaterOutputStream(bytesOut, deflater)) {
+            String messageStr = samlFactory.toString(XMLObjectSupport.marshall(message), false);
+            deflaterStream.write(messageStr.getBytes(StandardCharsets.UTF_8));
+            deflaterStream.finish();
+            return base64Encode(bytesOut.toByteArray());
+        }
+    }
+
+    private byte[] sign(String text, String algo, X509Credential credential) throws SecurityException {
+        return sign(text.getBytes(StandardCharsets.UTF_8), algo, credential);
+    }
+
+    private byte[] sign(byte[] content, String algo, X509Credential credential) throws SecurityException {
+        return XMLSigningUtil.signWithURI(credential, algo, content);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java
new file mode 100644
index 0000000000000..07aef6cb21ad7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.XmlValidator;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.joda.time.Duration;
+import org.junit.Before;
+import org.opensaml.saml.saml2.core.Response;
+
+import java.net.URL;
+import java.time.Clock;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SuccessfulAuthenticationResponseMessageBuilderTests extends IdpSamlTestCase {
+
+    private SamlIdentityProvider idp;
+    private XmlValidator validator;
+    private SamlFactory samlFactory;
+
+    @Before
+    public void setupSaml() throws Exception {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+        validator = new XmlValidator("saml-schema-protocol-2.0.xsd");
+
+        idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://cloud.elastic.co/saml/idp");
+        when(idp.getSigningCredential()).thenReturn(readCredentials("RSA", 2048));
+        when(idp.getServiceProviderDefaults())
+            .thenReturn(new ServiceProviderDefaults("elastic-cloud", TRANSIENT, Duration.standardMinutes(5)));
+    }
+
+    public void testSignedResponseIsValidAgainstXmlSchema() throws Exception {
+        final Response response = buildResponse();
+        final String xml = super.toString(response);
+        assertThat(xml, containsString("SignedInfo>"));
+        validator.validate(xml);
+    }
+
+    private Response buildResponse() throws Exception{
+        final Clock clock = Clock.systemUTC();
+
+        final SamlServiceProvider sp = mock(SamlServiceProvider.class);
+        final String baseServiceUrl = "https://" + randomAlphaOfLength(32) + ".us-east-1.aws.found.io/";
+        final String acs = baseServiceUrl + "api/security/saml/callback";
+        when(sp.getEntityId()).thenReturn(baseServiceUrl);
+        when(sp.getAssertionConsumerService()).thenReturn(new URL(acs));
+        when(sp.getAuthnExpiry()).thenReturn(Duration.standardMinutes(10));
+        when(sp.getAttributeNames()).thenReturn(new SamlServiceProvider.AttributeNames("principal", null, null, null));
+
+        final UserServiceAuthentication user = mock(UserServiceAuthentication.class);
+        when(user.getPrincipal()).thenReturn(randomAlphaOfLengthBetween(4, 12));
+        when(user.getRoles()).thenReturn(Set.of(randomArray(1, 5, String[]::new, () -> randomAlphaOfLengthBetween(4, 12))));
+        when(user.getEmail()).thenReturn(randomAlphaOfLength(8) + "@elastic.co");
+        when(user.getName()).thenReturn(randomAlphaOfLength(6) + " " + randomAlphaOfLength(8));
+        when(user.getServiceProvider()).thenReturn(sp);
+
+        final SuccessfulAuthenticationResponseMessageBuilder builder =
+            new SuccessfulAuthenticationResponseMessageBuilder(samlFactory, clock, idp);
+        return builder.build(user, null);
+    }
+
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java
new file mode 100644
index 0000000000000..9fcf28eb98377
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java
@@ -0,0 +1,536 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.mockito.Mockito;
+import org.opensaml.security.x509.X509Credential;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.util.List;
+
+import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ALLOWED_NAMEID_FORMATS;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_EMAIL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_GIVEN_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_SURNAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ENTITY_ID;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_DISPLAY_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_URL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SLO_POST_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SLO_REDIRECT_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_POST_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlIdentityProviderBuilderTests extends IdpSamlTestCase {
+
+    public void testAllSettings() throws Exception {
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        final X509Certificate signingCert = CertParsingUtils.readX509Certificates(List.of(signingCertPath))[0];
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .putList(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(ServiceProviderDefaults.APPLICATION_NAME_SETTING.getKey(), "my_super_idp")
+            .put(ServiceProviderDefaults.NAMEID_FORMAT_SETTING.getKey(), PERSISTENT)
+            .put(ServiceProviderDefaults.AUTHN_EXPIRY_SETTING.getKey(), "2m")
+            .put("xpack.idp.signing.key", destSigningKeyPath)
+            .put("xpack.idp.signing.certificate", destSigningCertPath)
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = ServiceProviderDefaults.forSettings(settings);
+        final SamlIdentityProvider idp = SamlIdentityProvider.builder(resolver).fromSettings(env).serviceProviderDefaults(defaults).build();
+        assertThat(idp.getEntityId(), equalTo("urn:elastic:cloud:idp"));
+        assertThat(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI).toString(), equalTo("https://idp.org/sso/redirect"));
+        assertThat(idp.getSingleSignOnEndpoint(SAML2_POST_BINDING_URI).toString(), equalTo("https://idp.org/sso/post"));
+        assertThat(idp.getSingleLogoutEndpoint(SAML2_REDIRECT_BINDING_URI).toString(), equalTo("https://idp.org/slo/redirect"));
+        assertThat(idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI).toString(), equalTo("https://idp.org/slo/post"));
+        assertThat(idp.getAllowedNameIdFormats(), hasSize(1));
+        assertThat(idp.getAllowedNameIdFormats(), Matchers.contains(TRANSIENT));
+        assertThat(idp.getOrganization(), equalTo(new SamlIdentityProvider.OrganizationInfo("organization_name",
+            "organization_display_name", "https://idp.org")));
+        assertThat(idp.getServiceProviderDefaults().applicationName, equalTo("my_super_idp"));
+        assertThat(idp.getServiceProviderDefaults().nameIdFormat, equalTo(PERSISTENT));
+        assertThat(idp.getServiceProviderDefaults().authenticationExpiry, equalTo(Duration.standardMinutes(2)));
+        assertThat(idp.getSigningCredential().getEntityCertificate(), equalTo(signingCert));
+        assertThat(idp.getSigningCredential().getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testMissingCredentials() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .put(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = new ServiceProviderDefaults(
+            randomAlphaOfLengthBetween(4, 8), randomFrom(TRANSIENT, PERSISTENT),
+            Duration.standardMinutes(randomIntBetween(2, 90)));
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).serviceProviderDefaults(defaults).build());
+        assertThat(e, instanceOf(ValidationException.class));
+        assertThat(e.getMessage(), containsString("Signing credential must be specified"));
+    }
+
+    public void testDefaultAllowedNameIdFormats() throws Exception {
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .put(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(ServiceProviderDefaults.APPLICATION_NAME_SETTING.getKey(), "my_super_idp")
+            .put(ServiceProviderDefaults.NAMEID_FORMAT_SETTING.getKey(), PERSISTENT)
+            .put(ServiceProviderDefaults.AUTHN_EXPIRY_SETTING.getKey(), "2m")
+            .put("xpack.idp.signing.key", destSigningKeyPath)
+            .put("xpack.idp.signing.certificate", destSigningCertPath)
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = ServiceProviderDefaults.forSettings(settings);
+        final SamlIdentityProvider idp = SamlIdentityProvider.builder(resolver).fromSettings(env).serviceProviderDefaults(defaults).build();
+        assertThat(idp.getAllowedNameIdFormats(), hasSize(1));
+        assertThat(idp.getAllowedNameIdFormats(), Matchers.contains(TRANSIENT));
+    }
+
+    public void testConfigurationWithForbiddenAllowedNameIdFormats() throws Exception {
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .putList(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT, PERSISTENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(ServiceProviderDefaults.APPLICATION_NAME_SETTING.getKey(), "my_super_idp")
+            .put(ServiceProviderDefaults.NAMEID_FORMAT_SETTING.getKey(), PERSISTENT)
+            .put(ServiceProviderDefaults.AUTHN_EXPIRY_SETTING.getKey(), "2m")
+            .put("xpack.idp.signing.key", destSigningKeyPath)
+            .put("xpack.idp.signing.certificate", destSigningCertPath)
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = ServiceProviderDefaults.forSettings(settings);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString("are not valid NameID formats. Allowed values are"));
+        assertThat(e.getMessage(), containsString(PERSISTENT));
+    }
+
+    public void testInvalidSsoEndpoint() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "not a url")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString(IDP_SSO_REDIRECT_ENDPOINT.getKey()));
+        assertThat(e.getMessage(), containsString("Not a valid URL"));
+    }
+
+    public void testMissingSsoRedirectEndpoint() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_ORGANIZATION_NAME.getKey(), "The Organization")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString(IDP_SSO_REDIRECT_ENDPOINT.getKey()));
+        assertThat(e.getMessage(), containsString("is required"));
+    }
+
+    public void testMalformedOrganizationUrl() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_ORGANIZATION_URL.getKey(), "The Org")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(IDP_ORGANIZATION_NAME.getKey(), "The Organization")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString(IDP_ORGANIZATION_URL.getKey()));
+        assertThat(e.getMessage(), containsString("Not a valid URL"));
+    }
+
+    public void testCreateSigningCredentialFromPemFiles() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        final X509Certificate signingCert = CertParsingUtils.readX509Certificates(List.of(signingCertPath))[0];
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+        builder.put("xpack.idp.signing.key", destSigningKeyPath);
+        builder.put("xpack.idp.signing.certificate", destSigningCertPath);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+        assertThat(credential.getPublicKey(), equalTo(signingCert.getPublicKey()));
+    }
+
+    public void testCreateSigningCredentialFromΚeystoreWithSingleEntry() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithSingleEntryAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", "signing1");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithSingleEntryButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithMultipleEntriesAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+        final String ks = randomFrom("multi_signing.jks", "multi_signing.p12");
+        final String alias = randomFrom("signing1", "signing2", "signing3", "signing4");
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath(ks);
+        final Path destSigningKeyStorePath = dir.resolve(ks);
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", alias);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+        final Path signingKeyPath = getDataPath(alias + ".key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithMultipleEntriesButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("multi_signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("multi_signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+    public void testCreateMetadataSigningCredentialFromΚeystoreWithSingleEntry() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithSingleEntryAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", "signing1");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithSingleEntryButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.metadata_signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithMultipleEntriesAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+        final String ks = randomFrom("multi_signing.jks", "multi_signing.p12");
+        final String alias = randomFrom("signing1", "signing2", "signing3", "signing4");
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath(ks);
+        final Path destSigningKeyStorePath = dir.resolve(ks);
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", alias);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        final Path signingKeyPath = getDataPath(alias + ".key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithMultipleEntriesButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("multi_signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("multi_signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.metadata_signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java
new file mode 100644
index 0000000000000..d536c201303dd
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.common.util.NamedFormatter;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.saml2.metadata.impl.EntityDescriptorMarshaller;
+import org.w3c.dom.Element;
+
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlIdpMetadataBuilderTests extends IdpSamlTestCase {
+
+    private SamlFactory samlFactory;
+
+    @Before
+    public void setup() throws Exception {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+    }
+
+    public void testSimpleMetadataGeneration() throws Exception {
+        final String entityId = "https://idp.org";
+        final EntityDescriptor entityDescriptor = new SamlIdPMetadataBuilder(entityId)
+            .withSingleSignOnServiceUrl(SAML2_REDIRECT_BINDING_URI, new URL(entityId + "/sso/redirect"))
+            .build();
+        final Element element = new EntityDescriptorMarshaller().marshall(entityDescriptor);
+        final String xml = samlFactory.toString(element, false);
+        assertThat(xml, equalTo("<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
+            "<md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"https://idp.org\">" +
+            "<md:IDPSSODescriptor WantAuthnRequestsSigned=\"false\" protocolSupportEnumeration=\"urn:oasis:names:tc:SAML:2.0:protocol\">" +
+            "<md:SingleSignOnService Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\" " +
+            "Location=\"https://idp.org/sso/redirect\"/>" +
+            "</md:IDPSSODescriptor>" +
+            "</md:EntityDescriptor>"));
+    }
+
+    public void testMetadataGenerationWithAllParameters() throws Exception {
+        final String entityId = "https://idp.org";
+        final EntityDescriptor entityDescriptor = new SamlIdPMetadataBuilder(entityId)
+            .withLocale(Locale.forLanguageTag("en"))
+            .withSingleSignOnServiceUrl(SAML2_REDIRECT_BINDING_URI, new URL(entityId + "/sso/redirect"))
+            .withSingleSignOnServiceUrl(SAML2_POST_BINDING_URI, new URL(entityId + "/sso/post"))
+            .withSingleLogoutServiceUrl(SAML2_REDIRECT_BINDING_URI, new URL(entityId + "/slo/redirect"))
+            .withSingleLogoutServiceUrl(SAML2_POST_BINDING_URI, new URL(entityId + "/slo/post"))
+            .withNameIdFormat(PERSISTENT)
+            .withNameIdFormat(TRANSIENT)
+            .wantAuthnRequestsSigned(true)
+            .withSigningCertificate(readCredentials("RSA", 2048).getEntityCertificate())
+            .withSigningCertificate(readCredentials("RSA", 4096).getEntityCertificate())
+            .withContact("technical", "Tony", "Stark", "tony@starkindustries.com")
+            .organization("Avengers", "The Avengers", "https://linktotheidp.org")
+            .build();
+        final Element element = new EntityDescriptorMarshaller().marshall(entityDescriptor);
+        final String xml = samlFactory.toString(element, false);
+
+        // RSA_2048
+        final String signingCertificateOne = joinCertificateLines(
+            "MIIDYzCCAkugAwIBAgIVAITQVqXYYUT0w04Z2gWAZ6pv7gwbMA0GCSqGSIb3DQEBCwUAMDQxMjAw",
+            "BgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMCAXDTIwMDEy",
+            "ODA2MzczNloYDzIxNjgxMDE5MDYzNzM2WjBRMRMwEQYKCZImiZPyLGQBGRYDb3JnMR0wGwYKCZIm",
+            "iZPyLGQBGRYNZWxhc3RpY3NlYXJjaDEMMAoGA1UECxMDaWRwMQ0wCwYDVQQDEwR0ZXN0MIIBIjAN",
+            "BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu",
+            "+gmU9Ke0LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUwzNTYZAEv",
+            "nZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcSf6D6nSDcKmCNpVAK3jX9",
+            "iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34R06JTMGTji+RULPISH1bc8FHdurRASkG",
+            "msei5GlJqPSuKdViuaKPmDrvKR8OK6gMzUd3pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQID",
+            "AQABo00wSzAdBgNVHQ4EFgQUD87H31WVQNfCc85/H2qhpzs3XfowHwYDVR0jBBgwFoAUVenAN+T0",
+            "6rqNDxjMcvgimnTw+FgwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAIvHYxT30cvoHWUE2",
+            "saDVJ4qs/e0G3WusDyem3e4HkqwLEah06RDSgVCaOfW3ey5Q6CIQW3HHGUYqO0nU8JVCWdAk3+bU",
+            "YJJOeLnwD+SbDxxKBhxLdx+BjWata85lfJTR9+dXs0RXAAN8dSiIaj9NSgnwiJqQQZf7i66S7XB5",
+            "8TDTdZlV3d26STLy5h7Uy6vyCka8Xu8HFQ4hH2qf2L6EhBbzVTB6tuyPQOQwrlLE65nhUNkfBbjZ",
+            "lre45UMc9GuxzHkbvd3HEQaroMHZxnu+/n/JDlgsrCYUEXnZnOXvgUPupPynoRdDN1F6r95TLyU9",
+            "pYjDf/6zNPE854VF6y1TqQ=="
+            );
+        // RSA_4096
+        final String signingCertificateTwo = joinCertificateLines(
+            "MIIFCTCCAvGgAwIBAgIUei1EtkLvWStQusThWwgO14R+gFowDQYJKoZIhvcNAQELBQAwFDESMBAG",
+            "A1UEAwwJc2FtbC10ZXN0MB4XDTIwMDIwNjIwMzA0OFoXDTIwMDMwNzIwMzA0OFowFDESMBAGA1UE",
+            "AwwJc2FtbC10ZXN0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuylnpy9h6cKUz6uC",
+            "Bi6eHBNW5GN6fQb1rU3BLjj5BMbXOa4Ebj84OOsNVtS5/vskMtBH5R+Y1IT49Evd0pKgkJtvbso7",
+            "avq/jfmi18uDI2Q7BjTzWC7aOkfaYXr5aBPEQpapjxV6QxirgMqz5x4WTgfBDfRr+gw2oXWsaVaQ",
+            "7xkECeIIra7j0vgOfTfSR4BQlbEiZktVvaE882kKGZPdDvklGs1/3DMHM09kRpxR/CUc3s8XtjH3",
+            "DjUhK2MxX77gXxRXMMrWNWpk1/oXF/MnTkIiJhYqD7lnudbGXaLnaP38M6/OtHPluqomS5j2tVGj",
+            "hh8vIHA/kXTh6lPH3rmGtHe3PgkARohojPsf62RXk22ycbXeMrvaw/V1cQmDUrmm8ha8qAi48uu6",
+            "oRK3pk81j3LI+Q4SCbP1gllaRnwP7b8xgxbKEzNJp83+a8eM+6zLlLchxjP2MhU41VkzKtomQ+tJ",
+            "lT9mSFLD48b06EKTRUQ9rK5bpv1j+TOBCxfKAB9NSGwxcjlVhokxKKGrJlPb7aPJzqY4lcRzZJyl",
+            "hqz7gM+QvSwOvGbw7m45hbBNGV2VJi00VP6d+O/2p2bg/fniV4rjh5pWgjKh0kLUzp9kqFDnbpZh",
+            "TcBP1YrfR1M2YUP7BBA887AQ5nOD5PRaV58Sz4F+K4pbwG4c5Dld1WNWJTsCAwEAAaNTMFEwHQYD",
+            "VR0OBBYEFFlOCjw+wyUxFg2kpSzLYROZ1WLjMB8GA1UdIwQYMBaAFFlOCjw+wyUxFg2kpSzLYROZ",
+            "1WLjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAIxs61pAUU0R7j8guc9vOziN",
+            "Cm48KwrrqzSQ7JvgrPdrZ0mv3JN+luAFcA+4itVU5LqO92V5PVuxhbNLOvAx7+qR32zRF863Crpq",
+            "/jAux+3azBsGyGyjqmA+bL1wXGLrn+Tmndwc90iEcRPcOZIDreSF0mRXesBVcXEg1E5gdKg4Hm7O",
+            "dbMZEHITzQ1/uSAQ0WxONsABS0SYgplk20IyWm2nOjtt5zw54mD6TfSgeirQa6Jaoht6n7EfIIMo",
+            "5wWxOXnAXOMdBzN4xf3S6u57fOXDkbHcS0zDYiEFP+FpXB+eabsHfUUKou7YHrzno2vaoAASLpfR",
+            "Xy3WlHqulg0ZaDspY/0mAZxwwUL294yO+/tBsh8y3a9qep/mEl0h0ghj25PeWR+XeNPBth1Y54XD",
+            "KSXGjWXRekn0dLMzazksvJgDN6Ip0hX5StqHbiJX10WpWdSOT06ynh+N6OI+VBj8ndcnLGfSE6gt",
+            "L9JH3IKNtUgodr6Z+CcyZswWKutHyyZE5vteNQFKeTidCQAw9kRW6gtGUVRU0+PrMvD/8WhSd6Wk",
+            "FS4XjN+BXrmruCSGugdL9fgpg21qKcZwkR9rYQXqRPK+nTiVCRrOzUyTFnPmusz8fg7eg6ONaf2x",
+            "MUeWfI+F8kK4NH5GkGggGqQDtes3Y+bWQ28lV7ny44TkMBARz6zH"
+            );
+
+        final String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            + "<md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"https://idp.org\">"
+            + "  <md:IDPSSODescriptor"
+            + "      WantAuthnRequestsSigned=\"true\""
+            + "      protocolSupportEnumeration=\"urn:oasis:names:tc:SAML:2.0:protocol\">"
+            + "    <md:KeyDescriptor use=\"signing\">"
+            + "      <ds:KeyInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">"
+            + "        <ds:X509Data>"
+            + "          <ds:X509Certificate>%(signingCertificateOne)</ds:X509Certificate>"
+            + "        </ds:X509Data>"
+            + "      </ds:KeyInfo>"
+            + "    </md:KeyDescriptor>"
+            + "    <md:KeyDescriptor use=\"signing\">"
+            + "      <ds:KeyInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">"
+            + "        <ds:X509Data>"
+            + "          <ds:X509Certificate>%(signingCertificateTwo)</ds:X509Certificate>"
+            + "        </ds:X509Data>"
+            + "      </ds:KeyInfo>"
+            + "    </md:KeyDescriptor>"
+            + "    <md:SingleLogoutService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\""
+            + "        Location=\"https://idp.org/slo/post\"/>"
+            + "    <md:SingleLogoutService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\""
+            + "        Location=\"https://idp.org/slo/redirect\"/>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:persistent</md:NameIDFormat>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:transient</md:NameIDFormat>"
+            + "    <md:SingleSignOnService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\""
+            + "        Location=\"https://idp.org/sso/post\"/>"
+            + "    <md:SingleSignOnService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\""
+            + "        Location=\"https://idp.org/sso/redirect\"/>"
+            + "  </md:IDPSSODescriptor>"
+            + "  <md:Organization>"
+            + "    <md:OrganizationName xml:lang=\"en\">Avengers</md:OrganizationName>"
+            + "    <md:OrganizationDisplayName xml:lang=\"en\">The Avengers</md:OrganizationDisplayName>"
+            + "    <md:OrganizationURL xml:lang=\"en\">https://linktotheidp.org</md:OrganizationURL>"
+            + "  </md:Organization>"
+            + "  <md:ContactPerson contactType=\"technical\">"
+            + "    <md:GivenName>Tony</md:GivenName>"
+            + "    <md:SurName>Stark</md:SurName>"
+            + "    <md:EmailAddress>tony@starkindustries.com</md:EmailAddress>"
+            + "  </md:ContactPerson>"
+            + "</md:EntityDescriptor>";
+
+        final Map<String, Object> replacements = new HashMap<>();
+        replacements.put("signingCertificateOne", signingCertificateOne);
+        replacements.put("signingCertificateTwo", signingCertificateTwo);
+
+        final String expectedXmlWithCertificate = NamedFormatter.format(expectedXml, replacements);
+
+        assertThat(xml, Matchers.equalTo(normaliseXml(expectedXmlWithCertificate)));
+
+        assertValidXml(xml);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java
new file mode 100644
index 0000000000000..2cc1a78cb51d3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.util.NamedFormatter;
+import org.elasticsearch.xpack.idp.action.SamlMetadataResponse;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.security.impl.SAMLSignatureProfileValidator;
+
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.signature.Signature;
+import org.opensaml.xmlsec.signature.support.SignatureValidator;
+import org.w3c.dom.Element;
+
+import java.net.URL;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+
+public class SamlMetadataGeneratorTests extends IdpSamlTestCase {
+
+    public void testGenerateMetadata() throws Exception {
+        SamlServiceProvider sp = mock(SamlServiceProvider.class);
+        when(sp.getEntityId()).thenReturn("https://sp.org");
+        when(sp.shouldSignAuthnRequests()).thenReturn(true);
+        SamlIdentityProvider idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://idp.org");
+        when(idp.getMetadataSigningCredential()).thenReturn(null);
+        when(idp.getSigningCredential()).thenReturn(readCredentials("RSA", 2048));
+        when(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI)).thenReturn(new URL("https://idp.org/sso/redirect"));
+        when(idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI)).thenReturn(new URL("https://idp.org/slo/post"));
+        mockRegisteredServiceProvider(idp, "https://sp.org", sp);
+
+        SamlFactory factory = new SamlFactory();
+        SamlMetadataGenerator generator = new SamlMetadataGenerator(factory, idp);
+        PlainActionFuture<SamlMetadataResponse> future = new PlainActionFuture<>();
+        generator.generateMetadata("https://sp.org", future);
+        SamlMetadataResponse response = future.actionGet();
+        final String xml = response.getXmlString();
+
+        // RSA_2048
+        final String signingCertificate = joinCertificateLines(
+            "MIIDYzCCAkugAwIBAgIVAITQVqXYYUT0w04Z2gWAZ6pv7gwbMA0GCSqGSIb3DQEBCwUAMDQxMjAw",
+            "BgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMCAXDTIwMDEy",
+            "ODA2MzczNloYDzIxNjgxMDE5MDYzNzM2WjBRMRMwEQYKCZImiZPyLGQBGRYDb3JnMR0wGwYKCZIm",
+            "iZPyLGQBGRYNZWxhc3RpY3NlYXJjaDEMMAoGA1UECxMDaWRwMQ0wCwYDVQQDEwR0ZXN0MIIBIjAN",
+            "BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu",
+            "+gmU9Ke0LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUwzNTYZAEv",
+            "nZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcSf6D6nSDcKmCNpVAK3jX9",
+            "iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34R06JTMGTji+RULPISH1bc8FHdurRASkG",
+            "msei5GlJqPSuKdViuaKPmDrvKR8OK6gMzUd3pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQID",
+            "AQABo00wSzAdBgNVHQ4EFgQUD87H31WVQNfCc85/H2qhpzs3XfowHwYDVR0jBBgwFoAUVenAN+T0",
+            "6rqNDxjMcvgimnTw+FgwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAIvHYxT30cvoHWUE2",
+            "saDVJ4qs/e0G3WusDyem3e4HkqwLEah06RDSgVCaOfW3ey5Q6CIQW3HHGUYqO0nU8JVCWdAk3+bU",
+            "YJJOeLnwD+SbDxxKBhxLdx+BjWata85lfJTR9+dXs0RXAAN8dSiIaj9NSgnwiJqQQZf7i66S7XB5",
+            "8TDTdZlV3d26STLy5h7Uy6vyCka8Xu8HFQ4hH2qf2L6EhBbzVTB6tuyPQOQwrlLE65nhUNkfBbjZ",
+            "lre45UMc9GuxzHkbvd3HEQaroMHZxnu+/n/JDlgsrCYUEXnZnOXvgUPupPynoRdDN1F6r95TLyU9",
+            "pYjDf/6zNPE854VF6y1TqQ=="
+        );
+
+        final String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            + "<md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"https://idp.org\">"
+            + "  <md:IDPSSODescriptor"
+            + "      WantAuthnRequestsSigned=\"true\""
+            + "      protocolSupportEnumeration=\"urn:oasis:names:tc:SAML:2.0:protocol\">"
+            + "    <md:KeyDescriptor use=\"signing\">"
+            + "      <ds:KeyInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">"
+            + "        <ds:X509Data>"
+            + "          <ds:X509Certificate>%(signingCertificate)</ds:X509Certificate>"
+            + "        </ds:X509Data>"
+            + "      </ds:KeyInfo>"
+            + "    </md:KeyDescriptor>"
+            + "    <md:SingleLogoutService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\""
+            + "        Location=\"https://idp.org/slo/post\"/>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:persistent</md:NameIDFormat>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:transient</md:NameIDFormat>"
+            + "    <md:SingleSignOnService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\""
+            + "        Location=\"https://idp.org/sso/redirect\"/>"
+            + "  </md:IDPSSODescriptor>"
+            + "</md:EntityDescriptor>";
+
+        final Map<String, Object> replacements = Map.of("signingCertificate", signingCertificate);
+
+        final String expectedXmlWithCertificate = NamedFormatter.format(expectedXml, replacements);
+        assertThat(xml, Matchers.equalTo(normaliseXml(expectedXmlWithCertificate)));
+        assertValidXml(xml);
+    }
+
+    public void testGenerateAndSignMetadata() throws Exception {
+        SamlServiceProvider sp = mock(SamlServiceProvider.class);
+        when(sp.getEntityId()).thenReturn("https://sp.org");
+        when(sp.shouldSignAuthnRequests()).thenReturn(true);
+        SamlIdentityProvider idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://idp.org");
+        when(idp.getMetadataSigningCredential()).thenReturn(readCredentials("RSA", 4096));
+        when(idp.getSigningCredential()).thenReturn(readCredentials("RSA", 2048));
+        when(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI)).thenReturn(new URL("https://idp.org/sso/redirect"));
+        when(idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI)).thenReturn(new URL("https://idp.org/slo/post"));
+        mockRegisteredServiceProvider(idp, "https://sp.org", sp);
+
+        X509Credential signingCredential = readCredentials("RSA", 4096);
+        SamlFactory factory = new SamlFactory();
+        SamlMetadataGenerator generator = new SamlMetadataGenerator(factory, idp);
+        Element element = generator.possiblySignDescriptor(generator.buildEntityDescriptor(sp), signingCredential);
+        EntityDescriptor descriptor = factory.buildXmlObject(element, EntityDescriptor.class);
+        Signature signature = descriptor.getSignature();
+        assertNotNull(signature);
+        SAMLSignatureProfileValidator profileValidator = new SAMLSignatureProfileValidator();
+        profileValidator.validate(signature);
+        SignatureValidator.validate(signature, signingCredential);
+        //no exception thrown
+        SignatureException e = expectThrows(SignatureException.class,
+            () -> SignatureValidator.validate(signature, readCredentials("RSA", 2048)));
+        assertThat(e.getMessage(), containsString("Unable to evaluate key against signature"));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java
new file mode 100644
index 0000000000000..e820406e62bf9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.TestUtils;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class IdpBaseRestHandlerTests extends ESTestCase {
+
+    public void testIdpAvailableOnTrialOrEnterprise() {
+        final IdpBaseRestHandler handler = buildHandler(randomFrom(License.OperationMode.ENTERPRISE, License.OperationMode.TRIAL));
+        assertThat(handler.isIdpFeatureAllowed(), equalTo(true));
+    }
+
+    public void testIdpNotAvailableOnOtherLicenses() {
+        License.OperationMode mode =
+            randomValueOtherThanMany(m -> m == License.OperationMode.ENTERPRISE || m == License.OperationMode.TRIAL,
+                () -> randomFrom(License.OperationMode.values()));
+        final IdpBaseRestHandler handler = buildHandler(mode);
+        assertThat(handler.isIdpFeatureAllowed(), equalTo(false));
+    }
+
+    private IdpBaseRestHandler buildHandler(License.OperationMode licenseMode) {
+        final Settings settings = Settings.builder()
+            .put("xpack.idp.enabled", true)
+            .build();
+        final TestUtils.UpdatableLicenseState licenseState = new TestUtils.UpdatableLicenseState(settings);
+        licenseState.update(licenseMode, true, null);
+        return new IdpBaseRestHandler(licenseState) {
+            @Override
+            protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+                return null;
+            }
+
+            @Override
+            public String getName() {
+                return "idp-rest-test";
+            }
+
+            @Override
+            public List<Route> routes() {
+                return List.of();
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java
new file mode 100644
index 0000000000000..1fb78fbe692b2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.opensaml.saml.saml2.core.NameID;
+import org.opensaml.security.x509.X509Credential;
+
+import java.io.IOException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SamlServiceProviderDocumentTests extends IdpSamlTestCase {
+
+    public void testValidationFailuresForMissingFields() throws Exception {
+        final SamlServiceProviderDocument doc = new SamlServiceProviderDocument();
+        doc.setDocId(randomAlphaOfLength(16));
+        final ValidationException validationException = doc.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), not(emptyIterable()));
+        assertThat(validationException.validationErrors(), Matchers.containsInAnyOrder(
+            "field [name] is required, but was [null]",
+            "field [entity_id] is required, but was [null]",
+            "field [acs] is required, but was [null]",
+            "field [created] is required, but was [null]",
+            "field [last_modified] is required, but was [null]",
+            "field [privileges.resource] is required, but was [null]",
+            "field [attributes.principal] is required, but was [null]"
+        ));
+    }
+
+    public void testValidationSucceedsWithMinimalFields() throws Exception {
+        final SamlServiceProviderDocument doc = createMinimalDocument();
+        final ValidationException validationException = doc.validate();
+        assertThat(validationException, nullValue());
+    }
+
+    public void testXContentRoundTripWithMinimalFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createMinimalDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertXContentRoundTrip(doc2), equalTo(doc1));
+    }
+
+    public void testXContentRoundTripWithAllFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createFullDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertXContentRoundTrip(doc2), equalTo(doc1));
+    }
+
+    public void testStreamRoundTripWithMinimalFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createMinimalDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertSerializationRoundTrip(doc2), equalTo(doc1));
+    }
+
+    public void testStreamRoundTripWithAllFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createFullDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertSerializationRoundTrip(doc2), equalTo(doc1));
+    }
+
+    private SamlServiceProviderDocument createFullDocument() throws CertificateException, IOException {
+        final List<X509Credential> credentials = readCredentials();
+        final List<X509Certificate> certificates = credentials.stream()
+            .map(X509Credential::getEntityCertificate)
+            .collect(Collectors.toUnmodifiableList());
+        final List<X509Certificate> spCertificates = randomSubsetOf(certificates);
+        final List<X509Certificate> idpCertificates = randomSubsetOf(certificates);
+        final List<X509Certificate> idpMetadataCertificates = randomSubsetOf(certificates);
+
+        final SamlServiceProviderDocument doc1 = new SamlServiceProviderDocument();
+        doc1.setDocId(randomAlphaOfLength(16));
+        doc1.setName(randomAlphaOfLengthBetween(8, 12));
+        doc1.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.setAcs("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        doc1.setCreatedMillis(System.currentTimeMillis() - randomIntBetween(100_000, 1_000_000));
+        doc1.setLastModifiedMillis(System.currentTimeMillis() - randomIntBetween(1_000, 100_000));
+        doc1.setNameIdFormat(randomFrom(NameID.TRANSIENT, NameID.PERSISTENT, NameID.EMAIL));
+        doc1.setAuthenticationExpiryMillis(randomLongBetween(100, 5_000_000));
+        doc1.certificates.setServiceProviderX509SigningCertificates(spCertificates);
+        doc1.certificates.setIdentityProviderX509SigningCertificates(idpCertificates);
+        doc1.certificates.setIdentityProviderX509MetadataSigningCertificates(idpMetadataCertificates);
+
+        doc1.privileges.setResource("service:" + randomAlphaOfLength(12) + ":" + randomAlphaOfLength(12));
+        final Map<String, String> roleActions = new HashMap<>();
+        for (int i = randomIntBetween(1, 6); i > 0; i--) {
+            roleActions.put(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLength(6) + ":" + randomAlphaOfLength(6));
+        }
+        doc1.privileges.setRoleActions(roleActions);
+
+        doc1.attributeNames.setPrincipal("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.attributeNames.setEmail("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.attributeNames.setName("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.attributeNames.setRoles("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        return doc1;
+    }
+
+    private SamlServiceProviderDocument createMinimalDocument() {
+        final SamlServiceProviderDocument doc1 = new SamlServiceProviderDocument();
+        doc1.setDocId(randomAlphaOfLength(16));
+        doc1.setName(randomAlphaOfLengthBetween(8, 12));
+        doc1.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.setAcs("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        doc1.setCreatedMillis(System.currentTimeMillis() - randomIntBetween(100_000, 1_000_000));
+        doc1.setLastModifiedMillis(System.currentTimeMillis() - randomIntBetween(1_000, 100_000));
+        doc1.privileges.setResource("service:" + randomAlphaOfLength(12) + ":" + randomAlphaOfLength(12));
+        doc1.attributeNames.setPrincipal("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        return doc1;
+    }
+
+    private SamlServiceProviderDocument assertXContentRoundTrip(SamlServiceProviderDocument obj1) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
+        final BytesReference bytes1 = XContentHelper.toXContent(obj1, xContentType, humanReadable);
+        try (XContentParser parser = XContentHelper.createParser(
+            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes1, xContentType)) {
+            final SamlServiceProviderDocument obj2 = SamlServiceProviderDocument.fromXContent(obj1.docId, parser);
+            assertThat(obj2, equalTo(obj1));
+
+            final BytesReference bytes2 = XContentHelper.toXContent(obj2, xContentType, humanReadable);
+            assertThat(bytes2, equalTo(bytes1));
+
+            return obj2;
+        }
+    }
+
+    private SamlServiceProviderDocument assertSerializationRoundTrip(SamlServiceProviderDocument doc) throws IOException {
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final SamlServiceProviderDocument read = copyWriteable(doc, new NamedWriteableRegistry(List.of()),
+            SamlServiceProviderDocument::new, version);
+        MatcherAssert.assertThat("Serialized document with version [" + version + "] does not match original object", read, equalTo(doc));
+        return read;
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java
new file mode 100644
index 0000000000000..e1abf62ee95f0
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.idp.IdentityProviderPlugin;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ENTITY_ID;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlServiceProviderIndexTests extends ESSingleNodeTestCase {
+
+    private ClusterService clusterService;
+    private SamlServiceProviderIndex serviceProviderIndex;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(LocalStateCompositeXPackPlugin.class, IdentityProviderPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(IDP_ENTITY_ID.getKey(), "urn:idp:org")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/init")
+            .put(super.nodeSettings())
+            .build();
+    }
+
+    @Before
+    public void setupComponents() throws Exception {
+        clusterService = super.getInstanceFromNode(ClusterService.class);
+        serviceProviderIndex = new SamlServiceProviderIndex(client(), clusterService);
+    }
+
+    @After
+    public void deleteTemplateAndIndex() {
+        client().admin().indices().delete(new DeleteIndexRequest(SamlServiceProviderIndex.INDEX_NAME + "*")).actionGet();
+        client().admin().indices().deleteTemplate(new DeleteIndexTemplateRequest(SamlServiceProviderIndex.TEMPLATE_NAME)).actionGet();
+        serviceProviderIndex.close();
+    }
+
+    public void testWriteAndFindServiceProvidersFromIndex() {
+        final int count = randomIntBetween(3, 5);
+        List<SamlServiceProviderDocument> documents = new ArrayList<>(count);
+
+        final ClusterService clusterService = super.getInstanceFromNode(ClusterService.class);
+        // Install the template
+        assertTrue("Template should have been installed", installTemplate());
+        // No need to install it again
+        assertFalse("Template should not have been installed a second time", installTemplate());
+
+        // Index should not exist yet
+        assertThat(clusterService.state().metaData().index(SamlServiceProviderIndex.INDEX_NAME), nullValue());
+
+        for (int i = 0; i < count; i++) {
+            final SamlServiceProviderDocument doc = randomDocument(i);
+            writeDocument(doc);
+            documents.add(doc);
+        }
+
+        final IndexMetaData indexMetaData = clusterService.state().metaData().index(SamlServiceProviderIndex.INDEX_NAME);
+        assertThat(indexMetaData, notNullValue());
+        assertThat(indexMetaData.getSettings().get("index.format"), equalTo("1"));
+        assertThat(indexMetaData.getAliases().size(), equalTo(1));
+        assertThat(indexMetaData.getAliases().keys().toArray(), arrayContainingInAnyOrder(SamlServiceProviderIndex.ALIAS_NAME));
+
+        refresh();
+
+        final Set<SamlServiceProviderDocument> allDocs = getAllDocs();
+        assertThat(allDocs, iterableWithSize(count));
+        for (SamlServiceProviderDocument doc : documents) {
+            assertThat(allDocs, hasItem(Matchers.equalTo(doc)));
+        }
+
+        final SamlServiceProviderDocument readDoc = randomFrom(documents);
+        assertThat(readDocument(readDoc.docId), equalTo(readDoc));
+
+        final SamlServiceProviderDocument findDoc = randomFrom(documents);
+        assertThat(findByEntityId(findDoc.entityId), equalTo(findDoc));
+
+        final SamlServiceProviderDocument deleteDoc = randomFrom(documents);
+        final DeleteResponse deleteResponse = deleteDocument(deleteDoc);
+        assertThat(deleteResponse.getId(), equalTo(deleteDoc.docId));
+        assertThat(deleteResponse.getResult(), equalTo(DocWriteResponse.Result.DELETED));
+
+        refresh();
+
+        assertThat(readDocument(deleteDoc.docId), nullValue());
+        assertThat(findAllByEntityId(deleteDoc.entityId), emptyIterable());
+    }
+
+    public void testWritesViaAliasIfItExists() {
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        assertTrue(installTemplate.actionGet());
+
+        // Create an index that will trigger the template, but isn't the standard index name
+        final String customIndexName = SamlServiceProviderIndex.INDEX_NAME + "-test";
+        client().admin().indices().create(new CreateIndexRequest(customIndexName)).actionGet();
+
+        final IndexMetaData indexMetaData = clusterService.state().metaData().index(customIndexName);
+        assertThat(indexMetaData, notNullValue());
+        assertThat(indexMetaData.getSettings().get("index.format"), equalTo("1"));
+        assertThat(indexMetaData.getAliases().size(), equalTo(1));
+        assertThat(indexMetaData.getAliases().keys().toArray(), arrayContainingInAnyOrder(SamlServiceProviderIndex.ALIAS_NAME));
+
+        SamlServiceProviderDocument document = randomDocument(1);
+        writeDocument(document);
+
+        // Index should not exist because we created an alternate index, and the alias points to that.
+        assertThat(clusterService.state().metaData().index(SamlServiceProviderIndex.INDEX_NAME), nullValue());
+
+        refresh();
+
+        final Set<SamlServiceProviderDocument> allDocs = getAllDocs();
+        assertThat(allDocs, iterableWithSize(1));
+        assertThat(allDocs, hasItem(Matchers.equalTo(document)));
+
+        assertThat(readDocument(document.docId), equalTo(document));
+    }
+
+    public void testInstallTemplateAutomaticallyOnClusterChange() {
+        // Create an index that will trigger a cluster state change
+        client().admin().indices().create(new CreateIndexRequest(randomAlphaOfLength(7).toLowerCase(Locale.ROOT))).actionGet();
+
+        IndexTemplateMetaData templateMeta = clusterService.state().metaData().templates().get(SamlServiceProviderIndex.TEMPLATE_NAME);
+        assertNotNull(templateMeta);
+
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        assertFalse("Template is already installed, should not install again", installTemplate.actionGet());
+    }
+
+    public void testInstallTemplateAutomaticallyOnDocumentWrite() {
+        final SamlServiceProviderDocument doc = randomDocument(1);
+        writeDocument(doc);
+
+        assertThat(readDocument(doc.docId), equalTo(doc));
+
+        IndexTemplateMetaData templateMeta = clusterService.state().metaData().templates().get(SamlServiceProviderIndex.TEMPLATE_NAME);
+        assertNotNull(templateMeta);
+
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        assertFalse("Template is already installed, should not install again", installTemplate.actionGet());
+    }
+
+    private boolean installTemplate() {
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        return installTemplate.actionGet();
+    }
+
+    private Set<SamlServiceProviderDocument> getAllDocs() {
+        final PlainActionFuture<Set<SamlServiceProviderDocument>> future = new PlainActionFuture<>();
+        serviceProviderIndex.findAll(ActionListener.wrap(
+            set -> future.onResponse(set.stream().map(doc -> doc.document.get()).collect(Collectors.toUnmodifiableSet())),
+            future::onFailure
+        ));
+        return future.actionGet();
+    }
+
+    private SamlServiceProviderDocument readDocument(String docId) {
+        final PlainActionFuture<SamlServiceProviderIndex.DocumentSupplier> future = new PlainActionFuture<>();
+        serviceProviderIndex.readDocument(docId, future);
+        final SamlServiceProviderIndex.DocumentSupplier supplier = future.actionGet();
+        return supplier == null ? null : supplier.getDocument();
+    }
+
+    private void writeDocument(SamlServiceProviderDocument doc) {
+        final PlainActionFuture<DocWriteResponse> future = new PlainActionFuture<>();
+        serviceProviderIndex.writeDocument(doc, DocWriteRequest.OpType.INDEX, WriteRequest.RefreshPolicy.WAIT_UNTIL, future);
+        doc.setDocId(future.actionGet().getId());
+    }
+
+    private DeleteResponse deleteDocument(SamlServiceProviderDocument doc) {
+        final PlainActionFuture<DeleteResponse> future = new PlainActionFuture<>();
+        serviceProviderIndex.readDocument(doc.docId, ActionListener.wrap(
+            info -> serviceProviderIndex.deleteDocument(info.version, WriteRequest.RefreshPolicy.IMMEDIATE, future),
+            future::onFailure));
+        return future.actionGet();
+    }
+
+    private SamlServiceProviderDocument findByEntityId(String entityId) {
+        final Set<SamlServiceProviderDocument> docs = findAllByEntityId(entityId);
+        assertThat(docs, iterableWithSize(1));
+        return docs.iterator().next();
+    }
+
+    private Set<SamlServiceProviderDocument> findAllByEntityId(String entityId) {
+        final PlainActionFuture<Set<SamlServiceProviderDocument>> future = new PlainActionFuture<>();
+        serviceProviderIndex.findByEntityId(entityId, ActionListener.wrap(
+            set -> future.onResponse(set.stream().map(doc -> doc.document.get()).collect(Collectors.toUnmodifiableSet())),
+            future::onFailure
+        ));
+        return future.actionGet();
+    }
+
+    private void refresh() {
+        PlainActionFuture<Void> future = new PlainActionFuture<>();
+        serviceProviderIndex.refresh(future);
+        future.actionGet();
+    }
+
+    public static SamlServiceProviderDocument randomDocument() {
+        return randomDocument(randomIntBetween(1, 999_999));
+    }
+
+    public static SamlServiceProviderDocument randomDocument(int index) {
+        final SamlServiceProviderDocument document = new SamlServiceProviderDocument();
+        document.setName(randomAlphaOfLengthBetween(5, 12));
+        document.setEntityId(randomUri() + index);
+        document.setAcs(randomUri("https") + index + "/saml/acs");
+
+        document.setEnabled(randomBoolean());
+        document.setCreatedMillis(System.currentTimeMillis() - TimeValue.timeValueDays(randomIntBetween(2, 90)).millis());
+        document.setLastModifiedMillis(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(1, 36)).millis());
+
+        if (randomBoolean()) {
+            document.setNameIdFormat(TRANSIENT);
+        }
+        if (randomBoolean()) {
+            document.setAuthenticationExpiryMillis(TimeValue.timeValueMinutes(randomIntBetween(1, 15)).millis());
+        }
+
+        document.privileges.setResource("app:" + randomAlphaOfLengthBetween(3, 6) + ":" + Math.abs(randomLong()));
+        final int roleCount = randomIntBetween(0, 4);
+        final Map<String, String> roles = new HashMap<>();
+        for (int i = 0; i < roleCount; i++) {
+            roles.put(randomAlphaOfLengthBetween(4, 8), randomAlphaOfLengthBetween(3, 6) + ":" + randomAlphaOfLengthBetween(3, 6));
+        }
+        document.privileges.setRoleActions(roles);
+
+        document.attributeNames.setPrincipal(randomUri());
+        if (randomBoolean()) {
+            document.attributeNames.setName(randomUri());
+        }
+        if (randomBoolean()) {
+            document.attributeNames.setEmail(randomUri());
+        }
+        if (roles.isEmpty() == false) {
+            document.attributeNames.setRoles(randomUri());
+        }
+
+        assertThat(document.validate(), nullValue());
+        return document;
+    }
+
+    private static String randomUri() {
+        return randomUri(randomFrom("urn", "http", "https"));
+    }
+
+    private static String randomUri(String scheme) {
+        return scheme + "://" + randomAlphaOfLengthBetween(2, 6) + "."
+            + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(2, 4) + "/";
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java
new file mode 100644
index 0000000000000..f637e9b517f85
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.junit.Before;
+import org.opensaml.saml.saml2.core.NameID;
+
+import java.net.URL;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SamlServiceProviderResolverTests extends ESTestCase {
+
+    private SamlServiceProviderIndex index;
+    private ServiceProviderDefaults serviceProviderDefaults;
+    private SamlIdentityProvider identityProvider;
+    private SamlServiceProviderResolver resolver;
+
+    @Before
+    public void setupMocks() {
+        index = mock(SamlServiceProviderIndex.class);
+        identityProvider = mock(SamlIdentityProvider.class);
+        serviceProviderDefaults = configureIdentityProviderDefaults();
+        resolver = new SamlServiceProviderResolver(Settings.EMPTY, index, serviceProviderDefaults);
+    }
+
+    public void testResolveWithoutCache() throws Exception {
+
+        final String entityId = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/";
+        final URL acs = new URL(entityId + "saml/acs");
+
+        final String principalAttribute = randomAlphaOfLengthBetween(6, 36);
+        final String rolesAttribute = randomAlphaOfLengthBetween(6, 36);
+        final String resource = "ece:" + randomAlphaOfLengthBetween(6, 12);
+        final Map<String, String> rolePrivileges = Map.of(randomAlphaOfLengthBetween(3, 6), "role:" + randomAlphaOfLengthBetween(4, 8));
+
+        final DocumentVersion docVersion = new DocumentVersion(
+            randomAlphaOfLength(12), randomNonNegativeLong(), randomNonNegativeLong());
+        final SamlServiceProviderDocument document = new SamlServiceProviderDocument();
+        document.setEntityId(entityId);
+        document.setAuthenticationExpiry(null);
+        document.setAcs(acs.toString());
+        document.privileges.setResource(resource);
+        document.privileges.setRoleActions(rolePrivileges);
+        document.attributeNames.setPrincipal(principalAttribute);
+        document.attributeNames.setRoles(rolesAttribute);
+
+        mockDocument(entityId, docVersion, document);
+
+        final SamlServiceProvider serviceProvider = resolveServiceProvider(entityId);
+        assertThat(serviceProvider.getEntityId(), equalTo(entityId));
+        assertThat(serviceProvider.getAssertionConsumerService(), equalTo(acs));
+        assertThat(serviceProvider.getAllowedNameIdFormat(), equalTo(serviceProviderDefaults.nameIdFormat));
+        assertThat(serviceProvider.getAuthnExpiry(), equalTo(serviceProviderDefaults.authenticationExpiry));
+        assertThat(serviceProvider.getSpSigningCredentials(), emptyIterable());
+        assertThat(serviceProvider.shouldSignAuthnRequests(), equalTo(false));
+        assertThat(serviceProvider.shouldSignLogoutRequests(), equalTo(false));
+
+        assertThat(serviceProvider.getAttributeNames(), notNullValue());
+        assertThat(serviceProvider.getAttributeNames().principal, equalTo(principalAttribute));
+        assertThat(serviceProvider.getAttributeNames().name, nullValue());
+        assertThat(serviceProvider.getAttributeNames().email, nullValue());
+        assertThat(serviceProvider.getAttributeNames().roles, equalTo(rolesAttribute));
+
+        assertThat(serviceProvider.getPrivileges(), notNullValue());
+        assertThat(serviceProvider.getPrivileges().getApplicationName(), equalTo(serviceProviderDefaults.applicationName));
+        assertThat(serviceProvider.getPrivileges().getResource(), equalTo(resource));
+        assertThat(serviceProvider.getPrivileges().getRoleActions(), equalTo(rolePrivileges));
+    }
+
+    public void testResolveReturnsCachedObject() throws Exception {
+        final SamlServiceProviderDocument document1 = SamlServiceProviderIndexTests.randomDocument(1);
+        final SamlServiceProviderDocument document2 = SamlServiceProviderIndexTests.randomDocument(2);
+        document2.entityId = document1.entityId;
+
+        final DocumentVersion docVersion = new DocumentVersion(randomAlphaOfLength(12), 1, 1);
+
+        mockDocument(document1.entityId, docVersion, document1);
+        final SamlServiceProvider serviceProvider1 = resolveServiceProvider(document1.entityId);
+
+        mockDocument(document1.entityId, docVersion, document2);
+        final SamlServiceProvider serviceProvider2 = resolveServiceProvider(document1.entityId);
+
+        assertThat(serviceProvider2, sameInstance(serviceProvider1));
+    }
+
+    public void testResolveIgnoresCacheWhenDocumentVersionChanges() throws Exception {
+        final SamlServiceProviderDocument document1 = SamlServiceProviderIndexTests.randomDocument(1);
+        final SamlServiceProviderDocument document2 = SamlServiceProviderIndexTests.randomDocument(2);
+        document2.entityId = document1.entityId;
+
+        final DocumentVersion docVersion1 = new DocumentVersion(randomAlphaOfLength(12), 1, 1);
+        final DocumentVersion docVersion2 = new DocumentVersion(randomAlphaOfLength(12), randomIntBetween(2, 10), randomIntBetween(1, 10));
+
+        mockDocument(document1.entityId, docVersion1, document1);
+        final SamlServiceProvider serviceProvider1a = resolveServiceProvider(document1.entityId);
+        final SamlServiceProvider serviceProvider1b = resolveServiceProvider(document1.entityId);
+        assertThat(serviceProvider1b, sameInstance(serviceProvider1a));
+
+        mockDocument(document1.entityId, docVersion2, document2);
+        final SamlServiceProvider serviceProvider2 = resolveServiceProvider(document1.entityId);
+
+        assertThat(serviceProvider2, not(sameInstance(serviceProvider1a)));
+        assertThat(serviceProvider2.getEntityId(), equalTo(document2.entityId));
+        assertThat(serviceProvider2.getAssertionConsumerService().toString(), equalTo(document2.acs));
+        assertThat(serviceProvider2.getAttributeNames().principal, equalTo(document2.attributeNames.principal));
+        assertThat(serviceProvider2.getAttributeNames().name, equalTo(document2.attributeNames.name));
+        assertThat(serviceProvider2.getAttributeNames().email, equalTo(document2.attributeNames.email));
+        assertThat(serviceProvider2.getAttributeNames().roles, equalTo(document2.attributeNames.roles));
+        assertThat(serviceProvider2.getPrivileges().getResource(), equalTo(document2.privileges.resource));
+    }
+
+    private SamlServiceProvider resolveServiceProvider(String entityId) {
+        final PlainActionFuture<SamlServiceProvider> future = new PlainActionFuture<>();
+        resolver.resolve(entityId, future);
+
+        final SamlServiceProvider serviceProvider = future.actionGet();
+        assertThat(serviceProvider, notNullValue());
+        return serviceProvider;
+    }
+
+    private ServiceProviderDefaults configureIdentityProviderDefaults() {
+        final String defaultNameId = NameID.TRANSIENT;
+        final String defaultApplication = randomAlphaOfLengthBetween(4, 12);
+        final Duration defaultExpiry = Duration.standardMinutes(12);
+        final ServiceProviderDefaults defaults = new ServiceProviderDefaults(
+            defaultApplication, defaultNameId, defaultExpiry);
+        when(identityProvider.getServiceProviderDefaults()).thenReturn(defaults);
+        return defaults;
+    }
+
+    private void mockDocument(String entityId, DocumentVersion docVersion, SamlServiceProviderDocument document) {
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(2));
+
+            assertThat(args[0], equalTo(entityId));
+
+            ActionListener<Set<SamlServiceProviderIndex.DocumentSupplier>> listener
+                = (ActionListener<Set<SamlServiceProviderIndex.DocumentSupplier>>) args[args.length - 1];
+            listener.onResponse(Set.of(new SamlServiceProviderIndex.DocumentSupplier(docVersion, () -> document)));
+            return null;
+        }).when(index).findByEntityId(anyString(), any(ActionListener.class));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java
new file mode 100644
index 0000000000000..a5d92a8b82b40
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.opensaml.saml.saml2.core.NameID;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class SamlAuthenticationStateTests extends IdpSamlTestCase {
+
+    public void testValidation() {
+        final SamlAuthenticationState state = new SamlAuthenticationState();
+        final ValidationException validationException = state.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), not(emptyIterable()));
+        assertThat(validationException.validationErrors(), Matchers.containsInAnyOrder(
+            "field [entity_id] is required, but was [null]",
+            "field [acs_url] is required, but was [null]"
+        ));
+    }
+
+    public void testXContentRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateAuthnState();
+        final SamlAuthenticationState state2 = assertXContentRoundTrip(state1);
+        assertThat(assertXContentRoundTrip(state2), equalTo(state1));
+    }
+
+    public void testNullStateXContentRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateMinimalAuthnState();
+        final SamlAuthenticationState state2 = assertXContentRoundTrip(state1);
+        assertThat(assertXContentRoundTrip(state2), equalTo(state1));
+    }
+
+    public void testStreamRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateAuthnState();
+        final SamlAuthenticationState state2 = assertSerializationRoundTrip(state1);
+        assertThat(assertSerializationRoundTrip(state2), equalTo(state1));
+    }
+
+    public void testNullStateStreamRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateMinimalAuthnState();
+        final SamlAuthenticationState state2 = assertSerializationRoundTrip(state1);
+        assertThat(assertSerializationRoundTrip(state2), equalTo(state1));
+    }
+
+    private SamlAuthenticationState generateAuthnState() {
+        SamlAuthenticationState authnState = new SamlAuthenticationState();
+        authnState.setRequestedAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        authnState.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        authnState.setAuthnRequestId(randomAlphaOfLength(12));
+        authnState.setRequestedNameidFormat(randomFrom(NameID.TRANSIENT, NameID.PERSISTENT, NameID.EMAIL));
+        return authnState;
+    }
+
+    private SamlAuthenticationState generateMinimalAuthnState() {
+        SamlAuthenticationState authnState = new SamlAuthenticationState();
+        authnState.setRequestedAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        authnState.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        authnState.setAuthnRequestId(null);
+        authnState.setRequestedNameidFormat(null);
+        return authnState;
+    }
+
+    private SamlAuthenticationState assertXContentRoundTrip(SamlAuthenticationState obj1) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
+        final BytesReference bytes1 = XContentHelper.toXContent(obj1, xContentType, humanReadable);
+        try (XContentParser parser = XContentHelper.createParser(
+            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes1, xContentType)) {
+            final SamlAuthenticationState obj2 = SamlAuthenticationState.fromXContent(parser);
+            assertThat(obj2, equalTo(obj1));
+
+            final BytesReference bytes2 = XContentHelper.toXContent(obj2, xContentType, humanReadable);
+            assertThat(bytes2, equalTo(bytes1));
+
+            return obj2;
+        }
+    }
+
+    private SamlAuthenticationState assertSerializationRoundTrip(SamlAuthenticationState state) throws IOException {
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final SamlAuthenticationState read = copyWriteable(state, new NamedWriteableRegistry(List.of()),
+            SamlAuthenticationState::new, version);
+        MatcherAssert.assertThat("Serialized state with version [" + version + "] does not match original object", read, equalTo(state));
+        return read;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java
new file mode 100644
index 0000000000000..7fe52a51baacf
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.opensaml.saml.common.SignableSAMLObject;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.LogoutRequest;
+import org.opensaml.saml.saml2.core.NameID;
+import org.opensaml.saml.saml2.core.NameIDType;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.Status;
+import org.opensaml.saml.saml2.core.StatusCode;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.xmlsec.signature.support.SignatureValidator;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SamlObjectSignerTests extends IdpSamlTestCase {
+
+    private SamlFactory samlFactory;
+    @Before
+    public void setupState() {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+    }
+
+    public void testSignLogoutRequest() throws Exception {
+        final List<X509Credential> credentials = readCredentials();
+        assertThat(credentials.size(), greaterThanOrEqualTo(2));
+        final X509Credential credential = credentials.get(0);
+        final X509Credential alternateCredential = credentials.get(1);
+
+        final String entityId = "https://" + randomAlphaOfLengthBetween(3, 8) + ".example.com/" + randomAlphaOfLengthBetween(2, 12);
+        final LogoutRequest request = createLogoutRequest(entityId);
+        final Element signedElement = sign(request, entityId, credential);
+        verifySignatureExists(signedElement);
+
+        final LogoutRequest signedRequest = domElementToXmlObject(signedElement, LogoutRequest.class);
+        try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
+            // verify with correct credential
+            SignatureValidator.validate(signedRequest.getSignature(), credential);
+            // fail with incorrect credential
+            expectThrows(SignatureException.class,
+                () -> SignatureValidator.validate(signedRequest.getSignature(), alternateCredential)
+            );
+        }
+    }
+
+    public void testSignAuthResponse() throws Exception {
+        final List<X509Credential> credentials = readCredentials();
+        assertThat(credentials.size(), greaterThanOrEqualTo(2));
+        final X509Credential credential = credentials.get(0);
+        final X509Credential alternateCredential = credentials.get(1);
+
+        final String entityId = "uri://" + randomAlphaOfLengthBetween(3, 8) + ".example.com/" + randomAlphaOfLengthBetween(2, 12);
+        final Response request = createAuthnResponse(entityId);
+        final Element signedElement = sign(request, entityId, credential);
+        verifySignatureExists(signedElement);
+
+        final Response signedResponse = domElementToXmlObject(signedElement, Response.class);
+        try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
+            // verify with correct credential
+            SignatureValidator.validate(signedResponse.getSignature(), credential);
+            // fail with incorrect credential
+            expectThrows(SignatureException.class,
+                () -> SignatureValidator.validate(signedResponse.getSignature(), alternateCredential)
+            );
+        }
+    }
+
+    private void verifySignatureExists(Element signedElement) {
+        final NodeList signatures = signedElement.getElementsByTagNameNS("http://www.w3.org/2000/09/xmldsig#", "Signature");
+        assertThat(signatures.getLength(), Matchers.equalTo(1));
+
+        final Node sigNode = signatures.item(0);
+        assertThat(sigNode, Matchers.instanceOf(Element.class));
+        assertThat(sigNode.getLocalName(), Matchers.equalTo("Signature"));
+
+        final List<Element> children = getChildren(sigNode, Element.class);
+        assertThat(children, iterableWithSize(2));
+        assertThat(children.get(0).getLocalName(), Matchers.equalTo("SignedInfo"));
+        assertThat(children.get(1).getLocalName(), Matchers.equalTo("SignatureValue"));
+    }
+
+    private Element sign(SignableSAMLObject request, String entityId, X509Credential credential) {
+        SamlIdentityProvider idp = buildIdP(entityId, credential);
+
+        SamlObjectSigner signer = new SamlObjectSigner(samlFactory, idp);
+        return signer.sign(request);
+    }
+
+    private SamlIdentityProvider buildIdP(String entityId, X509Credential credential) {
+        SamlIdentityProvider idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn(entityId);
+        when(idp.getSigningCredential()).thenReturn(credential);
+        when(idp.getEntityId()).thenReturn(entityId);
+        return idp;
+    }
+
+    private LogoutRequest createLogoutRequest(String entityId) {
+        final LogoutRequest request = samlFactory.buildObject(LogoutRequest.class, LogoutRequest.DEFAULT_ELEMENT_NAME);
+        request.setNotOnOrAfter(DateTime.now().plusMinutes(15));
+
+        final NameID nameID = samlFactory.buildObject(NameID.class, NameID.DEFAULT_ELEMENT_NAME);
+        nameID.setFormat(NameIDType.TRANSIENT);
+        nameID.setValue(randomAlphaOfLengthBetween(24, 64));
+        request.setNameID(nameID);
+
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        request.setIssuer(issuer);
+        return request;
+    }
+
+    private Response createAuthnResponse(String entityId) {
+        final Response response = samlFactory.buildObject(Response.class, Response.DEFAULT_ELEMENT_NAME);
+        final DateTime now = DateTime.now();
+        response.setIssueInstant(now);
+        response.setID(randomAlphaOfLength(24));
+
+        final StatusCode code = samlFactory.buildObject(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+        code.setValue(StatusCode.AUTHN_FAILED);
+
+        Status status = samlFactory.buildObject(Status.class, Status.DEFAULT_ELEMENT_NAME);
+        status.setStatusCode(code);
+        response.setStatus(status);
+
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        response.setIssuer(issuer);
+        return response;
+    }
+
+    private <T extends Node> List<T> getChildren(Node node, Class<T> type) {
+        final NodeList childNodes = node.getChildNodes();
+        final List<T> result = new ArrayList<>(childNodes.getLength());
+        for (int i = 0; i < childNodes.getLength(); i++) {
+            final Node child = childNodes.item(i);
+            if (type.isInstance(child)) {
+                result.add(type.cast(child));
+            }
+        }
+        return result;
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
new file mode 100644
index 0000000000000..404f58fa6a3c1
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
@@ -0,0 +1,297 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.test;
+
+import io.netty.util.ThreadDeathWatcher;
+import io.netty.util.concurrent.GlobalEventExecutor;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.SecurityField;
+import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.idp.LocalStateIdentityProviderPlugin;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.ExternalResource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_EMAIL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_GIVEN_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_SURNAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ENTITY_ID;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_DISPLAY_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_URL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults.APPLICATION_NAME_SETTING;
+import static org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults.NAMEID_FORMAT_SETTING;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public abstract class IdentityProviderIntegTestCase extends ESIntegTestCase {
+
+    // Local Security Cluster user
+    public static final String SAMPLE_USER_NAME = "es_user";
+    public static final String SAMPLE_USER_PASSWORD = "es_user_password";
+    public static final String SAMPLE_USER_PASSWORD_HASHED =
+        new String(Hasher.resolve("bcrypt9").hash(new SecureString(SAMPLE_USER_PASSWORD.toCharArray())));
+    public static final String SAMPLE_USER_ROLE = "es_user_role";
+    // User that is authenticated to the Security Cluster in order to perform SSO to cloud resources
+    public static final String SAMPLE_IDPUSER_NAME = "idp_user";
+    public static final String SAMPLE_IDPUSER_PASSWORD = "idp_user_password";
+    public static final String SAMPLE_IDPUSER_PASSWORD_HASHED =
+        new String(Hasher.resolve("bcrypt9").hash(new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray())));
+    public static final String SAMPLE_IDPUSER_ROLE = "idp_user_role";
+    // Cloud console user that calls all IDP related APIs
+    public static final String CONSOLE_USER_NAME = "console_user";
+    public static final String CONSOLE_USER_PASSWORD = "console_user_password";
+    public static final String CONSOLE_USER_PASSWORD_HASHED =
+        new String(Hasher.resolve("bcrypt9").hash(new SecureString(CONSOLE_USER_PASSWORD.toCharArray())));
+    public static final String CONSOLE_USER_ROLE = "console_user_role";
+    public static final String SP_ENTITY_ID = "ec:abcdef:123456";
+    public static final RequestOptions REQUEST_OPTIONS_AS_CONSOLE_USER = RequestOptions.DEFAULT.toBuilder()
+        .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+            new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+        .build();
+    private static Path PARENT_DIR;
+
+    @BeforeClass
+    public static void setup() {
+        PARENT_DIR = createTempDir();
+    }
+
+    /**
+     * A JUnit class level rule that runs after the AfterClass method in {@link ESIntegTestCase},
+     * which stops the cluster. After the cluster is stopped, there are a few netty threads that
+     * can linger, so we wait for them to finish otherwise these lingering threads can intermittently
+     * trigger the thread leak detector
+     */
+    @ClassRule
+    public static final ExternalResource STOP_NETTY_RESOURCE = new ExternalResource() {
+        @Override
+        protected void after() {
+            try {
+                GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } catch (IllegalStateException e) {
+                if (e.getMessage().equals("thread was not started") == false) {
+                    throw e;
+                }
+                // ignore since the thread was never started
+            }
+
+            try {
+                ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    };
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        final Path home = nodePath(PARENT_DIR, nodeOrdinal);
+        final Path xpackConf = home.resolve("config");
+        try {
+            Files.createDirectories(xpackConf);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        writeFile(xpackConf, "roles.yml", configRoles());
+        writeFile(xpackConf, "users", configUsers());
+        writeFile(xpackConf, "users_roles", configUsersRoles());
+        Settings.Builder builder = Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(XPackSettings.SECURITY_ENABLED.getKey(), true)
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
+            .put(NetworkModule.HTTP_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
+            .put("xpack.idp.enabled", true)
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_ORGANIZATION_NAME.getKey(), "Identity Provider")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "Identity Provider")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(APPLICATION_NAME_SETTING.getKey(), "elastic-cloud")
+            .put(NAMEID_FORMAT_SETTING.getKey(), TRANSIENT)
+            .put("xpack.idp.signing.key", resolveResourcePath("/keypair/keypair_RSA_2048.key"))
+            .put("xpack.idp.signing.certificate", resolveResourcePath("/keypair/keypair_RSA_2048.crt"))
+            .put("xpack.security.authc.realms." + FileRealmSettings.TYPE + ".file.order", 0)
+            .put("xpack.security.authc.realms." + NativeRealmSettings.TYPE + ".index.order", "1")
+            .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)
+            .put("xpack.license.self_generated.type", "trial");
+        return builder.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(LocalStateIdentityProviderPlugin.class, Netty4Plugin.class, CommonAnalysisPlugin.class);
+    }
+
+    @Override
+    protected boolean addMockTransportService() {
+        return false; // security has its own transport service
+    }
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected Function<Client, Client> getClientWrapper() {
+        Map<String, String> headers = Collections.singletonMap("Authorization",
+            basicAuthHeaderValue(SAMPLE_USER_NAME, new SecureString(SAMPLE_USER_PASSWORD.toCharArray())));
+        // we need to wrap node clients because we do not specify a user for nodes and all requests will use the system
+        // user. This is ok for internal n2n stuff but the test framework does other things like wiping indices, repositories, etc
+        // that the system user cannot do. so we wrap the node client with a user that can do these things since the client() calls
+        // return a node client
+        return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client;
+    }
+
+    @Override
+    protected Path nodeConfigPath(int nodeOrdinal) {
+        return nodePath(PARENT_DIR, nodeOrdinal).resolve("config");
+    }
+
+    private String configRoles() {
+        // test role allows for everything
+        return SAMPLE_USER_ROLE + ":\n" +
+            "  cluster: [ ALL ]\n" +
+            "  indices:\n" +
+            "    - names: '*'\n" +
+            "      allow_restricted_indices: true\n" +
+            "      privileges: [ ALL ]\n" +
+            "\n" +
+            // IDP end user doesn't need any privileges on the security cluster
+            SAMPLE_IDPUSER_ROLE + ":\n" +
+            // Could switch to grant apikey for user and call this as console_user
+            "  cluster: ['cluster:admin/xpack/security/api_key/create']\n" +
+            "  indices: []\n" +
+            "  applications:\n " +
+            "    - application: elastic-cloud\n" +
+            "       resources: [ '" + SP_ENTITY_ID + "' ]\n" +
+            "       privileges: [ 'sso:superuser' ]\n" +
+            "\n" +
+            // Console user should be able to call all IDP related endpoints
+            CONSOLE_USER_ROLE + ":\n" +
+            "  cluster: ['cluster:admin/idp/*']\n" +
+            "  indices: []\n";
+    }
+
+    private String configUsers() {
+        return SAMPLE_USER_NAME + ":" + SAMPLE_USER_PASSWORD_HASHED + "\n" +
+            SAMPLE_IDPUSER_NAME + ":" + SAMPLE_IDPUSER_PASSWORD_HASHED + "\n" +
+            CONSOLE_USER_NAME + ":" + CONSOLE_USER_PASSWORD_HASHED + "\n";
+    }
+
+    private String configUsersRoles() {
+        return SAMPLE_USER_ROLE + ":" + SAMPLE_USER_NAME + "\n" +
+            SAMPLE_IDPUSER_ROLE + ":" + SAMPLE_IDPUSER_NAME + "\n" +
+            CONSOLE_USER_ROLE + ":" + CONSOLE_USER_NAME + "\n";
+    }
+
+    Path nodePath(Path confDir, final int nodeOrdinal) {
+        return confDir.resolve(getCurrentClusterScope() + "-" + nodeOrdinal);
+    }
+
+    protected Scope getCurrentClusterScope() {
+        return getCurrentClusterScope(this.getClass());
+    }
+
+    private static Scope getCurrentClusterScope(Class<?> clazz) {
+        ClusterScope annotation = getAnnotation(clazz);
+        return annotation == null ? Scope.SUITE : annotation.scope();
+    }
+
+    private static ClusterScope getAnnotation(Class<?> clazz) {
+        if (clazz == Object.class || clazz == IdentityProviderIntegTestCase.class) {
+            return null;
+        }
+        ClusterScope annotation = clazz.getAnnotation(ClusterScope.class);
+        if (annotation != null) {
+            return annotation;
+        }
+        return getAnnotation(clazz.getSuperclass());
+    }
+
+    private static String writeFile(Path folder, String name, byte[] content) {
+        final Path path = folder.resolve(name);
+        Path tempFile = null;
+        try {
+            tempFile = Files.createTempFile(path.getParent(), path.getFileName().toString(), "tmp");
+            try (OutputStream os = Files.newOutputStream(tempFile, CREATE, TRUNCATE_EXISTING, WRITE)) {
+                Streams.copy(content, os);
+            }
+
+            try {
+                Files.move(tempFile, path, REPLACE_EXISTING, ATOMIC_MOVE);
+            } catch (final AtomicMoveNotSupportedException e) {
+                Files.move(tempFile, path, REPLACE_EXISTING);
+            }
+        } catch (final IOException e) {
+            throw new UncheckedIOException(String.format(Locale.ROOT, "could not write file [%s]", path.toAbsolutePath()), e);
+        } finally {
+            // we are ignoring exceptions here, so we do not need handle whether or not tempFile was initialized nor if the file exists
+            IOUtils.deleteFilesIgnoringExceptions(tempFile);
+        }
+        return path.toAbsolutePath().toString();
+    }
+
+    private static String writeFile(Path folder, String name, String content) {
+        return writeFile(folder, name, content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private Path resolveResourcePath(String resourcePathToFile) {
+        try {
+            Path path = createTempFile();
+            try (InputStream resourceInput = IdentityProviderIntegTestCase.class.getResourceAsStream(resourcePathToFile)) {
+                Files.copy(resourceInput, path, StandardCopyOption.REPLACE_EXISTING);
+            }
+            return path;
+        } catch (IOException e) {
+            throw new ElasticsearchException("Failed to resolve resource (Path=[{}])", e, resourcePathToFile);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java
new file mode 100644
index 0000000000000..54ba329dedc1b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.FileMatchers;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.XmlValidator;
+import org.hamcrest.Matchers;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.mockito.Mockito;
+import org.opensaml.core.xml.XMLObject;
+import org.opensaml.core.xml.io.MarshallingException;
+import org.opensaml.core.xml.io.Unmarshaller;
+import org.opensaml.core.xml.io.UnmarshallerFactory;
+import org.opensaml.core.xml.io.UnmarshallingException;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.security.x509.BasicX509Credential;
+import org.opensaml.security.x509.X509Credential;
+import org.w3c.dom.Element;
+
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.security.PrivateKey;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+import static org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport.getUnmarshallerFactory;
+
+public abstract class IdpSamlTestCase extends ESTestCase {
+
+    private static Locale restoreLocale;
+
+    @BeforeClass
+    public static void localeChecks() throws Exception {
+        Logger logger = LogManager.getLogger(IdpSamlTestCase.class);
+        SamlInit.initialize();
+        if (isTurkishLocale()) {
+            // See: https://github.com/elastic/elasticsearch/issues/29824
+            logger.warn("Attempting to run SAML test on turkish-like locale, but that breaks OpenSAML. Switching to English.");
+            restoreLocale = Locale.getDefault();
+            Locale.setDefault(Locale.ENGLISH);
+        }
+    }
+
+    private static boolean isTurkishLocale() {
+        return Locale.getDefault().getLanguage().equals(new Locale("tr").getLanguage())
+            || Locale.getDefault().getLanguage().equals(new Locale("az").getLanguage());
+    }
+
+    @AfterClass
+    public static void restoreLocale() {
+        if (restoreLocale != null) {
+            Locale.setDefault(restoreLocale);
+            restoreLocale = null;
+        }
+    }
+
+    protected static void mockRegisteredServiceProvider(SamlIdentityProvider idp, String entityId, SamlServiceProvider sp) {
+        Mockito.doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(3));
+            assertThat(args[0], Matchers.equalTo(entityId));
+            assertThat(args[args.length-1], Matchers.instanceOf(ActionListener.class));
+            ActionListener<SamlServiceProvider> listener = (ActionListener<SamlServiceProvider>) args[args.length-1];
+
+            listener.onResponse(sp);
+            return null;
+        }).when(idp).getRegisteredServiceProvider(Mockito.eq(entityId), Mockito.anyBoolean(), Mockito.any(ActionListener.class));
+    }
+
+    protected static void mockRegisteredServiceProvider(SamlServiceProviderResolver resolverMock, String entityId,
+                                                        SamlServiceProvider sp) {
+        Mockito.doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(2));
+            assertThat(args[0], Matchers.equalTo(entityId));
+            assertThat(args[args.length-1], Matchers.instanceOf(ActionListener.class));
+            ActionListener<SamlServiceProvider> listener = (ActionListener<SamlServiceProvider>) args[args.length-1];
+
+            listener.onResponse(sp);
+            return null;
+        }).when(resolverMock).resolve(Mockito.eq(entityId), Mockito.any(ActionListener.class));
+    }
+
+    protected List<X509Credential> readCredentials() throws CertificateException, IOException {
+        List<X509Credential> list = new ArrayList<>(2);
+        list.add(readCredentials("RSA", 1024));
+        list.add(readCredentials("RSA", 2048));
+        Collections.shuffle(list, random());
+        return list;
+    }
+
+    protected X509Credential readCredentials(String type, int size) throws CertificateException, IOException {
+        Path certPath = getDataPath("/keypair/keypair_" + type + "_" + size + ".crt");
+        Path keyPath = getDataPath("/keypair/keypair_" + type + "_" + size + ".key");
+        assertThat(certPath, FileMatchers.isRegularFile());
+        assertThat(keyPath, FileMatchers.isRegularFile());
+
+        final X509Certificate[] certificates = CertParsingUtils.readX509Certificates(List.of(certPath));
+        assertThat("Incorrect number of certificates in " + certPath, certificates, Matchers.arrayWithSize(1));
+
+        final PrivateKey privateKey = PemUtils.readPrivateKey(keyPath, () -> new char[0]);
+        return new BasicX509Credential(certificates[0], privateKey);
+    }
+
+    protected <T extends XMLObject> T domElementToXmlObject(Element element, Class<T> type) throws UnmarshallingException {
+        final UnmarshallerFactory unmarshallerFactory = getUnmarshallerFactory();
+        Unmarshaller unmarshaller = unmarshallerFactory.getUnmarshaller(element);
+        assertThat(unmarshaller, Matchers.notNullValue());
+        final XMLObject object = unmarshaller.unmarshall(element);
+        assertThat(object, Matchers.instanceOf(type));
+        return type.cast(object);
+    }
+
+    protected void print(Element element, Writer writer, boolean pretty) throws TransformerException {
+        final Transformer serializer = new SamlFactory().getHardenedXMLTransformer();
+        if (pretty) {
+            serializer.setOutputProperty(OutputKeys.INDENT, "yes");
+        }
+        serializer.transform(new DOMSource(element), new StreamResult(writer));
+    }
+
+    protected String toString(XMLObject object) {
+        try {
+            return toString(XMLObjectSupport.marshall(object));
+        } catch (MarshallingException e) {
+            throw new RuntimeException("cannot marshall XML object to DOM", e);
+        }
+    }
+
+    protected String toString(Element element) {
+        try (StringWriter writer = new StringWriter()) {
+            print(element, writer, true);
+            return writer.toString();
+        } catch (TransformerException e) {
+            throw new RuntimeException("cannot transform XML element to string", e);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    protected void assertValidXml(String xml) throws Exception {
+        new XmlValidator( "saml-schema-metadata-2.0.xsd").validate(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
+    }
+
+    protected String joinCertificateLines(String... lines) {
+        return Arrays.stream(lines).collect(Collectors.joining(System.lineSeparator()));
+    }
+
+    protected String normaliseXml(String input) {
+        // Remove spaces between elements, and compress other spaces. These patterns don't use \s because
+        // that would match newlines.
+        return input.replaceAll("> +<", "><").replaceAll(" +", " ");
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt b/x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt
new file mode 100644
index 0000000000000..9e7dc1180c97f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt
@@ -0,0 +1,6 @@
+# The RSA keypair certificates were generated with:
+
+elasticsearch-certutil cert --pem --out ${PWD}/keypair-rsa-2048.zip --days 54321 --keysize 2048 --name "CN=test,OU=idp,DC=elasticsearch,DC=org"
+
+elasticsearch-certutil cert --pem --out ${PWD}/keypair-rsa-1024.zip --days 54321 --keysize 1024 --name "CN=test-1024,OU=idp,DC=elasticsearch,DC=org"
+
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt
new file mode 100644
index 0000000000000..77bef513f58e7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCTCCAvGgAwIBAgIUC6eGyW4ccdmz4bbQHVtbogQ5Ip0wDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJc2FtbC10ZXN0MB4XDTIwMDIxMzA2MjEzNVoXDTIwMDMx
+NDA2MjEzNVowFDESMBAGA1UEAwwJc2FtbC10ZXN0MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAmy/NbPNOgv4YCA+RvTyWwS/pneWiVeqGEbCxTWetVK8k
+83hypxjSWZleQk8bVcfRVKwswYnIK1XwR1W0bnJyjXwo4dVT5nSOrzyLa3Z+BEIf
+KgRAk+rQ/o5u46HxXDifoiY45hP+AOA/0btvRXmQYkMI7R+/BjpntD3hNnh+8Z4V
+O+5+nTvlpVbOqEsanMG9VdeA8hL6rzve0mBU8zZkOkTlDOiWzIJST5n84phowt8a
+cNucrKyHxa90PJBC9vyva5o460QMiwRWe5SByP+h5j5n5mHMb3e8km22fctzMOct
+fCcIEAJMbw347euE7/ZjaueXtic5QCxcaoyDvf1v+Y/DJFb3xDnqXHwS+xzD4lM/
+3dScqHI5atpkghnhWSaFhEuH9BtQWwsTCojcckUbTFS/yQM5PJqiLhWk6QpazIOj
+S8kP9T9ta0zJDsfPKUNgrLWUn0cj8L6eZ87ET1E4fkJiD1WfeuvDmsPhImYCBWKY
+Iv3SXncs7uwe3EyJCuiJmncFXTJ2FruR6OkIOpWi6qh7V/ocTvSnP2MHJ08M8NwJ
+vv+ZIdjsAcEhn0kosfyI3ZkDsAwxait3ipG/asMp97T4qvDrnhHQXVYLsILq/+SV
+bZx42g29vzIFjGflqw0RxtxQvfb3xM9V71mnbJd5WHmN1IBp02Ct9dHi58Bh1z8C
+AwEAAaNTMFEwHQYDVR0OBBYEFFTTlGzPZb0FlDrRKRp1jB7e1J5HMB8GA1UdIwQY
+MBaAFFTTlGzPZb0FlDrRKRp1jB7e1J5HMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggIBAAM3Uz89I/7tfxtWW4lsPdE1jVJEzhufgu5Kj3JX7BfUpR1O
+6lz3RNHr55hjffAoiUd2//ZprmRgDnOcGx7P82fR7DXVsIBR6Jj19hO4w1Mwz2Ae
+kmdQ0Uiz1UYzul+U5o0+2WwH8sdgoNi+0U/LbgMFC3OT9XPAYvWfmJh07OKCJct1
++Eq0bVEZKbyQUzpwN1+fuX6DM7e21uOsZZgELi+pMx3UM8siw1veU6umbK6o1pML
+9px9j3rYGs8jP+s7bkrdfMam4tWmkcA9rYNHcjaTODuemaN5MH/q4HtvGUFyXTHH
+Z9cEc0LWg+83LmCIzCevmHmPpUHo285DFbs9gjC7effnXait7RH/SZ5ygR2c9bfX
+pOp5laVCnmgSg/EMkA47XnJI11iowlzqKZ/Bv6sydmB5V15ZJmyr61p/i7u0J6xj
+cCg0hZHZHBf2Wfihu2Cf1oK7NB7iwfJNHlKCcwc2RxIWAdCarF0e7yeLQdzbCy94
+orTQGIGgFu/ct8ip44yg5/HcJMvruDende9NKb7rjnSTxfqhBjjT6gnrxroKksWH
+eiMTg6g+ICTsAj4LHQ8LCoqdCBvGLOqmQeeDKZrYu1LRXbMD0PJchopGnCg0DAxP
+ndxNt7jpXjkrKH/C31LLRBD94ZSjeRCPqkoxeGTBTu0Uru3HNdlSnaxmr6gn
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key
new file mode 100644
index 0000000000000..8d9454d33df4b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCbL81s806C/hgI
+D5G9PJbBL+md5aJV6oYRsLFNZ61UryTzeHKnGNJZmV5CTxtVx9FUrCzBicgrVfBH
+VbRucnKNfCjh1VPmdI6vPItrdn4EQh8qBECT6tD+jm7jofFcOJ+iJjjmE/4A4D/R
+u29FeZBiQwjtH78GOme0PeE2eH7xnhU77n6dO+WlVs6oSxqcwb1V14DyEvqvO97S
+YFTzNmQ6ROUM6JbMglJPmfzimGjC3xpw25ysrIfFr3Q8kEL2/K9rmjjrRAyLBFZ7
+lIHI/6HmPmfmYcxvd7ySbbZ9y3Mw5y18JwgQAkxvDfjt64Tv9mNq55e2JzlALFxq
+jIO9/W/5j8MkVvfEOepcfBL7HMPiUz/d1Jyocjlq2mSCGeFZJoWES4f0G1BbCxMK
+iNxyRRtMVL/JAzk8mqIuFaTpClrMg6NLyQ/1P21rTMkOx88pQ2CstZSfRyPwvp5n
+zsRPUTh+QmIPVZ9668Oaw+EiZgIFYpgi/dJedyzu7B7cTIkK6ImadwVdMnYWu5Ho
+6Qg6laLqqHtX+hxO9Kc/YwcnTwzw3Am+/5kh2OwBwSGfSSix/IjdmQOwDDFqK3eK
+kb9qwyn3tPiq8OueEdBdVguwgur/5JVtnHjaDb2/MgWMZ+WrDRHG3FC99vfEz1Xv
+Wadsl3lYeY3UgGnTYK310eLnwGHXPwIDAQABAoICAH5j8GtLviXxzKDEDW6SajXt
+T9fJru1KlObTgZQJXKIcA3xIHRj6nldbEenvg9PZaiQMFmeKT/z7gTaYFtvdWm7h
+MGq91Bdd1tfh96sOVpQhRkByNiZCmPS6DJZYzrrNYzYs6yp4HeeYvGGUxotO9skQ
+z7RQpsPrzYp+224BtWIT3jdxhq2ImwCOMsHeaU5CfHHtkpeV3ZXarR+qVYXARuEu
+O17IEAmAGom6YLwsZSZRjrdcJb58xhe7TnAFDoUaR8TbVoBGa/DNF6KO0SVjDfj3
+2qdH/7jJTEv2DytlB+xMXD9Bis9/D8FjH9qmdx75DP+En4s7ZuVdO+eLicCzKP5T
+2Na8bJ+QRo20k4P0hQLsT2oMag7jaTi9n3Qv8/Ia3V6108uGWtsT9xH9pbVxptMt
+4Y2TpGqe2ljs6nGoO4wVE55I++exutUpXyGC/NUS9TBhqwsjMtDd+43JI9hhZvFY
+0r6f4MJgr23WYPyVSNFi1c6p4wun3oJNd8HLxMLwtYCB3eS4+SgdFKfYRXXAZqIP
+dc0X4TtE90VzyCtOSmUSQVRJ3mDz9ey80ZH7bsGg6ZZ4JefdGNpCD6/QUqR4eHLy
+GDyr0jAkLWAG5q0TbnbhsihSaCQGgcPYSagTZUUDMAyQ7BkxoX+T12xMmgcXqGme
+ATu2I+sMT1mmMIW1HvABAoIBAQDLr1jhBRWObR6JFQr0frotimKsdlt1ENKzIbYj
+pP6lf2eFr7x0qie5gkpcWJizEyFs2mWmNtsdgHSpj8L0+PHnnRczSnYk/eo0AS+I
+vtci9/9dTJb7X+nwQ49LsijQt3UVqNfdSiFpxphqixB1C0R0ISxF4CfxpmGH4YYN
+TIBmhZwFS7/Dutrw/MtzTg7kpa/IK72CTn9b+Re9vWhFghgCUW12jnsUXKriYCRr
+73kUCHk6h8L7ZA4HseIDzKOzAwr/pyD1wl+wh2YPchy/FFIm4tMFYVZZnMFw7jIX
+cR3q1G0vI4pIzuze2x33Sad6VGL3eymatSJZUQ0ZQLBKctCRAoIBAQDDC5wWeBLA
+fHv6MxO5uToFZAfqWvdItQthxe8wDsLMJ0tWfB/j2ySqzVh6Nt3BeWkeR/E6ZAAr
+JqzNXvpoCwq4udWZVE3FjPBY/YlxQj0tjwfJaoDcWeCpxIj9AC+veKh8NH/5t7e8
+YLMgeoG4JsKQWUI7FZAZlRV36cWnkc3ICZcKv+A9Kxz9PtZWge5Fudd4cib8zMxX
++jrG/NvIhHYuQVee5jm2BflQB7OWntabjj8Q141IDPDiMIvarNiBliQQH5aL+Orv
+Q2uq8R3sOXfyTAh1P3blD3H3DOoi9kalqyoYCCpUP1J3H1hFe3M5xS7gt2D2b8ZI
+xWYCQQItURLPAoIBAGOVpZTO0Y1DGIzZNvP3Cbu+TyQW8fw4b2uZbd3bi40Kz2hz
+VWaM0sNGSmkAABh9n/gCh3ROma7A4UkNurnfBRbCnVc8bqsDYgBvG/h1peW47qeF
+DMR2ToBj56mQv5fcOmiOlovg9JksudRNjYxMJ/nHRkgboivptwGiZ9IUtypo5LFK
+KcXek7EDR81mOq3bvCfYCt2s12P2U383HcXuJyLrAOPYIaEA9aOccfI7sSz0P6E4
+Q0J51so4VFKbOOYK9NOtuNWvZO2AJYwjtJ5PpyLP/3A/+OHzXDiEnQUNDx2DIqbL
+b47NbM5Av1PqNPIAVCq/ksXLDbIxiH/yOr8Lh6ECggEBAKzYtwR6eDO4na3GYe6u
+pSpt0U3wO0BloKaO4D74dcyx7ePAsAofBEmRHJ2BzddNHsjE/JPAVx3mcjC1wLc/
+QcsedJoiyaEOG1jhplGGX+zl2gK2rWeZBq2sC7IZ3ihkhvs7E0Zbdyorj9Jyfk6R
+ms6NK1Js83yYT6QquQyc+34QcZgHHlNWx6PtccjL8Do+TSLiFoblfxlgGYKKRsNW
+D91B7sJUSER02tH/YTlTnd/QmTb1rrBxN29bkjTCBEQip+bZ93InrxtF5j5g2yGi
+dBd3Je+xxE2N+HL1MPC2yzi5jN4rLkfmrc//KRy6IAzMH5TpLbZ2q4OF27aXobRD
+KqUCggEATNxcmKNCEKBtpRHtcXa5qi4dEDIPYb2eeIrJMc0O/fD/SCiDzZL8Ewp3
+C4A7h1fTLxVCeAM2A+LYTLZXdZ04M4keOiBmagOMkTfl5UYcSJfP3UjHYf71HSFY
+pbJmow5iNlLizGExH+ZgV+3bKz2mJmBQVmVU46/1i5CeoJECllS/NBB9Qc571eG4
+ENJ1EcIMVcYsdAgB82L9mQtUYdxZJNy1QRMNbinuISqew6+NEzZyJlyyOtI0kEGm
+gOo6hf27PjiHHGvERgHVLLy4ZPz6JOVITWaYRVs5Oxemk/Q+v1Lv60MEYxzh9rni
+eJirRnJDCpyAo0tJ/FyjbG07Mu+dcg==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt
new file mode 100644
index 0000000000000..33d1a7d03bf2e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt
@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICYjCCAcugAwIBAgIUL9T74f75INSxmpirmjJTZHJXOGAwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwIBcNMjAwMTI4MDgwNzA1WhgPMjE2ODEwMTkwODA3MDVaMFYxEzAR
+BgoJkiaJk/IsZAEZFgNvcmcxHTAbBgoJkiaJk/IsZAEZFg1lbGFzdGljc2VhcmNo
+MQwwCgYDVQQLEwNpZHAxEjAQBgNVBAMTCXRlc3QtMTAyNDCBnzANBgkqhkiG9w0B
+AQEFAAOBjQAwgYkCgYEAt2bngLBAj+gq7rBNSdfetiTemIAxtGZiSuXJj6+C/2A/
+PAA9iMh9zGxtyv5YEw7zz5dDHf3v3pp3LZOm1N16/J5V1kmZrkpLFYkIb+xmxVmp
+H0hOdtHhvEKUMoi77oFGLf/pP+TR+2TGW1v9O8kGzcaTV8gwPVVF6trJdpmzdkMC
+AwEAAaNNMEswHQYDVR0OBBYEFG932Q/6totkjHTHRdwkXOfxyjl5MB8GA1UdIwQY
+MBaAFGRvsxxsx22dsQX8cVcBaNlNz5uDMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQEL
+BQADgYEAhVvFReQd4rmd7gHOSJyZz8gEfHKNPY9G7CX82ojdHpHBQb2m5JSctHfx
+uk4G3CKimRTxgkA91JYoD7z6eHCVWfSuqSEqi8DX0RTkQ6/iEeU75uFPktMmqwR9
+Oy0uR7N64MOPBa7jcyPlNFxKsi0VTCvqUa4CeArghjheCrTuhZE=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key
new file mode 100644
index 0000000000000..2cd0756824103
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC3ZueAsECP6CrusE1J1962JN6YgDG0ZmJK5cmPr4L/YD88AD2I
+yH3MbG3K/lgTDvPPl0Md/e/emnctk6bU3Xr8nlXWSZmuSksViQhv7GbFWakfSE52
+0eG8QpQyiLvugUYt/+k/5NH7ZMZbW/07yQbNxpNXyDA9VUXq2sl2mbN2QwIDAQAB
+AoGBAIDiqD4fnr+DAOjMrQP/2FtYfbXeIRAOA3VvFr8LMJwXH1kjMhs2CnHmoSHO
+rEZ4c46nVlt1YWSmWKqG4O8m1o4QiESSbLNxP1AJddWBXeosDgK7pHNNZKprGGUz
+aef3IRYBsoVwo76TsdFkFskFe1Ibq16qdH/JZgEEom/3Y65BAkEA2mcAIiusLdkl
+WMIkX+JbJequ7s3jCqC4NSB5YSVQjAAyUENhX2dOqjwHj8280LK0Nh32E2Yg1OjL
+U2ks+dWp7wJBANb5b5jFQ/qiZpn8ZUzSFICJ1yZHtYTdL+YghdFe4V1/5VkRCntn
+p8BdB/KPYlqIh4nNah9nvI1Q7PRY1f+hHO0CQHuJM2wgo92zLArOleCemswRuS1e
+zAPg8REr8zxd7PejzNHVqyhZuQdVuTlAOIMb8g+CVwSWChAkNY5SuZv/nYUCQBqa
+N3MiRi1H14aKs0Z0YNEfPZ6hFRdh/8FsmOK16RGW+ByzfSaURzAI0yjzEWq+pncp
+Ek7cmXqrXwdFzwWZPGkCQC18L7C5mnT4it6z5DHRimCDkwzbNOEEl8xjt1k55lm0
+dy3nq3Ca4FQF5Nbz4SL7e0STIzT5zbd/N8BsRXe1npA=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt
new file mode 100644
index 0000000000000..f7d5e0dd71bf9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDYzCCAkugAwIBAgIVAITQVqXYYUT0w04Z2gWAZ6pv7gwbMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMCAXDTIwMDEyODA2MzczNloYDzIxNjgxMDE5MDYzNzM2WjBRMRMw
+EQYKCZImiZPyLGQBGRYDb3JnMR0wGwYKCZImiZPyLGQBGRYNZWxhc3RpY3NlYXJj
+aDEMMAoGA1UECxMDaWRwMQ0wCwYDVQQDEwR0ZXN0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu+gmU
+9Ke0LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUw
+zNTYZAEvnZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcS
+f6D6nSDcKmCNpVAK3jX9iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34
+R06JTMGTji+RULPISH1bc8FHdurRASkGmsei5GlJqPSuKdViuaKPmDrvKR8OK6gM
+zUd3pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQIDAQABo00wSzAdBgNVHQ4E
+FgQUD87H31WVQNfCc85/H2qhpzs3XfowHwYDVR0jBBgwFoAUVenAN+T06rqNDxjM
+cvgimnTw+FgwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAIvHYxT30cvoH
+WUE2saDVJ4qs/e0G3WusDyem3e4HkqwLEah06RDSgVCaOfW3ey5Q6CIQW3HHGUYq
+O0nU8JVCWdAk3+bUYJJOeLnwD+SbDxxKBhxLdx+BjWata85lfJTR9+dXs0RXAAN8
+dSiIaj9NSgnwiJqQQZf7i66S7XB58TDTdZlV3d26STLy5h7Uy6vyCka8Xu8HFQ4h
+H2qf2L6EhBbzVTB6tuyPQOQwrlLE65nhUNkfBbjZlre45UMc9GuxzHkbvd3HEQar
+oMHZxnu+/n/JDlgsrCYUEXnZnOXvgUPupPynoRdDN1F6r95TLyU9pYjDf/6zNPE8
+54VF6y1TqQ==
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key
new file mode 100644
index 0000000000000..52eb2d494d60e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu+gmU9Ke0
+LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUwzNTY
+ZAEvnZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcSf6D6
+nSDcKmCNpVAK3jX9iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34R06J
+TMGTji+RULPISH1bc8FHdurRASkGmsei5GlJqPSuKdViuaKPmDrvKR8OK6gMzUd3
+pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQIDAQABAoIBAFLSQMf+Xxg0NnpT
+4DOdFG3K4M/rB6CXjeaqB5GDRYaImmnQ6C47aWf91l281OeGbsZn4tjQZ0s6c90E
+z9oMu9HsRPN8yBiUTG+ilNodIXt2qjR++RpI+L9V5oxgoaEpy/dsBueA3nArbva1
+u0j1fohoJI4AQ2js+xOvZUVpqG4QZcSmcYf8xM2t3hTE+jYXfOuNnX4PIU3fDqTp
+N0BP8A0sTpzPhuoSiICbqVSWiBe2UXojbhLp1D17R1Jm/PWuGYp8mUBVs5xJe4e+
+DGoek5JfC1H18b8/lmJZbN7njEsdA9XYOg67F02bgU0BGif3d0T5Sr0Yol19v2/o
+NNJhbFUCgYEA7Ts6kW/WRaZrRdBC3jhtOcZa1YiaoRFKk6m5hBpsmxmuOdIMtfgP
+94TN3hhi3XDrnG59vIukT0FSJy04qjbkIXToEi3C8BR6ALv228N34GLbq2zwfyqK
+1BnJkoGqCp8X3pScrg8nX11pSeS4ZGITJApSTnMT09BwwFqA48rpenMCgYEAyPTt
+05j4Fyg2jvXvS2oAJE4KUAo5kOy5PrHHkiJ7jaCC+ztCiebTa1V6QlwAeywhohBD
+kPcLDuNAtXlHq/9JC0Oym99fedQ7Fowk7VDjzNkPL2ZTbSTbzMwPeQVBrNP8AH06
+Kb9LCbvNlrymnciGbqRsnD3k4nVV2wwY0qBSno8CgYEAv8JjqlIbgvtttOYOVzCH
+Ye8rV3XvgAjJHqlQEy+E4K6WfZiDu8fDqFFKU5+kstjnKvDJZFIARCxJ6u8L4dMy
+/rRQLvmcnJIAuH03oxbS3KcsZ9pMYI6T37Tkki+LssA8/CdmqvAG5HhOdnOSdVot
+nLFXLKCqw0m3ClVd4EQSDG8CgYEAoSWYGZiPSbKbnaSmEemc14AlUO4CH69VFWtm
+yE8SmQ3gOyrYPtF1NJtFATXZxOwsxFMae4M/wN6Ho37v9IMwxfGxpsFHRODlyTz0
+18GIC1r8jFwYBmwKPn5c4gVPFxKfAJ1rKNMXj2Skhn4ddmW5N7xuxBP2GYtSH+fh
+d/XMiw0CgYEA6QjLGwgDJ3zmnrkJ+qcMjSS1yMWlhLpqHuSGj9P6hiF+szl3H9LU
+aH6ZpFWumzHPSqr3MJG0sHo/+Q1qQQduhiC5u5CXM1YEpnLQ6foK4TS5UXfOYfUJ
+lp0VuNrnPrycQ7tCtJ7s8uqWlFh3QufiUvCOLD5XS9Sw2rXs/vzI3aM=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt
new file mode 100644
index 0000000000000..191f5ceb8db13
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCTCCAvGgAwIBAgIUei1EtkLvWStQusThWwgO14R+gFowDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJc2FtbC10ZXN0MB4XDTIwMDIwNjIwMzA0OFoXDTIwMDMw
+NzIwMzA0OFowFDESMBAGA1UEAwwJc2FtbC10ZXN0MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAuylnpy9h6cKUz6uCBi6eHBNW5GN6fQb1rU3BLjj5BMbX
+Oa4Ebj84OOsNVtS5/vskMtBH5R+Y1IT49Evd0pKgkJtvbso7avq/jfmi18uDI2Q7
+BjTzWC7aOkfaYXr5aBPEQpapjxV6QxirgMqz5x4WTgfBDfRr+gw2oXWsaVaQ7xkE
+CeIIra7j0vgOfTfSR4BQlbEiZktVvaE882kKGZPdDvklGs1/3DMHM09kRpxR/CUc
+3s8XtjH3DjUhK2MxX77gXxRXMMrWNWpk1/oXF/MnTkIiJhYqD7lnudbGXaLnaP38
+M6/OtHPluqomS5j2tVGjhh8vIHA/kXTh6lPH3rmGtHe3PgkARohojPsf62RXk22y
+cbXeMrvaw/V1cQmDUrmm8ha8qAi48uu6oRK3pk81j3LI+Q4SCbP1gllaRnwP7b8x
+gxbKEzNJp83+a8eM+6zLlLchxjP2MhU41VkzKtomQ+tJlT9mSFLD48b06EKTRUQ9
+rK5bpv1j+TOBCxfKAB9NSGwxcjlVhokxKKGrJlPb7aPJzqY4lcRzZJylhqz7gM+Q
+vSwOvGbw7m45hbBNGV2VJi00VP6d+O/2p2bg/fniV4rjh5pWgjKh0kLUzp9kqFDn
+bpZhTcBP1YrfR1M2YUP7BBA887AQ5nOD5PRaV58Sz4F+K4pbwG4c5Dld1WNWJTsC
+AwEAAaNTMFEwHQYDVR0OBBYEFFlOCjw+wyUxFg2kpSzLYROZ1WLjMB8GA1UdIwQY
+MBaAFFlOCjw+wyUxFg2kpSzLYROZ1WLjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggIBAIxs61pAUU0R7j8guc9vOziNCm48KwrrqzSQ7JvgrPdrZ0mv
+3JN+luAFcA+4itVU5LqO92V5PVuxhbNLOvAx7+qR32zRF863Crpq/jAux+3azBsG
+yGyjqmA+bL1wXGLrn+Tmndwc90iEcRPcOZIDreSF0mRXesBVcXEg1E5gdKg4Hm7O
+dbMZEHITzQ1/uSAQ0WxONsABS0SYgplk20IyWm2nOjtt5zw54mD6TfSgeirQa6Ja
+oht6n7EfIIMo5wWxOXnAXOMdBzN4xf3S6u57fOXDkbHcS0zDYiEFP+FpXB+eabsH
+fUUKou7YHrzno2vaoAASLpfRXy3WlHqulg0ZaDspY/0mAZxwwUL294yO+/tBsh8y
+3a9qep/mEl0h0ghj25PeWR+XeNPBth1Y54XDKSXGjWXRekn0dLMzazksvJgDN6Ip
+0hX5StqHbiJX10WpWdSOT06ynh+N6OI+VBj8ndcnLGfSE6gtL9JH3IKNtUgodr6Z
++CcyZswWKutHyyZE5vteNQFKeTidCQAw9kRW6gtGUVRU0+PrMvD/8WhSd6WkFS4X
+jN+BXrmruCSGugdL9fgpg21qKcZwkR9rYQXqRPK+nTiVCRrOzUyTFnPmusz8fg7e
+g6ONaf2xMUeWfI+F8kK4NH5GkGggGqQDtes3Y+bWQ28lV7ny44TkMBARz6zH
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key
new file mode 100644
index 0000000000000..1d83738b7a2b9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC7KWenL2HpwpTP
+q4IGLp4cE1bkY3p9BvWtTcEuOPkExtc5rgRuPzg46w1W1Ln++yQy0EflH5jUhPj0
+S93SkqCQm29uyjtq+r+N+aLXy4MjZDsGNPNYLto6R9phevloE8RClqmPFXpDGKuA
+yrPnHhZOB8EN9Gv6DDahdaxpVpDvGQQJ4gitruPS+A59N9JHgFCVsSJmS1W9oTzz
+aQoZk90O+SUazX/cMwczT2RGnFH8JRzezxe2MfcONSErYzFfvuBfFFcwytY1amTX
++hcX8ydOQiImFioPuWe51sZdoudo/fwzr860c+W6qiZLmPa1UaOGHy8gcD+RdOHq
+U8feuYa0d7c+CQBGiGiM+x/rZFeTbbJxtd4yu9rD9XVxCYNSuabyFryoCLjy67qh
+EremTzWPcsj5DhIJs/WCWVpGfA/tvzGDFsoTM0mnzf5rx4z7rMuUtyHGM/YyFTjV
+WTMq2iZD60mVP2ZIUsPjxvToQpNFRD2srlum/WP5M4ELF8oAH01IbDFyOVWGiTEo
+oasmU9vto8nOpjiVxHNknKWGrPuAz5C9LA68ZvDubjmFsE0ZXZUmLTRU/p347/an
+ZuD9+eJXiuOHmlaCMqHSQtTOn2SoUOdulmFNwE/Vit9HUzZhQ/sEEDzzsBDmc4Pk
+9FpXnxLPgX4rilvAbhzkOV3VY1YlOwIDAQABAoICAFLRl5RbWzBdcgwTEI47wqsZ
+w7F8c48vrTbq2Tji7Q44DrTvU/aU8wP8vwJVT5iM+Q+jKq0wtigUTzWK/LVZPMPA
+hCa6RmCoZGsms/BZlcXrbFLqy2OSF+8CLJhGGmb7mDT/BjjSgC+AkyOCjukOX0BY
+Hg1WwxD6ppH7yDc0nx2uLCNTahOw+A86xO6T7PDGzuHuaBJr85zd5GKxcE6xJ3ig
+ttKNbK67xcvmrbCxK3Yv1f6iFRQCOBiJWgwg8JA1noN0NMDagL9SPR11BRML/bCb
+gxDnbeVXXZbWeyCkHVJQUXFtskIXpuhasmdxYHoLjhijY+8uvHJZDj7qo97iqcAf
+jzbvnDq+59hDrcYiLAeYY9VPCyhzMTF1dybdqVRLslwGsryzIGAqVGZi7Suf6Ufw
+8HuLd2uJmkRf/g/hFxVcvkavylwpNKYniaw8O0lHqqh1jTVZFufF/E6MvH63y+wB
+2lj1PyXgg27Lh2ehm0btu/9/iuLYyNUSEuKJ9lTSHLdzHlWxmk7Wg3jOus/9B++X
+NQpCtpE8IZA4KoTkvnmsepPwzzGXgmSwnEraveV4c/59JOyVa7V8hEYGQMdeQcsa
+av7PZkgdSlYygfjyK0gu/d6tpTdbVWk6Wo2suTWvTHVosisqUdODiaK5rqwIeAyD
+wHWsTSlDiqZyuIqoyB3BAoIBAQDsUqYM8izDFLXhcstgdLfaU1drMDZze0FLqWyw
+ItcCeQNpAbev1w5FgaJn41NdYDSKmfH3yJ1rDlUwz7qI4L0PZO7mHe3AgVDvNbJO
+9w1cooQrv17zG2wPGbFosmiEzJdTl0Wbtd44AxSwSjCMmO5v0SJWbxnz/hkM47pG
+98TB2QMY30hh9fJTguVDzf8uo0Ba4IdK59k59JyBai3hIuVhwEpA+hzRMmX0IMHN
+QphmKcsM13B8mxJQtrw5WOeXiJypaEQ3g5z+fIxAP4ZPHFKLu6CR5+dYnbvRn7Qs
+v54JanujCLaQoY5lvMlAtc/ahQohuxHdiPY22JqeshdKfg4FAoIBAQDKvtw27L81
+GOEY6rC3qtcEVZbXWheytp2CzxYITXgRzDIcr64ftJijN107/n9R+bnfSCEKZXfb
+h+1MHM2ieVsGfT3WjRviEtmNXCWuMOUJJt/QeNXKLJ2D3G82P9QFdHn6KALOxE1L
+VCE6xRoTmGIw1zO7MhdZqURD7CiE5XOTQOpwJ1wxxJAJK1agbV8mI9+lGdpSDoex
+Ye4VxRVKFg+lAImy6NE+I7ILx08Wuo8tAe25lGKdnxaST4DfFSNQ97JMawJ259+7
+/4f/Z1k528UwiCBmmkPd32YRES1AQBS4dlyYo7b5d7JCfGgnluu0Ij4IAd6YG0Sq
+fDOX/wcJWoo/AoIBAQCScdT8iMY8TYU4JA7rwuh7ntG5VrMndOXNoTv9Glnq6lq2
+o53UlYv+forJnhF297sXeLZ9OhBy2vrnuJJ7Z+lAtOdmKn2+oWQqGDhavrXFBeCp
+y3lnZULdSKqrmoMyaCYOYtt71oPSafZUGGVlew+msw54SXjnvjp/xk8U001WYsiM
+jTEpo2N7ZM316p2LpRaoOiMcw/fU9kRV3hp4PUuv+50IDvX0nSqY0gPG0tHl3Vg6
+qPbIAMCky9hwISvCt6F9SWOcky6idUPkPHlaZM6yUJ78S50NSRGuoSzh3gxtAVOE
+gE3B6pt3aDm+o+weGfthU6+KfOk2q5uFZA8S1nKRAoIBAAiLZqiB7cUFGWd9bJkx
+AALUwJiu2VTU0WqT46/R4/+MBYij9sirScuHJ8zBewgvaB/64DmZ8SU1Y1DT3e69
+ApMRcCu8orIybSzB92yR7vUlyET28rukibbm8WDt67eXIxTh3sxsBX3pO8VITUMK
+Jwk76Jg5EjbbJV9fco23Rdms4LR9VCtcx107kFkOgvYOiPxiTIssxZc6M1iDh9Qp
+fdtMXIFHwpnF01kfuzuP/Ty+WhB6KnXOIJtl8l50e5qL9d22nNH6D7nSmR6wclg3
+5OWYtxTb54RQR/LsMGYGzVjA1EIylCGLY93ddF5GxARBdS/NNF5gt9IfjeUYAGnq
+gO0CggEAMUDRZs1YYiuwCMQRuvi3a9JtoJxQ05YdN04Njn5bC/0E4CFDT2eTFozQ
+JBLgxhwZ+UxFVMRLCGLzL7zaCYvwnnpzIHkTnfdPQJF5/LEEbYTlvKDdk6jqrwAW
+BhIuLxJk1r/opNWKOyAEsCADXnpPgbb+0jYuJgEHCIFeyExx8nZcCRVhZoZeFsJ3
+fO2fqc+mV+UN4r1wHejlKp10OrNEeuPE5tJuwO1/QtKVTFIxOUn0PqftoCUUwzI7
+R/2dJC5AFueMbVQlu7Rx2rI0S816sapYniPZrnv66QsDAYRic3/m84c3mVj3YCrp
+vkIxxqLwG5fIEleJmcvzZGyM1zAPew==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt
new file mode 100644
index 0000000000000..af802e68a2236
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt
@@ -0,0 +1,24 @@
+## Create JKS keystore with multiple entries
+keytool -genkey -alias signing1 -keyalg RSA -keysize 2048 -keystore multi_signing.jks -storepass signing -dname "CN=saml1-test"
+
+keytool -genkey -alias signing2 -keyalg RSA -keysize 2048 -keystore multi_signing.jks -storepass signing -dname "CN=saml2-test"
+
+keytool -genkey -alias signing3 -keyalg RSA -keysize 2048 -keystore multi_signing.jks -storepass signing -dname "CN=saml3-test"
+
+keytool -genkey -alias signing4 -keyalg EC -keysize 256 -keystore multi_signing.jks -storepass signing -dname "CN=saml4-test"
+
+## Create a PKCS12 with only signing1
+keytool -importkeystore -srckeystore multi_signing.jks  -destkeystore signing.p12 -deststoretype PKCS12 -deststorepass signing -destkeypass signing -alias signing1
+
+## Convert the keystore to a PKCS#12 one
+keytool -importkeystore -srckeystore multi_signing.jks  -destkeystore multi_signing.p12 -deststoretype PKCS12 -deststorepass signing -destkeypass signing
+
+## Export all keys (openssl pkcs12 doesn't have an `-alias` flag :( )
+openssl pkcs12 -in multi_signing.p12 -nocerts -nodes -out all_keys
+
+## Copy and paste the private keys files from all_keys into signing{1,2,3,4}.key
+
+## Export all certificates (openssl pkcs12 doesn't have an `-alias` flag :( )
+openssl pkcs12 -in multi_signing.p12 -nokeys -out all_certs
+
+## Copy and paste the certificates into signing{1,2,3,4}.crt
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.jks b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.jks
new file mode 100644
index 0000000000000000000000000000000000000000..6c5d73771197afa5aadf62e9afba59a175dc6530
GIT binary patch
literal 6640
zcmajjbx<75wg7N;aredDEi7)q-5r9%qDycM?hxF91%f-lH8_Fb5IjJFOJH#i1bMvg
z-aqaq_r0p=nmJQ*rhEEK)vvlAj~<WU;NXDI4dJ-~;85J5woXtdTi)lu`MM{(^Z9NN
zH~@@=gvN#hKmf1;aRC5CIE2@fUx7w5+p)Dw(o-xnx}gZxxisOnQ$P7>6CTh*5`BdD
z9^&hbaD?Qxdk&{#F9ye0v&L*>$32W0VpxCi1|3kNbMGoFf2y7qj<I3C=B0AnkefDS
z5AP|OhsJCZLx2gHtSb&n>D8{cR{uLd037107kpr$=PPM&C_o$>vXc)F1OR~ukc0^&
zFa{C|Yox-YC;~hX5e`hqg%8F>g4acW$47IwaCG41@UV9G08<iSa)ZHKJkJN85SYvG
zkC2yJ=-=UgOEUoDBO(9sb|B#2-M|Bo;icdZZbP!g$M027nZ4}fFx)^D%0w6+Z3rvp
zB}Lw%=-aqJ`Yft$<JQ(se7(eP@J^HZR*Wda&$f|5EQ}j?MqDwaifF)8V2bB~QeYu~
z5D04Ot3_yK``=oG3>nbMH_e{vTyFmP%i!hVz%oGUz?XewW5HeIv%@pLG<;_)#``28
zm{P`5QvICA<eF>cKx&oQeX?Q8ww?OpeC|E^{B62j5d|kX$B*Kf=tH&0?65zTn&*#F
zCuY2ZgF^&@LlGYTR_ss&SR73pXYw)WHK4%ow(-*$E4uno%)+eTkYv%0lJAD3xlEzv
z%Ws7aCM6Y{spDHKFFz;q4GV)8Mr-dY23T%v7nzPoI?(dHSIa*R`+c#KaNGVedFf;^
zaxgHF<^zdyhBxCqD__ugTB|)&lIU{W3KToE&Qd4GH@R6EX5obpRDjM#%BBu7rG<iv
zX6RA^W~Dx=m!J%Z|7@zrf%baebrXat44b09t$WzhOd<+*B_eG1iC}j($<P-Z^zSeR
zLM<81l?L$0ut)O51RX6qugSAByjAl+i073HkzI*`B;5#tmx7$_<<5wMmUrK7a%tDQ
zD{G9c(a%5TB8Kx6{E&;=P^6M^G^U`SuyMfr5-4_A5UI9kM5pHOLZ;*U8p7T(ZN_wJ
zCVem#5FK+z+{j&pl|R9;j!LwBi6IciG-E9;T1uU)Xepl8k>y^#DbfD)IW($I+~6gB
zGym;RAeW4vfFK=)nJ;7YJ<e;ObtG1niOpn&nH?${PbKREys}_IG+UILIwk^|4h&eV
zN=oyXP*f)PJ-J<EYDh6AfG_;&Me=JzeuTl}@CMU%T-gRN!#L+8dnnYyQG-MZJK=$s
z;{2x!n02Q-*=WyIi3AiN!6uup?8K+usi|w)E-+h3U7XoB!M1L9Tef8@l(AcyFo|RK
zMks-or!wgq!e=qI_g9>GLDL^XNG<4MflhcBlf{c50(%PeQlwe?jC+U5lx|v#qh3ZT
zt)XqIUKH%oh;C)x74PmBYHAmrXQUQ4+(UJk-bmh%H?Qon90*~L=-&ie_~kKUl#lTs
zfDu@Uhj^~@WO;@5m>yGkipzE#C7ouNXR?{_xpXXI0R0o)V@3v!X-3RY!wJcp$SHTm
zbh*YNG>4+K{BhD8B^h=J8J=m}f5RM%ENZSRkTVzx*;NyES7L*$3oKFUy{9Fq^Y0FR
zWVebC?);gQ_c0wZ`*J&eMxc}qaif`9hu@od@jA$gApQtE#^NGwux!*-7LB2|BT`t*
zY1^Y~w+XzS7?umvjddxb25pA9B&Olf;efirs$;DDhLx4F8x(RzW0>+W`evAB_#Amu
zs6wyl94<0TkxeR$#t;*?tSKe(`4wjN*OPlzsa@a?xMJF_+{)`cTXRI+1bWZ#a083i
zV!U)K4nBS4D9v0u(s83k&Nm}>?o+d%>6M1%hC^-p$IY%UXVSD0Yizni(MYNDPv(iT
zwMnMn0UdifCocq>nI_0Fk!tqR#nr@c`C01^H{}OMZwUq(cDx#c(=V5ji4~klW#{w<
z#u@OdV|LTie3=E@CcpdY{~GF&t8Kxv(0;<ckjXP1$X{{xSW?IEcWu=&74>2rP)3bp
za{Dk69BdlZ&kkhrgu9KU6?E(5vjd)Wm@puseXW1L4IR!Pxhr>po!y}bTcLWqW6;{x
z4C`@Qp+J~ubhJyfRAH|4&}><}pFCA!L7Jp3@kDn0Ky2~SBk70#Hq6Z=1@}}ZGGXMh
zGNEl&LRS5OzgUQs5pw=|Ii6NkL*)Z!bUfO(_d^7q$J3tHe58M5q&pVBFtJ$1pt0sO
zUc<!M%tkM%iye({Q9HjmkAs+VHR+;4+m9KBn96*OworTc`5YL?yvnfxX8HjYv8|3@
z-+reI9eF``t;+;mN&es#497<FmEC|puW}BB>QCALC%~b=k!Nx0|IH2h!p$8R6AASh
zgvc9A008|Tyy5wqH(-8V?*H)SkMM8a{5vfaK>ME^JYx$V3c!YY9s?RM6aauLm=k0*
zCW0*L?VamuC*d5JVi<|_uz>;^kMEFeve5OmkVcnMxsH~hjt&oP9BcOD36<kEk${G)
zTkQL9yUZ6C0*hR~?w3jZS_-T70q{&7L<)^DHk3gm>G+1^*@yE3BmUB)uJ9AR#+HL}
zwHZ-nAA(0^e@IW`LIb9Ei0A}QA6GNqhif3zZ<z(T5)^2K-5E$-DMSw@OMIQq?&PBs
zwoxZ?iV9`DJQJsD<nHaWdc20gBn(hXn?ow1@0JzE!^*Dd-w54>yiQ3LG$Tn7843}n
za)xC9^?lTxX``1vrE^kiR>YfMnqkux@+K_%V7Gre5&W6GDDr(sRDw4f2oHb*{Dsa>
zZjLf1sd_Zq-9Cc^dOB-`Q{p}UIf>6zdY(i89Ot#ElSe$QB5JIdhuKAUi95fXH;c>1
zv8##iso~pG%cS%RH;>rnka=}RklE}K1micv8mFymIOzvLx&Jn>Hkn+GBLV1No?t8i
z6>gSMOO;p-p@<3@YKTTtS}o-|mUEO2w3u2XUt$aq<efX7q3#A=Dti!T8T|a7p~7jf
zkJoZsCYSAhf@(!M{!`{+fG@)SvRg<UM@ZBW0AbJmoCIKAI4@EUns}9@v|!O;Yf3zz
z-yG#kM!p_dv(veCfW85_U>ry`<3gdw(=X)!3eDxeXxbi1<S*F2(>^D)ErkCBx8X<C
z8{TGFC&Kpd*=th1e`zi8_8sGr8|h84%%9xn{+IjC&*$f#I27UjAKc%2X}d8^6u@Ch
z_lC4e&}gw(H+A%?q)E=eApIp4574=&dmOdr$XH(1x8o5TnZC<lxHY!zPGSg8WS;28
z+TmqsEZIwrhn@{LRU#7qfqvK(_{^4E(?$IIOd}rxTda(Ud+~4ur8lO^>Op>Mwy1BF
zK76Az`m#z?R!50<TLkBI$;`)SSD_lT19t`++(w1bh~gjwkzy&8>$>;h33S5Aw{9jP
z*a*Qup5k!VCVnB_<6V?+n%bB9b>|>#<x?k@%I?j~ktw<$xz6^&6%%`VGY)y)My&WE
z<=W8Y=m-L8RU|r!-CNk?3u@7BEB?`o;x~c05mf5@#j&~FbvsH-(;-73`n=jL10~h1
zGR3I{!y7N{E4q%;Wa(=Ug;7N9FeZFKXlYKQ?a{<9F2=7(z*=d2Pdib{zDmWx@*m9b
zp99Oen~+03e}(6BhZ1Ok@vb6Jwcy!d&?ntcv%Yfl*v2-J((Ao-=pOUo%5RG<CZV3X
z><71m`H0|kWm8O38*$~V&FDGH(Lz^^%fq+t8D(u<4^3ITR_jG+)i6!oo?=67HBVC`
zK|UT&>X3-yMLrY)!Zr@yV1X%W!5fo0?nrB{dHU3pK#e+SvePIn{++olNF@MiuitYF
zsae|AQn{^-lGdtrcd&_cZaw2Ny^Vzwt{t@3B!$>HsNErDK&WJAL@E3$OTk*q(<O*v
zwGK!2DNj6JWvQ*S>SeqZHxPrzZw$SSh!`c3b@NpEy}V^Pb5MXzjkNg?_-F+Y-W9}#
zz-}<egx%Zims3U!Br)~ei*>J)?6j})VwHPU7lAt2LR19XXo|8?$tEcOlC<r*8x_kE
zB`17$|M9vbEur4uDRrGq_#qfSfSCp7tbj|*LCP$X6A-$yIl)hIkKN3HvmmI}HrgvE
zCbe~@iP0MO#al8+$(SOj&DPWYW#(@-AB`QB4385*oYg~E@we#R!*NSi0u#aCB|L$l
zF~sgAc$*@eb!8WWbRLclih&CQB_V|G0H5ul=-)c;8r^U6F=hz36d4>zFsAvEVJ+i*
zESw`@3n;I9(1^vmwu;V$e|553?Bk`Ll@vSxs2j%&v9`<sI(mEE+9Tn*HlJ+2^~}vB
zD%flmsgs)cr<HMt$NCLlMOSYrUndB!+(fi{NLY5+*-Ra8Y$)WuWzk}~VRfZxOcTZb
z#ZbfeIe!ICJGgsOSZuCJ0Y(-QS$p&0$B&#j$L{mC`#buq2R;$S;a~<A^}vF>iWB@;
zpRBF@&rd|};1-o*7)VX5WpQKHCwCFPxE+VdmuioE=^r`3llAA~YM7`;Ad5<W6JZpO
zb02}3LAf=;(?R<R9$#MmizVa>iRg}pvzB~hjm)szD_{N>st42{*~e9Q#ZUvLUm-Wa
zqW3Su1G3w^f=Kwdz4vtky;NnNUdKO4=rr^h@uuDuTv#ZpmEi{lar@O!f0pi-k&|&f
zm+aMX4{j_|b1bI%CKf^RUMt+hs=`;N*4e8g+I)1WWMtlLynZ!y)7-?bO5`mes@p41
z`?mJHS%ScsQw8tbxa&UO9NdY(92Z=C(Z$u4%!3KE#Ru<V2Yyhj^_@5@6KPSix{+?@
zOWW1)EYJJ$+AkdhR|Yq_9kOd@0Sl{$rG^K0(vd>!z%fmGXyH6XX?cb$quC2#zP8i|
zWhA!$TYXu?`Hz3`<Zpa)|AYEGe^&oF{BLRhpZc|oMVFOy+Y(OI8prvwo=aO^g&1zo
z&&A$&)S`KjXqYuDDC%Q-25c6gNfq)fP}FA2xrQDYKiLziAWGk*+6ea91_Y(e&dXUP
z>tJF1_l0Z*du}sGuOV-#g*`s$D{DQ9YZkWG%QrP2dU{rUiMdPOQjYSqY!XVpzmenc
zAUc|!U_}b=@KK|x7{@=fISIojJ7k?GN9R(n<Jv?*innM&j0prWF(T)S+5z=w`E#zK
zdJ!MsF%L*5B#29cL4-JIY&2=FcSI<J+j_$XTP72jk$OD~p-#iDscW;^D_Q~ex#f=7
z!_kqBsAQ&HIWB12@1kqGT`{y#K_Rvmb-4ee{_oVH6yahIcg+-in9W5lY*Vsb?=MyQ
zL;Vki6UH{b9Y0BVl6n?(Ac?u8Xm-w8ABxHO#T||yr%tSKO`Bq8r0CKyxZ*s?cO+&9
z!^Axc9T7#{E-5Kxe{sCD!FuxiI=3@&BCJ4LKv2XV8gjn3*R7E{H5brRm`bQ>j}F`x
z*+JePbux@v<3GTG-)pJaAcTQ`T^^jQ>CE41ru4WxZRj%SDRcv?oF#b?s}e^j0>SQ8
z4_-o3!|hE$JwMFwEXeX++^8rtiQ3dj+Jw`&wN5US{wCqL+XU@*G)>WdnWXeS1fVk$
z<LFcy7*ZqjQnXX=vqM?LuCKnaaT$(0y((ksH)z|1(gN)!IUHj)efE*nM)x*<E<Ug8
z|6^l^>tE_~xjw58cvk=6AJpGJn6t~TUQn^z;$50=x=YA^S;Kx@iMc2KVsMq&NCa(|
z%@8e_eRxS!ljR{knlH${Q2b{ikjkh$$0v^4D#|UR0gnWCj>Rz`O^fw6DFbZ`PV3}n
zO8gLMQVPN2In7qoWLErcxdNkXvYH(`t>y)kEfD_ri-1@^tcVS}K?qnWsOnZIjCJf6
zVH$l`uHd*A#z`wS){5pQP8)&Ns&lo;A9dAHH#@5<hZ9LU9o)?+KGz!C<9gQK3uY!B
z)=Q3}OLhZgk-}q9hzwO}s8_T<8fh^1R_{n*Z|zc;dOMi!v*q+m<7lMN$u^*ho#Xm0
zIy&K0I^?jVCL*55q4_q9>vjsBUyU@ytw*1FHFR}U0DOmpxg>VBF$kwnm~qMG+Qpe)
z<u&1c^SH{LrEw)w@4?YF(&{)iB^rw69xh$gxIJwYq!?7IIFsm19}&T9=8h9Vki}IK
zyRhD>2Pc(34Z(_mVpG{u(>Sx3Tlx<2qtNdOlT3s2ApJvF1z%-#mv{B6Y-Zf(h_Uze
zRhsAf<*n)SoD?Mjmu#843LPFw%jxPrrQWj^$u(RDYFRVxBj(E8XdTvEn%>9w6$ao@
zmL%cmXk7}0c@DqtS<I!v_?b$xe)P-1{zY{)eSBDmpMZ6aeB}_LH%H=<2%IMEx*<rJ
zhOsp=%Bu3!lBlj+v*~N-`7Pd64qOaD@q|@j;5)=D8lk2Ox7SIX1l>I&1dKjt$cGho
z{8qdtOpUvA0l9}8bC65n=vCoejg?qZfzeQO8Km(f><f8EmSzxVui%EoV2cV)ON;ob
zkvI;MYExlHQmw+uyhNmcm-He8$w-}$uK}7P?~7U|!1&E_sTwP?Xsg&g6e*b53zQj;
z%cO{H;Wd9^rJ}`%3JV=^H8`|VBBl9iuzK!K=krq)1DDt{+)%X`5!1|(%$_w9;oemt
zr*gw^VLj<=I#52D;6Z3mMpz4<wOUV)`#b@v(od#E&pz?R%Z;J%8=;i{q>~&*Cc+-Y
zjKx={(b+rm3vPNTLrN>aSLM6`4eyh$Bzi30(-*I*V8wdtPC=%}I_>F3(cM_NV~Rcv
z?^o1(ROx$+cq*6;XIBcBkROm-U(kdQrJ;nj!rpo4N)mzS$#D2mV_l6n;C>~TVhR8|
zXzL5=%C^yET$O!(m7HmaCV5lZW*Dv@MdTrq21zaGLz(CJ{WO|dVUmjdEFJ!k2Oe#P
z;)48^4=lziKRSG{egbcrRL7z{L<aTtqp~o57=mmB5)?i(2Xc{bbIFQ$3hIr}O5<fH
z>NZYO`#Sm>J-IwEdpA;xT=J|fWsv{8@lSLJ5>nF#6?Jg%HGsX7EjCWIali)hiOs*;
z)_zXF)px$jY*-)9dw$?ZwtPDI9@^x)7?f-?e~w7C^}z@8*x$C_&-_YIs(CLgG}j+%
zZ8L-aiC8a)J!uvVOGa%c=TikD)6KQ4YDBc0xewuoEiwF*^n!XWWj<Q2SzItS@5FL?
z-tUk3es}}t)rB>hmk#_@?t_&4nM{C&iHQ0=OXDs4GqRi2B`Zk3kZ>0-Oo62BhIM0b
z<|+hPi3Ib$x3v3vJNbf4Q;rbhm&X_2P`Qj{d#thH(DQ2bGY!f9gV%X)q87+vh+u*b
z+Ezolx~WNKReiz1!Kxz{@@`LqLo!iUg;X`_WdAq&$R3jK{sPM1{O0-x_PKcewAg<R
z|6AJs$Ns7kiomUWscghh0Y(7xUJ)a(_Pb6iPux7Yo3YUhm*sWY>|DEFT5w!i3va^+
z>*A*$2r-*6pwJubG(_un^SOC&KjiWzmPC8%coYJo>AYw*zQ%3cAK!RB#%wAiBWo9N
zOlTEu<H6^51~d)3C9{A&p*QV#aDpex15`4bS{&EL-ys(C*l)F;YWoiUc$x=&z?mgf
zy&jB^#Li-SB)U4}avskmY^rw0DzLTTTz2049ZA<h5R)S?pnAyzg8L|G=Cc${Xy;t$
zX@`U*P#oFml5%i{_(-;EKK6>2XcYa!@+z`ac5ZK{eKGG>-<-FRQ(SxMDr@8r^dhz7
z4CTguV!z43F)@2IiO9{ojY&`j<2|j%Tf)Co=@0fht5;^FFK};JMrUj#6G%`!(@OZM
zw}G3@5hPV_0yZpyp?a9(KQ(f2fC}8bQ5z(vmNm6;J7o=9Cqy<}wdyMb@t`JR-UT_C
zg72=ZxBe+w_A4~&i^6G)pZyhY1JlLh5XLT4uTnSO4&@OazWd#JS)<-8wPkn2!tPC<
zOrJruX;rU~DgCWXF4s`tf=ui8kb1aJ-&%?fI)1oY!AFV(8`W{KAkIuXhk$uH`Y9FM
zMhp@>85;=uL&obcC1X-lu_g-QijbPOGpEfINK=TeZzP8$pHK>y6v9S=*{GZ-V`r&`
zS`+E}V-Nd8lx_;xX{+MD&-K?|49*CD?^Q|PeUP)u8gWp0C*sep`=w$?{7Qhg@_zs@
CU7=b4

literal 0
HcmV?d00001

diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.p12 b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.p12
new file mode 100644
index 0000000000000000000000000000000000000000..a08be5d0a7206abd2b5cadbf2b456660983996bb
GIT binary patch
literal 7462
zcmb7|RZtw<wylB2-6gndpn=9ExCeK4cY>1ujYDt^mc|Jlg1cLA4UN0I`^h=?-o0zr
zefaCVtXVbYoU7J&`D)bwg2`#%U;#id*%BfOOSoM49R@5CECft83J)e5`YZPV!Pxu%
z(ZXJY2V>9um8XEgqljq#cL^O877pSK^aL^^qW(YW@4ck(_%;--7A08@qIbY9>-c%_
zE*E2GGaLY>R~24vGbb5}MZIse<w(tnjAKN%-dX)c|7tCVXPl~N<k@m2Oz5HJq&KLp
zU-w?61UhF`$vb#ydKL@%CM_Gb)vb?ry}c%$vT%gODFvkb*BeprIdLhlVLV{$U~FKV
z|DMi&D+oya?*Rf%e0W?<J0~kDP9A<Bw*Z#_kQXSx2?Qgk|EC`?{1XrifA{};`lhkK
z*XlFrdN{YWt0-2d_--0G{{MIa!w*S=q(JAKOZC6Z;La4dKaOdyCecShZ>KR1jo{^}
zx3->MPv2T#QE+jruYr~)T-dE38txwJgiN#1q5^-u6iE(D7jryh_;=novu%+CJN1ND
zeGSqy*eRKkclJg@;x(_>Uu7zxKnv?%G)7Cs<||f-fLrBaW>qT0;Ewn+{O(`&jh9Oa
zM)B2-2CeL_c%5NE#uVIRxia2ypvcg&Ey$v#Ipl(uNn7+1hYi@9N_xV(F6ir-rja86
zGt~2>DA0FUimLMFjKeV<Eo%8}NsP%ms42OiwDW=D`>^oFaTW?{D+;9pQH0M{oG|&L
z<xg%n414>5-@wMwAN+RajQbi667_n!I~MD;U85TKzYZby#ILx4{JpOHuDLte?KBUI
z4|&=*3NhQuXnZrX{-FJn9MD-$8p_-2IrqkN{twf7`>Wx08M__vC<L$ihPd*qYiwN$
z4)Os_>{($E42tJD;U1bEJu1+r_EPPniV!N4s)+z{KRsD71t@{IG}mmwZ$3<oP+8lr
zz_5L?hn7oCaha~n(kiX*vUsQCLpN(FRiuGAtfKY9r&tiL6qpzHWl7Xj5va$SA07HE
z{N-2UhF-NbXuff&J2QXvbu3>LxnQ7fGMIPe4-1;%$J0kFw^g%*<BpU5`8A>T{f<vW
z;Y^6>?t1Sa_q-#?{<-swD;zuHKhfU%teHu|LM^*W;@K$aW;aQ@(nBK8^TuOVM{Q1{
zLW>VwRMf`j56t-XT(PlxdJ@ahZbq+(yHb`1Gpe!#Ga$Dnq%mlN4Jga<8p9YVc>K<q
zlxZ4RpYK#<&~M~W*<3x|><BZHQ<}?1fo0BimbFihV+A{aOV@<qHDlkZ7PKZw)O2%<
zR<EB@znWTW3M#x4h5HIPd)GZQ<Rsib&6HX+I63**EZ7j1Rlf=ZB~VvbS2g*8lXVYG
zv%Q>CRc#__co@}~4nDEqczQHS+?Ppr)V}8}rKN6MY%wFKMuNA)jQ60AK<XJYh&hGM
z(I}G2{+xF>m|SD>m+-TM;fnMSQ;MeQ9;{qLxW_eX0SGuId5N#QcGccY!#SWkt^z}0
ztCCa0!0vNbNnmNw?^2krSk)JbkfUeA?AzifbU{4FrF?xAm#GTeYCi_HHES7Y(@+38
z!zKURejIgI*c=%(%HC*MJ>n}VY(N%e)`k~F#t8yt(yad={D&Rz&QO`id;ynz4MoPI
z@J4QW!wQjNvyB4@Kt;u1iLN~ZJ|1WK?DjeKeinYr`xk}nSlR2CbFPs@6j!KDw`0e?
zp$z5`&2RTkb2Embcjk4}vz_*4(d;+;bs14zgR`i1M-?MWd^{?)6ZAIKPbMWod=tvs
zs>`<R3(kH>L+BcX6`rsGnwc{V^BRTGw0gGUf7H<S<$jpMi9dW)y;tkiwFA>xT*9Xr
zmk`eWJYCK0$S^rJC$5v19Lt<6hx%?8v3kY6`)mU5t&po``^g~?t;t}5gI2h7A<fi-
ze$5r3_|mP&3O&9n{+%D>c>qdCpufeddZ$uLqF>8*ujGxaFCt?Jl3IJi++-CWxX{D-
z_EC8s%wnLEJS#XFogO^bzu|1_*kU!M5l-szV>y$FTr?Mxk6>>m6iT-25sR+QJ`wTH
zROS9Bs&a7){4c1Qvf0Is2Y?4XL^!Q!Gu&Ap0i5dpMb&0q0p!s&<j!nrP=AaG-<cb6
z+86Ko03&FjmSa7V|9bWG9Pq=>eR+|F>IAgN-<?BqgEF(&=ng|XeJYvha~cmxHD<`M
z<n7G}f^J7~TtTs3A_5mjzF(nm3w>5%wEB*>^RfjmXIE`Elgq4uxHs*|>M;CwUkz!=
zmhKqS2^G4CGG_3$iKT04=kRKize4p=RiH{+M0-E^2^*%4RiV|>5kfCN0=~p$lr7_M
z@!IAb{`onsDt&xg_9R4CoF*8)F1fQ|ks1OU%UXc#D)5!#9E3UWL%G1h160h)&Ht3Y
zi8)@yrRW&2msnYWPimjKpj|AP3L_#%Q$CbfU3yYZpHXABRaC9rq5TXiR8VOD7?P!4
zpT(yi7G9?KqqPUFvfMz5L*zRX9viGQE7KcGb!10E(V5+!y7v_-1Y%X^Uq?sZ$@Q2!
zzV3k^?E>1$8+^YW7ljUz4jwfObD^t7#GWl1a0}UEupxLArxrl+@U?-I-0aiceM*v*
zMe9p&GT}LHqWUB&Cf)~anGvQ4f%<biJL($OU;2soCR$R3A~vobqMEHK+M|Hz^#+od
z&a|%<?c=z5xQYR7KLER<y`78dqokM>0X23CgNw3aYXLsnx8N@(5661z!M=wCoLA@X
zJ!kI7V6(+ho<3p465Gw6#qP(B2FWNGe2@ZryQAEue2X>=Y9bu5upk-NSGzoIM2df0
zNZ&(YlalQFiARr?9!}>YoE}PfP=L=$_)P$0MtMl~WoE$mHa2kS<++{KM2>RKRT9HL
zTp+zbVrz0Ph+UT4UbQO<kcMIBUgtuC2eG)ahHonm_BVJcj@TLUsi0%_fv{GEK$wJ?
zzrfb81w9viGeP@6J5(?gxo1LdNh93qTjz6WucKr|E1&hA@<kX2)<E+^)la2C=3u6-
z$~Dh)PFVKO{{1B}T{XEoLu@;Mno8Tu=S=t<ifZZ(ZsW`g!uyzDbv5CyS<ef+Y&vq=
z68<lyO^YAI6*sXSuFCMb6_viE@l7OpXmckSPKwa5&}{9ZA*+Qv92PaI;}~oB@coWa
z66bc?3>PYB!#_-_@g$Eg9DMbCd%$XVoI%dW%Mj|ObedM-MC*U=d2pHNFq3EDVQBCD
z0Hw^YT6e-7&?#S7HV6;?K`W11bnx_19_awHoJ66%a<g@JV+_5bh)1cn|F(xzS&#kI
zI`RN}R#8I#(k`cSVXD~oG)@E5^SM1WfLP}<8}Q{eNXnLDMA(ecqpR{mI#wL5?Wg)o
zPvXlH1RX3k|Arn0`RsZv!i#A20{P;Ex6cvahO7j^XqLNJi+<1t(o2z;)3z6B^0R{C
zqAo~v46SMMN$LkX6rp_0rPA+eb;NpFle!K2hGObe)<vNy=PM%Ab#znR(^;A0p}JFO
zEWAWQ0Ht_s?4lmJ3ezTYe`wk)H5_VE3$u=yQnIdeWCj71wDRtiy7Ok~^_=B>9O$D+
zuLwf?T>PvJ*o1Ot4UK_8*+|_VCu^T(6-xA}wJB(fsHl)q@JDAyvw1wzU^8tA6U7>p
z2C1M6mSk^Fr)iHq8q?Vo@{d2cVB*Sb12<Hb#h2+UgU@kEuNSkGc4TIly$`P1;`XrT
z8&Op})i%NPxq#mVrvFS;u79K||Nn-n_g8O3`mpL0lntju@eLrV$*RQae^GVSK3h9~
zhG&@eeV@B{q!NK^aq99aCQLS)#roEcF^VXekW()UG9Wx-N6yC^c!&SzDT!St1;<2;
z<s&D`pacM0_BIm*q?R6;bG{rVcQn}QyYKWzA+eggBWtlqy)az)-5CBhalP63?Y&?9
z-uABP@a#c$Ws+^@`O{KXMW88QQV-G4nbO~ap4IgA=uiXF`jU!HyJ_#eq$7%~_Bs#u
z2|{kY_DodbVJ}zatUL8g)sr_oiC&`5%Ru@WNYSDz!u72_Y&4qD8NBU~u5z%2Fb8q{
zE~;Jg-H2XouM#a=Ztjj=Mdt)rS?|?Sa9rnwML!)hNVEO@=;vXFl{1ufoo<xb_xh`U
z1~_nW#;LbXt8PmhMFv%zw5sn(YHK|{4Kki_sQZS3fX>dTxG=Q?rIYwrA^+5`eAalr
zRKZw??yCl|lFx^IM|scwp~c#MEia=zu*<4wUAAe(@`jqjN_*q82K@mdD5P#stDR{k
z_TZ_WwgSCpQB3v*Jq>!Mo*iA<xb!*RUa|`9R$l_d6+RA99Q1NrjnnF16>|H*n=Cb+
zSD~w}bS6X3ri7xz)&w-9?FSbosWWX@i3o7r5#Yd;Z=u;#-+h-O=y`ErBx~mktvzat
zR>V$1icjp!c-}D7uW?olP~AyIxpJF{U^VqL*&MuJF2-Q0m9;B|=fdn64o)dV!_l`H
zh@}(7Ph&&lT3&H%CB1e!(m|aURbe<ni!H0)i7o{$W57IFdXVi8&?-qJMg5vdhkGlS
zW*;9BV{aTn4cR326rF$HYH1YoDbX}5eY1vcD#+Sloy0pFzaki_o=E@<yI%IdAwv?x
zJq292-g*QcwVo)b;*9ZijxaCn0CvnI%pk~D+zCY{059(Hhu;SnHIoQi{zkB~PcDe@
zdL$jofnUc8S+wV{JK3Db&KdWmvJI){luxWck14m4PgltW9_*Ak`%ap+K8MrZl*s7Y
z)>iSi`tg}KeZHbya>})q3$vD`N!Fd)1%p6;L)`Pm<v~5WCqlvrX}TLTl=3`;tIC(+
z<baYdzY6sGLJY6$zF+0^cZyVUlI1oo<OKr4($BKpV`;A0ifbRF&G1QL3K#}HFM%Ij
z`0)qa<xzliC8>7x0LnhUF<+ZhK}!4T7N7df661wHt7uH`wC^dT8H@B(v}*X83^rGv
zoyn8w69)NWTtW>H+igShz3>7HX(@~h`x&~;@-*SX7hGc*>PQv0t8Kk2FV;jRwf0mm
zlJXT+198z=>`U({pF6A$c0G*}+u>E1Ga^YHpSzauuFPH3k5y%=6eH|$rROo!#8?Yo
z$&R$YWx%B74Y=^#B}^ETi7sd4aaxYqzt6x}X;x$!A&i2vdALut;uxfP+#Jl|1iT*@
z)>YRp4@}0Y>RUm+`_41ne0NMO-zQwO^;7Cvi_$_=`H<l$E4al_BiiU-?<>n}PgNZ%
z++|Dxt#lS;culQ|-N-QJ<uga-m%6h9Ox|yrlGKydjAmy%^<&Vo0F&h`NjS2k6>L#I
zR^8Bo^4KF-hC_?ih?Z8@yq=2K`2rw7%&J^~s7n&B#%tptnyo6$wI|rKe?m_MR?8%`
zQGiW+iFpjcI*1(zdz5LG6uAnda5nj8s&f7lRk^uAKrq_uzhVL;L<ktI|F6^y2MhDJ
zwEVjR`yYt_;(w<cKiMnrd<46)W`4)^I59x4<HX1D{)e(bV6>n{+QZ$*j<YX)HV!*c
z&b!>kt^@>@?g3}z<2igPJ?l(wt$YiGo6^?N!t-NUH$bHs>Iwlf_iAS&B}tXTsF;z-
z#qKXYtWz)m5&N;<v%&&P7n4D4MnU_#W!{W>w>raZvRI@_A#8CuSuP3}f{HFnnO*L;
zre{Spk$Un(?D_X2^7N_w>Ju_Koxa-lrN@A}jW;j6PW`zB1@4@QLIK*m+|D4|WsyG?
zUkPq?NEDmNl_e^Av{!+x-X^%{#Vbykh|V%8@B3JEjG5VnaMIRaW~!#;gz*b>1Uur@
zleQS4o{!$pzkhQtv^V+qHI6NZ@e0BX_7dk`>XQ@91-e_-=_crUo?An4C+>{A@0%za
z;Nbh}X6Lpq^YU1h=WDHAy-qXFc5O<=y6h3C_KORnH^Rv1>+f<rNeu+D0C3+wZoY+x
zm@;XrP0%ca2hm2bH}H-98S2g2;(kXa*9Ku{i#hDL#^{riB+B=ibDf>ylJ$mJ^g`6Y
znOqWDEo6?Z<MOXtFuA%OnctWVI%B&m<KqTR6&;mErE!M@>!YISY@Nr6+bTXt^{Dm4
zcWJh*=Z3*tYjPx_&y^mv)ZV?A%khmRE|_VZ&D-?lO(|~VE%&ThT2MpzfE(u72ciM0
z*^RphtmHT!u)=JZ=PP<_hvJmR)PGZ!4H<3RZswPeuFEs<;$=GbUc$|(2&N#Q*i0%C
zL}Iw$U$^6;?@Pf+KBV#1%?nbIWfIsN@A!klx<nI#1E8}`yZArzP1O98x(VWg_Z9Cp
z0K-GQ?-~s3YtyP2$#nI4*<!qG=RPdeZ%dzTT>sGu^l)dmLl?clc!%UN%WE=-Gl#F2
zwe2Di)GCfB>u%L+-)FWDl87aqybS*4n?2W76eZvu>xi9vN{fQS++WV?K8RsWgSNQi
z(~&6T*v7y`@Z|dB^c{w-<qP^Pz)hMd>Y!VAXTh{&&x5NjwKz(Qbh8Q`cF6y`7D9vE
zBbnCx=T3r5c1iD*Ejr3iD^qIfHxzjE1GmV^tJ?!9{Q5jZ39BhzRlUfkR3JwQZfq$K
zHFf&0De~=+6&MWqU9bgc=7p3CZZ#3@gw1q6brUW@IX?Npem_2I9{_Lfkk)a}zR=Ky
zN=4bkdr&^cj6O586}IF7VfJT-YW6$7ot<-klTG`r<%YKs5`Vn@+V4qMe^}!`bJ(os
zBAJV7WKs)1=geYDpr2qzl+D@KHv5xJh?_o%gn8F}>_ms=Gnsb%{%8x|OB;Ktu1&Jr
zlKb%C<Xw`iVH=qe+ag7Kt?f6+YrF&D$8+n~X#|6K3X?sb?rsYEJ=RwRYpu#m6CW%6
z@^t0jT*D-t{5eZ4qZH62?EI6l<3YPfg5(2|ltbNyHr^{Om4ptWO9z8#GLcw4MP@-t
z@5>JTbCdRAy~-u)nC9FTncnA2M8DsN^nu)6Bu!UcBr70+bfny^0(W(0x=H6d=U#$i
z(S;7el^c?0XIaFy<nCb~Y@CQsSgn-9cbNUk4Dx*+WQ&($dl*EU;uBAdf<v4GZKOL0
zh+$+C(NhG?uqS!yuh4XRZ-`GWmY3Oe_J8K~+pW&;sC{okPf$gOZVM0(C6c#X{FNoh
zKA|9w?Fqrf=<2%I&4U1ts`&OVuRhZ$01pN9lxcBDRT5Kf26?7@hNW9>9E~spBrRKV
zVh?PySqG843dH*;YAVL!wwBf&zs=2ziH3V{U5nZCh2OtVAb`_ZfPBSwI+Qs?jjf}C
z@KbxQ&D}aOcv)_*i6-yQ3QJ;OX~n8K65@YkCF6?Vv2dLA_@q@;&iMLk6PAJleMke4
zDiA3aMrU^~E5kYSS)+1gkBn=LbD{rK6!#I`YyI1r#~v@KA<}a(<ej4!CNsQF+JG4I
zv^n$U4>uA^5}O0r;pAwKV1^DSlA*{GIMCHjQy8~vAFG10HsreQ`a8RJzIaz*8Av)Z
zg}SmvCl@{e@d+gLdXaBDXuKz7uDDcU9Zg98(z|_pF~uElOv?VFpl3mq4zC>7cMe4S
z=6raZPuu>B)J(lj8=aKD8Ww<luX0|h<NEyi3Oj*U3%0v<mt_&70%t>YO(fsn?;~)+
ztNg^;;D54Z#~y*P3@DPuFI}>0*^ex`XLbFso2Pk}L|sb9FK`Ty61ULiNd}lYTJ-p%
zVUOV%uEZ!&O|!LE+)=By2Q5je*FM;}xY;J{J%oK}TCOV3Cv9s-ait<(>@C3E-(eHS
zh?OwNM8w=&rQFd{{L<p{!t8`zvij9zk_^=3=ML+-=NZW9U>e>=el0+Grd7I#btZ_)
zc&22O<Ng}JgpiGqD7nho_kEXafXBJZhiuAt1R%(*P~g0cheByYYJFI9>fH9`uSj^w
z<nF26eX{V-(G<#H)ErG4JkcE|jauA@J<qz$=$iCp8Jm2>=2CIR`z~+Xu|tN^TIYF~
z=(nJij|*~2&U=Mgfj9DU7Z6VYi;04R<nKhG+d4~vWtDd}yN|<1>R-_N{XVcIQnIRJ
z;o-=;eHzpL^iYL4^x%nw3c0lVP}3T;ws4uR)>1J7v;r)N#U1%bd=Gw)$YhFtZgiLp
zp%sr@shC8U8YVSjff^P}eG6sfOm~A7I#BY!vubwC3-lEmzWUa@l^DG9M9;*-(2X&t
zG-*?XqPF8^bMJ*8s^KAktMp!zZQM?9Qm24@r}?6o-RIj^UBbz8X3QKG0so1aRK9lY
zG=WpU_A^cJePL4Xi&<JYma|0__L#gM10HhA7z>AF@@-;q`AAHOB?iO7Z+Hb@A%;AC
zid2<nN1W=-k!huGx8K-(O^PyC4H1vGgQsv#_QNkem5>J&a6jByl|z$j*kqM^HAO%|
z$T#c!bhQQt{@0TZW%EDR9?xm_USPb$;tul?OJR83OF<QWy$3mcLdze?Ula{+P~9So
z-KVg#F{Fu*N}nlQk>x+HIE}nzu2mqm(GWd1Y6b9w1>aoUdcxrD5m`SS2OU8<!BEF%
zhcZj)3;wKQtcuV$k3qqY{Ao;68~YP3{aqYSpvI#?xjxZo7e&C&dWuQ&STD5~9pW>n
zUg$=;%JlP5kflNafv?&%g_Ik%v<Kh1yiPE8R9SkrRYb5uy4lF2T5R=*0oH6z%M#b7
zO@n96W}o5NjQU9$OmlzE?Y3(F!@;p(b7+|THD)%Fl`5<7T!1*1n`H~Km2_k->(&6`
z>Sn(D;si_6;;akQl%xCv_F{<a_sq3nrdpLiRbl51q7T)BDyjttnXvb09<UIidz#8b
z!)F^6H(65NN6CoDhD)Lw-B<W<6v62G4ESYEd<iZWmECI=@yZlBEr|ytTHr8!guKf=
zC3(H-k(VuVhn0jud-Rc;K*}|7pta<TciN@kS6h92Bs@2RabC#T<W#&!PCO&h$`kAL
zdns7Toj5XB;@g|%*f||Vw|~U!s!rwGYKH6-Pybn&fOriuu0b!?xuf|veeqo>+6x6L
z)+sthJbqgl=g^VbFMDX*PWOUDnvF8$LLEagGK38QS1Ht*>^2Fui;M%rWA}g99k*Wc
zwi>J9S#qq8`8gJdilsUW=k}!S&Wy7x8Ww46AB$(vdmZ=<pH=%l#>h>~ko;gK=%=0u
zrd<NuA5=-&7vrZ^@SuzP&aQ1Q8sR8gGe)KOgCYo~EGu!%Lx1UugmU!0e~N30#0&L5
zdOOA02bN%kB(q~#G}O~PG!>%Wlc3CNY2ps_Z{OIb<8eY*MCQ%fBx*mZxIwZ@2w!PD
zTrI)yzTC;bo}(E!Dxy$Lb!xjd#L3<6<WfQ!U4IPF<+@mge|CPrE;5cl(2x0>q60sX
z9CDqB6@mmu;V?bsb|jQE|E8c6XFiP`N?cG2-TSrbhTGny94Y*%$BBVVA~FUO%KQx)
zdgnZw3`w(D;xwZ5_ZlC_>NQJ?g~@@UKx!ZgA_6NW0PY<E3_Lyw;^f3$qTW`}<vFNI
m|I;BA6pI`l-|vqJy`insU0La!PVQy0#Q`5U5e@(w4*4(U;r<N(

literal 0
HcmV?d00001

diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing.p12 b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing.p12
new file mode 100644
index 0000000000000000000000000000000000000000..5e8f06d3d616a08495b6e2a486386303beac37c2
GIT binary patch
literal 2419
zcmY+EXH?S(7RA#cNRduLks=5N@J|Uy1Xg;JDn&Yjv{0ge(m}~#kY+$B5|JiKm!YGh
z^j@Tf3}6_VAkuq4Da-EJojLR2y?f5@-uFIUfQY;Y0Yd>IQWAl@8mk{mX8|*S^N2_k
zjELku$Ls(R?)MK1?gS&k?as0F`B)$>{O^jH2@J_2!hQlo7#)y8F#R8&Kj(*WbvFyO
znW5u?QXqHP@~3w-o*iPYLZF~_GZ+!pldTjcb=M}*Y3BRcbdS*&_;tseObOK(Wy9Js
zS$z|4vVuYJt8jnCzGw16)?M`yWI^Y|$;8<W<?xAaO9w9hXf21+JujmkyR$Fp*0dOn
zE6)e|T1~3h<P%gs1(jcJR`Ubz>+J>SR84nlTYAPY^!k)OKXOv|#alz&r1QhEVMo>)
zA_+v0vZ~qwk{~iO8-B4{uz9(q6QRdx;03+8(mt1Z`Ma8SgC&Oe*^tvBHXxWBU6MKK
z3mfg|Vs^+)>)uHOVt#VN%GQ#F!u(vRR{3V@`<0%TW~+5;(>Fhq8?#0?d0(hQq_WQA
zhrXJo-bU@^LyWieKK)>zu<fic*7L&FGGFGUip5*7W(p5q)M7lc+<Er^EEuT&4vl_v
zqWpsXQNDUqjgUq5Qt|D3?@r24Y8-xq-qhyXJRz*)5V|BA#r6{ENjXE+i-Clz3;6Ha
zg`;2NpgrNWH#0-{6l@NT!lq~fHjAaF=7+m3H>73ZimZT?C1=X^*l|{D7M|>zVau^L
zBW1m}9lmE#u`pP)$blfOY!I4J5e?8#2Ky5#(wC_rAlhm(sjSDmYk<eA01r4MB!0jO
zH{ARU&azZZBPT<TdJF7mq;cLBP*=-j`gFWDuGU(z{eG^Ed4i`u>Ic3jw@tz$!d6^{
z&X^<^5TIG8RLYDXSt!LVRbhwg^x_M2C5AEdA#1ax^iLVf=^bXpZGj(+D@jmNf>Thl
z&T^cokIpq^aF4iw_SPBo;SN8)G29}#=u{`N<+MjqgOVA?bp_lOW7#F7v%%_f7VH?g
z8YNYWFwwbm5br}FjM9_^LF+Cxe&C1ZP4~j+qH1C#4m6)vIFw^*?vPlMz2OxPfeyvp
z(ky&FVYcx>M^r@AGS+mYA8NvSXcX3p{6=`{BSv1CY^Y7wZ2#gF?1$p~o;5+|c`ZJ;
z4xBzcrZ06CK3<WnAO998F7cvUH*eg2-FbaN;C_wGy1pa#%?*MT`AFKiU0AcFZ|Y+f
zN4C<V46xVKsi%%@q?ra^=aM{UH5zxlbJ5Rqs}(6=Q7_$n^pISxJ7DusmW3txNxRt?
zE_x%wQs8n?G`10ku|<W)@JMXQHXN609O)&;<y<vPPa1WhR@U-N=}fuQTyc72N<>v7
z{}qQdAUk?0n+`ZOf4&xI5xz24rtz4SH-{pkoEKq9amUBrc{UJQg|_mUyoLf_G*g<l
zNDYf$MW_YqsYoV6c03grTSDK}3nOpt+)URGo#v}9=MGY?i`SuO%0pOXDRI5)m)!)b
zFbh$knhs)eM@orJ{7172E{2|q+ujD;ZR5cOnNu60CLXIN<D=}Tg3_qIhC^{~YHjLE
z_KKiI+sc`Q$7IOYf#>b~yP=<gtuhjE-`KuhHtV$nL}p*UKS<f7Bs>$}j1j#PC(hF0
z;H&!Q%3RgX92<<ZVw+Cekoffs^SsN&Ly}`~$mllY)$#Y=Gt%FTs&1zlPRM69jqoV?
zhOo+~-mmp?QvZpmDGKTTEV%rmNpATzC5?AAqT>>~xCR;${X&L4ZE)0WS=BDA*W&mC
zSdVtQxlc4~_`z1IxTrJRSim<>fGd2qtfx}if@iNK0A%WBjgt)7m?kZeI{n2Z<+OoI
ze^n)tOO8Y6A}A1q2YG@B=bLb@a)8L+0d6@i7>6a^&s|s!dmT_v0RTCyva$+5guePK
z#DK^nLZ6<acnBDDF2uhb@IP7>@t>Bx20VxIHe<rGFO0?6XAd5x6sgny+p>>|P<_`+
zAg<=2`%W)>1B^<R^RDqsc!(^R$?O~S(DYB;jFZdfYi9+_Nw|?rMSne#SWrELw`BOK
z$FEjj2<)ShBagNC-VPDd-<C!6)4Phn3@HK9jFcx;(<4@IsDJ1=K4(xn^>k(|3VyJe
zdZjRJHMD*F7j1U#iZ!&Pt!<ESV55F(cnIk5%ysu-FPgRbWRc5YUu%^YteJiIvSNxt
zQ^c7>XuUZoRK=MDhDrgIO(SC^Y_5u%F}d&Q#qK6{8Ll<89sGp_Z6hMK?vvw1h721^
zQ?J3D{cfLQI(hF=Fadg;xC*1LxTujHU`QL>p~{rw`S8b5!2pv$)^+QI4qNwx<f}JM
z_G<7l8rKIVRu^tT8pE{a5UFR<*-UO+hHlg<#FOyKYneW(uCJm#o;1$$uyK}P>^O8d
zIlCTRy)p268;_H;K30?}^L2QULuTHDLy9{*WWPCA?g=0d`sui_dxMnh%SW8>P3M<z
zRf$wR_Nd12yVvE&k?<KmNlrr9@RU2SR{{Me)kkB>ZK^N>ejOgh?(83x<y4fMGAx{#
zDzxEDn9oo!<W_$Vw#>p~n4Cmc^X+%*U07Npfp5FUV#TIeY?4x!VV&l;Mvv7)(7X;`
z?JD-7`}OKHi!Tg;{QHt3p@-iMXzkJuKjt&ziN_kC*#NJjN3oQ&3l$q=itfdV#0b~d
zhEqYy8l~{MKo5~XeH_i4S3BRlUd>|Z@F)N0&f<g?`*jAV`VIy$hv4`TZP9+t@et2F
z^y=>(@tA`zvML9h%xZf^vO+@dQ<<A5)*aVGEia+y>@!L3?psgNL?~(w2T~tGxq}5g
zH<jJ<OB%lcSQop=atn;tv3JXKz*q}ft@9INLA~3kNy2RQ<!eI;&h37&eYc40{9>Y8
zhB%87IBBwtV_^B9u`U$xY)<3Ht-^*+@BM3ytABf#I|pHm#qO(UHZ8y!g1@WAHkAoU
zSW47kNu2QZVViC5Lm{id<+%GCbPv?qTQB{rWbc^h)^4%=Nd)}9NLyJm;6Qv1K`B@=
zl}qflRv*sV8>{<Pj1y&gPkm4CUDXVj%^tK8zh*k#dm|i4N|D~2&v&9bV}zj}3OZJA
zF}a!|0d+tGKqBB6Rwx7w2f?^R{3Aj`i|#5+lWIgOquBrOqv>wLxK?jbr!lRd$m|5!
SnMOB%w{8=vJOm1k&HFbeab7$C

literal 0
HcmV?d00001

diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt
new file mode 100644
index 0000000000000..fe8e062bb7d6a
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIEB3RBlDANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwpz
+YW1sMS10ZXN0MB4XDTIwMDEzMDEzMDYxM1oXDTIwMDQyOTEzMDYxM1owFTETMBEG
+A1UEAxMKc2FtbDEtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AN5KCDf5R6ZFhcehDH0o6KMmAq/bVLczjNUcclxb0jFj9qfT1Lp7lYCMlbY0sskq
+2LncBIriihaC91OVBWWU1Z6fANxGn83ZPb+wM0h+iSN2IeHPjOP87vd1/oriSJEH
+U6MtzVKi5BIDoLx9tMhykSkWuQu05XQvMM6ofUyYtLZt38yUBaG/auO68FN6xtz/
+1RadENMVTfbGhUEPmSv+F/XsMW/MnhW0rXMOoWdmL9pv4vuGI7YUipw3xE30MxYA
+rEFfedmJFWhvonVoBg43pIa8cJLr8QX+G+Zgy1g0F8oLndreB0KbnsC8aqQ0z3qc
+iIbQMehPTN8cYgujTpGXS7MCAwEAAaMhMB8wHQYDVR0OBBYEFLRsbY+bypAXcmC4
+JzhEDJoidIgVMA0GCSqGSIb3DQEBCwUAA4IBAQC8rd3TQ/MR+SnK0mdBjhkJdZWl
+NiDkAuK1hRmreH3hYYBpVg3M51CcEAJIMsCH4RkJY66vjOWnsuPvF2Yxr0/dFI0W
+tBg01kZEodtxKvl8klJq3SHg2DqVJsJ8Sfl+lj+MBMvzTfWT4YjHnxjslPu39K5P
+tULjaO0pLHckkSSXH+JksUiYQ9enRp5aN/MbUvvHT4R5wd+SeQsThHKhrB3WZk3M
+Pn8vmGhsfdUjJNBMEbMMGRJEZmIs/Zdrg0pcGgk+tB0YqYGuiNLwtR0G0AVxeEHI
+Qf8IotlIg8mAKx9uHovTH8e3jyTDi+zBO0r3oSwiq43b1MOx88bSOtvATJbm
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key
new file mode 100644
index 0000000000000..29908feeee4ce
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDeSgg3+UemRYXH
+oQx9KOijJgKv21S3M4zVHHJcW9IxY/an09S6e5WAjJW2NLLJKti53ASK4ooWgvdT
+lQVllNWenwDcRp/N2T2/sDNIfokjdiHhz4zj/O73df6K4kiRB1OjLc1SouQSA6C8
+fbTIcpEpFrkLtOV0LzDOqH1MmLS2bd/MlAWhv2rjuvBTesbc/9UWnRDTFU32xoVB
+D5kr/hf17DFvzJ4VtK1zDqFnZi/ab+L7hiO2FIqcN8RN9DMWAKxBX3nZiRVob6J1
+aAYON6SGvHCS6/EF/hvmYMtYNBfKC53a3gdCm57AvGqkNM96nIiG0DHoT0zfHGIL
+o06Rl0uzAgMBAAECggEAV/2KH04HtKOUfx8a6HhHDj2/lokFkbdQKhOmmTSnkXF2
+c5yAeojs9wOvR364vrS9MWUMT0W5T4vsByPYDQvLO4zJNGioXT1/Gd/PjQS5mFXs
+t4wGw0Cd/0qvxZ4pB2+VBeTdf7aG/TJ+b956r1XabB9YhG/I3CLJ+SPs5WMVNZiv
+kmHldz/XWaa0zIIodk3eZYUM6B45DpVEE+oU2KpL+as4MiO5EhO5razMZARUwUH/
+D5dx6pWBMrkVtITEOJlUbIrjkPoN2uIVL+DjIXgI4t0pEVI8tP2M18xcb1s9C/2v
+wWkZkowaGXYADlvWtWRZSxRv8RAaIt7f+K9UkexRMQKBgQD5PfW5rI+fZr1xlto5
+NP0F8S5nM7Giw+L5DL5hPRhFX3TJR+mloItoIn2wBt5p7viJev1t+eFEITTQ/gnT
+3qj/EUFfjl3mCPCv/bGVFyRuKG9z5921BWEhyxQscQW47nahaeTfsVkqjBKuPQym
+FIa4d+TlIITdhQgwvKKDwmS+vQKBgQDkUPytZ4u9lEJAhBK37U1Oke2pZNwlwON7
+YJqhUV7rHtAffjHECjjOQFaU9qhUmQo0XMRMeyKjxZyxOBbPlNaDCEvj0pppK+fd
+Y70fNraWFKn8Oq8LtxfCJAzjySA8P3DKyyjtMNnONSzZZGAGVburjTv94SbKB933
+9YuQ1NVTLwKBgGqS0tp07eiz7rd+UIt7m+GJklTwprk5V3k0uMctb45iVbaneEGp
+F1MMxH+bQEk9tKpcm7cFzQvrwwDGT393yv1d7H0mjAILupcWPHPKtHoh8h42e/em
+eDUR9+tU3KSJ2tIETKhV1hG6WVaFpj3xsvV31rkeoZFzC+CDGF6MpbVlAoGBAMZD
+6dhvdbMQXyQFTPf5k0tqf7a9BkumoCSQtFboAYQtws/GV1MFCWVaC5XaSD0GlQz5
+jeFYgLE1NqfSrERg3JL3Dmfth/dSkEv5g87qRU447ZBCKaYDvhbyFVttWZfYA4un
+NogbBtkB5kt11clKg+kqgXErJvMEN+ZOjX8cw48zAoGAcW4ntY69UNo2XEm+bRTK
+Bde4WrlnLVztNckZOj6YbhqYzIkXIm9HRl7oOjPnK/M2D8f2UB8H9J8PEyNrFuis
+6MR0eoc4VXuMmi90VYL3yXjUiPYVKjEe1VcA6wDVwudK0v3okVSRjGIp0cJt2OIS
+5E+FvmJ79EQcNbRs/fGVyec=
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt
new file mode 100644
index 0000000000000..425bb42a2f487
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIERQXVrDANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwpz
+YW1sMi10ZXN0MB4XDTIwMDEzMDEzMDYzMloXDTIwMDQyOTEzMDYzMlowFTETMBEG
+A1UEAxMKc2FtbDItdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AK8mo/SqI+RAbq1Q7qDTddnjdqIMcmnCpHcSID2fhgoNrikIT8voWSthgpCpR7Zp
+CbVj1LK/RHtF7x9/HnrZr4Us07IESuK89NNKZYA6V5qiK1noctIGa2KIIDp0uVdL
+Uv4/UaK6sUe0tex1davFivqR40uHemO0OZb890YtdBft0c0qBoS7eU4fqcwT7Gbv
+gxMb7CrNqAsxT7Ax4gYGjWG0BYp+FicmB6A9aAJWIjac9YfABf0DDesazUAYpoAW
+FRCVKyGVa+U7HTq4wITGts6OKAbAdaJpbshxk9/TU91SfWqeqG0PyImGbQkbXr2c
+cAoylImud3EMUwkWgWfzsBECAwEAAaMhMB8wHQYDVR0OBBYEFPuTo5I6pHRzUZJX
+6mbznuriQbpWMA0GCSqGSIb3DQEBCwUAA4IBAQCdWs1cZvttuUJ1GnWjuwY+cJJR
+vNNl7D5Ge4zsyQeTzd8x0V4Pl5JVIyVxEP9Hu4+bgOo/daJtBT1y9B4eRfgtPGYO
+/3XP1OXJ7zpIIqEUozaCgfLo6L5Qk9DUfb+ikxVNagsC5DvlB+nKblqH3zbrEAPo
+tq7hFeow+PTr799U1flRkr9w/+FVJVZIvgKrb0E0BauPyR1+MHOr/XY50Mi6tH+/
+3F8SYRufLvdMSLQ9ZrBBZoQjcrfO1qb7GS364hbpu7TQIsXOHnfsAQsoGBBUrVla
+IATAHbog6SxpKT729dcrMVrJ/1inJ8NZuOZpIgJozi1tiuJ56QdOyuji59iU
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key
new file mode 100644
index 0000000000000..8419d4052068e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvJqP0qiPkQG6t
+UO6g03XZ43aiDHJpwqR3EiA9n4YKDa4pCE/L6FkrYYKQqUe2aQm1Y9Syv0R7Re8f
+fx562a+FLNOyBErivPTTSmWAOleaoitZ6HLSBmtiiCA6dLlXS1L+P1GiurFHtLXs
+dXWrxYr6keNLh3pjtDmW/PdGLXQX7dHNKgaEu3lOH6nME+xm74MTG+wqzagLMU+w
+MeIGBo1htAWKfhYnJgegPWgCViI2nPWHwAX9Aw3rGs1AGKaAFhUQlSshlWvlOx06
+uMCExrbOjigGwHWiaW7IcZPf01PdUn1qnqhtD8iJhm0JG169nHAKMpSJrndxDFMJ
+FoFn87ARAgMBAAECggEAQjfAmrpMUXoOndXQMdtgXd0CN+euxc9BsIaYy9NosJpV
+csvipzuTorP90cU+CHeTpRE2E9H+CQD6k5TSqC1uj1HQQygeS7SWdt9UmnL0NdED
+xLz90+t7Y8gFLzR78cXXfangqj4c0fg/I0ORFjzb+zl2jlKM8e7sKuo+bji3AGhX
+1Hl6F2yZmdsp+k3qLAiELc0ExBXq4M7SOCOzKE3BnMRtRE5QXieF39+l7g/YtSru
+OZI+gdA101ONNxiQa/w0m39wqaZgWex0dHjSztlg3Ja9ZralH1op/Zhhh2txLMf1
+CTlKZOEa/rFMw4YGqVfSX460/W+gXC3h+VBogHY9VQKBgQDZKpV0fVisV8pEbZaj
+kLtk1TGXX4gucgt9EOPqjuS4xF8RGGbS+Mk3DQgXWGRU7OE3Pd+SrQ+mqoesHtJK
+4Md+cqKx1F0E8Gg2Dr+jWsmbmAWsOxk6cLZJPNTpIRxtBcjzUgawJoVnimz7L38p
+4++VqpXZcG5CVvOKQAie7QiIwwKBgQDOeK4loplQF+UWSs7HmkJ0/MeELR7ICvhP
+LUo70qppiXZzjgCbfDoT5TLriR8fA+mP+YIjPMmdC9iGY4mK6CxaDYtbcNW64aOW
+j50VG7p3grS52yvzFgRX9V4/xJARO4UuYCab+gFLBaWd0LIZL/GbLZVD77Llg+qT
+yifkhFV2mwKBgEEAHfvgCMqM/F2+73fFulFT/qPwVDKSsB2fEJkEbDFhlZLoWt6K
+2K9Bl0pliqJgxIhkXKbqZeTc5f7+urLu1MdkSI4xoiX4Tzje8T1Tz0RjHE+Q4Nw9
+IFKLgq7E2l1Hff5WbuoKnaOAK16DuOww7WU+ydtra5UvsXWP9QScrhFBAoGAbjs1
+1nf53C31q7IitaQg6O5v4pxoO6rHqRXn9Qpi2td9+vGMV3a6Vxbd2awqhleDYc92
+8DMq/IJvhkgsDZIo46cpiJc0NIL04JQp7DQSmG2a6UMfMrfFd+RS/j6GH/0j/H9u
+/YU7HX+VTS03eIUFO7DZaRzW6PxFx8N9Ct6v9hMCgYABNFC7oPj24NwmJHZPXWuN
+V+ZRG2dC2E9K+MVg2Az1r+EyXCReBrzj0hYoZ2zS3SBGDD52ktIl6ekWCzEoC0VH
+uyg/237P1/Vs1IfuneI2/zq4R6sjeNmSEFsjhMqu1K1nCM4ZiWl6SzZrwo12bFCL
+93Lv9ufTBtMUgg154OM2xg==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt
new file mode 100644
index 0000000000000..3dbc9ae0486c6
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIEC9b3+jANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwpz
+YW1sMy10ZXN0MB4XDTIwMDEzMDEzMDY0MloXDTIwMDQyOTEzMDY0MlowFTETMBEG
+A1UEAxMKc2FtbDMtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AKHUOCpcF2LYVujyV1OQEOteDFs+vysWfbPu+gdmmQtW4wbK2ZMx7dppbWz9uC3f
+asyOqsxmhgGy9fkP/NfmKPPzN9gx+PzaGnu2oqvpAQMndBdIs4pmHn8ZlhNaHJvs
+1cTJfEMacQPNwFxjxkavl4dfnYDKRdxD0RFpfdHlFyM48P7emJqEUASx419/cRSh
+UoP6WUL1SInGkUDP0Zu8NR46Zk8XboeCKvTxPyOzMsDBZP726upAWQmmtYGpifra
+ScyDp/YkeDn6gWuSkThfGZI7x4E/H2/qlwJXeU9vIonauZYvIFGpjWD0Xw8iojSO
+2nkPutfvOOeb2Dvbxz1ANJsCAwEAAaMhMB8wHQYDVR0OBBYEFLlybUTvk1Yh5L55
+xhRWVN+i0D+/MA0GCSqGSIb3DQEBCwUAA4IBAQAv9l5udI0iSQmLPnRf876lczZG
+dylwrMv1zduThOTQ2hok1vf+D2Bi1U8mFl/T2WIM+2JlELi9yJDcFKh85AKvkRxG
+LY4CfKiOXEBpOrVETpNA2oEdh4HHsokKSt6mMe5GbUN+YdDYHNkmgTg01O7SIL4C
+9Et0GJlZ59uXTC9Z6RK27qdGm3zvCWQezOdE88Q1hWr0vjlPEDk9bQFiLJvCkAEo
+1vKjT3/NWJBK1mG7Z14YxFe1h28bHOCGruW84+sL4WLzJsSRXzEIJBJXpi0COdSg
+LrTkx482oen6U/IaZ6IDuQDhex/AhLhjsDsr/cJ2zkfD2d+P+ZQm2XIa96RE
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key
new file mode 100644
index 0000000000000..74cdc8cf9dd30
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCh1DgqXBdi2Fbo
+8ldTkBDrXgxbPr8rFn2z7voHZpkLVuMGytmTMe3aaW1s/bgt32rMjqrMZoYBsvX5
+D/zX5ijz8zfYMfj82hp7tqKr6QEDJ3QXSLOKZh5/GZYTWhyb7NXEyXxDGnEDzcBc
+Y8ZGr5eHX52AykXcQ9ERaX3R5RcjOPD+3piahFAEseNff3EUoVKD+llC9UiJxpFA
+z9GbvDUeOmZPF26Hgir08T8jszLAwWT+9urqQFkJprWBqYn62knMg6f2JHg5+oFr
+kpE4XxmSO8eBPx9v6pcCV3lPbyKJ2rmWLyBRqY1g9F8PIqI0jtp5D7rX7zjnm9g7
+28c9QDSbAgMBAAECggEAG43+DkBl/v+lNL7m3aSfLORzqCmz9Us8FiIv0ojGsGaX
++3UYvPZ4cIMZDcDZ1n2/OptkO6+NS8bEGSd+vuhCut8D+DzyD2O3tK7xXeIruPTD
+0J8hawReVvftVNg4HNijwYEpNmfPLH2hLloSzcBiRbujHuXSJWIqvBNBSDcAzI9E
+GjxNHNUT39OiRrwkWFWEP+GdytC1QN34/FVY3IHuqytIMAu49b7ejTDi5PWKjHdt
+7XjIwaFDQfxFihB3UxvifSZIosvc7nsQ6AZdIewmTHcmj3yRZoOWlh1UayeaCAu7
+6funIIpK0WqL6yaqDtB0X+Fc5nGiyxpNtO5ZytdPcQKBgQDy+QnUboC7LRQwiM1g
+Wk2pXzXMQ68Bt50Ta6a4klOnNfeTcklfmOgmuneCdXfEIQUER9Jy0e2jappZDVY/
+S7XEHfKZAld5b9j71xUr/J+RxgD20EVWALM07zBiHiinPOcAKpuWY0y0uVWumO5o
+blNtwlGAIwZLyjLcb+0RLDY/CQKBgQCqgW26zxM75tZWff7e0enXxhfEBeSdXVc3
+cTirnGWdI0ooGq3fLkpEL9gACg40GdoZtwLsiN8uN5JZJKaVAA0byO5jSDZwXPol
+XOW0guex/s37XsB1ps2Nq3XzL85+y7UG8dxUFXe5n0LNa51iDEhWEDJRAofD32vD
+iiLRHBgbgwKBgQDys8EV6au9TLI2KPC9pnadEPX0LEfMN1U36PiIteHJzSaYqjiQ
+vXRn+ysVrbEMwM76D6Qv/XKbIgk7mw7j+wR9+kwyexxVuVvNdHM5lQbv3hFoeXJJ
+Djn0TljOKcICC9vTVrxD5LziaZ8k4K6Esm+z2779d/76Bef4OfVcigVFwQKBgBMV
+T1m0KXvv1xdxHKCFr/11b12CJ1cxhjxvUJ9sbrcKpxcb/U/YV3mQPUWr2nEcX5u3
+Lv2YSBpdH+UYRAizK8RrVbsI7wbecSTTY+YzLIlwck9rCHFijOkJQrdIJ31rgURY
+t4c1q+O7nndtTv0VUNdJEsicVuOmUuJcyI7M7SnPAoGBAJhlAT+xRSolWcvNgdVH
+g20Wvu6o0uHpYNsp75Ogk2uwufK+lXQx5zPG7NBAd8s50nsAzvuONVLJvNwu9gg1
+xQ+ytfhazZgfUrUXhtLu2qOxN8bRGk8aKve8UQ9sRsMKKGvoPEuHVDm74Eh+x32D
+jdX6Jzav+5DL3/JvSGFGS6L/
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt
new file mode 100644
index 0000000000000..53d1f356d2456
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt
@@ -0,0 +1,9 @@
+-----BEGIN CERTIFICATE-----
+MIIBPzCB5aADAgECAgRijs0GMAwGCCqGSM49BAMCBQAwFTETMBEGA1UEAxMKc2Ft
+bDQtdGVzdDAeFw0yMDAxMzAxMzU5MDFaFw0yMDA0MjkxMzU5MDFaMBUxEzARBgNV
+BAMTCnNhbWw0LXRlc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAT5gZs/zPxN
+8Ch2aEYMchapSxcMrGYVqtVBO4gIV2ZwYsFhq/mM3+DvenY/9xLwkMHdWx6E8eQG
+gWFcsjPJcQ1CoyEwHzAdBgNVHQ4EFgQUrrGtOyJFw4hhgcdZIke00/BU9OIwDAYI
+KoZIzj0EAwIFAANHADBEAiB+PMEby4C9HNPI0vhQetJhzMFBYg1CXNBCwy/uHFHf
+qAIgTNPpG8hjuOWx7qAy6CSg5CPmO0huRm2sP5gLx68HLOo=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key
new file mode 100644
index 0000000000000..bf569cb418626
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key
@@ -0,0 +1,4 @@
+-----BEGIN PRIVATE KEY-----
+MEECAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcEJzAlAgEBBCDpKTGa+t0VcDav3WQ5
+ahJDxbmeCgjdZ/hhogVGE9GgyA==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java
index 77fe28f870145..7f074c8282c4f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java
@@ -230,7 +230,7 @@ private Assertion decrypt(EncryptedAssertion encrypted) {
 
     private List<Attribute> processAssertion(Assertion assertion, boolean requireSignature, Collection<String> allowedSamlRequestIds) {
         if (logger.isTraceEnabled()) {
-            logger.trace("(Possibly decrypted) Assertion: {}", SamlUtils.samlObjectToString(assertion));
+            logger.trace("(Possibly decrypted) Assertion: {}", SamlUtils.getXmlContent(assertion));
             logger.trace(SamlUtils.describeSamlObject(assertion));
         }
         // Do not further process unsigned Assertions
@@ -253,7 +253,7 @@ private List<Attribute> processAssertion(Assertion assertion, boolean requireSig
             for (EncryptedAttribute enc : statement.getEncryptedAttributes()) {
                 final Attribute attribute = decrypt(enc);
                 if (attribute != null) {
-                    logger.trace("Successfully decrypted attribute: {}" + SamlUtils.samlObjectToString(attribute));
+                    logger.trace("Successfully decrypted attribute: {}" + SamlUtils.getXmlContent(attribute));
                     attributes.add(attribute);
                 }
             }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
index 4f8677364dcf7..ca16bd4a9a095 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
@@ -656,7 +656,7 @@ public AuthnRequest buildAuthenticationRequest() {
                 .forceAuthn(forceAuthn)
                 .build();
         if (logger.isTraceEnabled()) {
-            logger.trace("Constructed SAML Authentication Request: {}", SamlUtils.samlObjectToString(authnRequest));
+            logger.trace("Constructed SAML Authentication Request: {}", SamlUtils.getXmlContent(authnRequest));
         }
         return authnRequest;
     }
@@ -672,7 +672,7 @@ public LogoutRequest buildLogoutRequest(NameID nameId, String session) {
             final LogoutRequest logoutRequest = new SamlLogoutRequestMessageBuilder(
                     Clock.systemUTC(), serviceProvider, idpDescriptor.get(), nameId, session).build();
             if (logoutRequest != null && logger.isTraceEnabled()) {
-                logger.trace("Constructed SAML Logout Request: {}", SamlUtils.samlObjectToString(logoutRequest));
+                logger.trace("Constructed SAML Logout Request: {}", SamlUtils.getXmlContent(logoutRequest));
             }
             return logoutRequest;
         } else {
@@ -688,7 +688,7 @@ public LogoutResponse buildLogoutResponse(String inResponseTo) {
         final LogoutResponse logoutResponse = new SamlLogoutResponseBuilder(
                 Clock.systemUTC(), serviceProvider, idpDescriptor.get(), inResponseTo, StatusCode.SUCCESS).build();
         if (logoutResponse != null && logger.isTraceEnabled()) {
-            logger.trace("Constructed SAML Logout Response: {}", SamlUtils.samlObjectToString(logoutResponse));
+            logger.trace("Constructed SAML Logout Response: {}", SamlUtils.getXmlContent(logoutResponse));
         }
         return logoutResponse;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java
index 9781783673afd..ccd0ec22f20ab 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.xpack.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
 import org.joda.time.DateTime;
 import org.opensaml.core.xml.XMLObject;
 import org.opensaml.core.xml.io.Unmarshaller;
@@ -47,7 +47,9 @@
 import javax.xml.parsers.DocumentBuilder;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.security.AccessController;
 import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.security.cert.CertificateEncodingException;
 import java.security.cert.X509Certificate;
 import java.time.Clock;
@@ -141,14 +143,20 @@ void validateSignature(Signature signature) {
         }
 
         checkIdpSignature(credential -> {
-            try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
-                SignatureValidator.validate(signature, credential);
-                logger.debug(() -> new ParameterizedMessage("SAML Signature [{}] matches credentials [{}] [{}]",
-                        signatureText, credential.getEntityId(), credential.getPublicKey()));
-                return true;
+            try {
+                return AccessController.doPrivileged((PrivilegedExceptionAction<Boolean>) () -> {
+                    try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
+                        SignatureValidator.validate(signature, credential);
+                        logger.debug(() -> new ParameterizedMessage("SAML Signature [{}] matches credentials [{}] [{}]",
+                            signatureText, credential.getEntityId(), credential.getPublicKey()));
+                        return true;
+                    } catch (PrivilegedActionException e) {
+                        logger.warn("SecurityException while attempting to validate SAML signature", e);
+                        return false;
+                    }
+                });
             } catch (PrivilegedActionException e) {
-                logger.warn("SecurityException while attempting to validate SAML signature", e);
-                return false;
+                throw new SecurityException("SecurityException while attempting to validate SAML signature", e);
             }
         }, signatureText);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java
index 8451f3553dfcb..7777229dd0fcd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java
@@ -43,7 +43,7 @@
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.xpack.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
 import org.opensaml.core.config.InitializationService;
 import org.opensaml.core.xml.XMLObject;
 import org.opensaml.core.xml.XMLObjectBuilderFactory;
@@ -162,9 +162,9 @@ static void print(Element element, Writer writer, boolean pretty) throws Transfo
         serializer.transform(new DOMSource(element), new StreamResult(writer));
     }
 
-    static String samlObjectToString(SAMLObject object) {
+    static String getXmlContent(SAMLObject object) {
         try {
-            return toString(XMLObjectSupport.marshall(object), true);
+            return toString(XMLObjectSupport.marshall(object), false);
         } catch (MarshallingException e) {
             LOGGER.info("Error marshalling SAMLObject ", e);
             return SAML_MARSHALLING_ERROR_STRING;
@@ -202,7 +202,7 @@ static String describeSamlObject(SAMLObject object) {
             sb.append("]");
             return sb.toString();
         }
-        return samlObjectToString(object);
+        return getXmlContent(object);
     }
 
     @SuppressForbidden(reason = "This is the only allowed way to construct a Transformer")
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
index 11c57fa3f5d4a..77d8c9d9737bb 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
@@ -22,6 +22,7 @@
 import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.IDP_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
@@ -115,6 +116,7 @@ public static void switchUserBasedOnActionOriginAndExecute(ThreadContext threadC
             case ROLLUP_ORIGIN:
             case INDEX_LIFECYCLE_ORIGIN:
             case ENRICH_ORIGIN:
+            case IDP_ORIGIN:
             case TASKS_ORIGIN:   // TODO use a more limited user for tasks
                 securityContext.executeAsUser(XPackUser.INSTANCE, consumer, Version.CURRENT);
                 break;
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 3ecd08cf3d48d..836653c56fbc5 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -4,9 +4,10 @@ grant {
   // needed because of problems in unbound LDAP library
   permission java.util.PropertyPermission "*", "read,write";
 
-  // needed because of SAML (cf. o.e.x.s.s.RestorableContextClassLoader)
+  // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader)
   permission java.lang.RuntimePermission "getClassLoader";
   permission java.lang.RuntimePermission "setContextClassLoader";
+
   // needed during initialization of OpenSAML library where xml security algorithms are registered
   // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220
   // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52

From 8ee2ee529fb2bb736bb52b1b2fe9c4f90a11ec07 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Tue, 24 Mar 2020 13:32:05 +0000
Subject: [PATCH 281/285] Revert "Clean up IndexTemplateMetaData xcontent
 serialization (#54003)"

This reverts commit 04c21affcf6b2215f22a690223ba1e520d3c6f9b.
---
 .../get/GetIndexTemplatesResponse.java        |   7 +-
 .../elasticsearch/cluster/ClusterState.java   |   5 +-
 .../metadata/IndexTemplateMetaData.java       | 253 +++++++++++-------
 .../cluster/metadata/MetaData.java            |   6 +-
 .../metadata/TemplateUpgradeService.java      |  10 +-
 .../metadata/IndexTemplateMetaDataTests.java  |  21 +-
 .../xpack/core/template/TemplateUtils.java    |   2 +-
 .../exporter/http/TemplateHttpResource.java   |   6 +-
 .../exporter/http/HttpExporterIT.java         |   6 +-
 9 files changed, 191 insertions(+), 125 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index c1fa7d7ffafc2..a726f4e108d7c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -30,6 +31,8 @@
 import java.util.List;
 import java.util.Objects;
 
+import static java.util.Collections.singletonMap;
+
 public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {
 
     private final List<IndexTemplateMetaData> indexTemplates;
@@ -74,9 +77,11 @@ public int hashCode() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params);
+
         builder.startObject();
         for (IndexTemplateMetaData indexTemplateMetaData : getIndexTemplates()) {
-            indexTemplateMetaData.toXContent(builder, params);
+            IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index f831207e2f0dd..f7ea8a68283bb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -53,7 +53,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -427,10 +426,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             coordinationMetaData().toXContent(builder, params);
             builder.endObject();
 
-            ToXContent.Params templateParams = new DelegatingMapParams(Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true"), params);
             builder.startObject("templates");
             for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
-                cursor.value.toXContent(builder, templateParams);
+                IndexTemplateMetaData templateMetaData = cursor.value;
+                IndexTemplateMetaData.Builder.toXContentWithTypes(templateMetaData, builder, params);
             }
             builder.endObject();
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index 935ab92f4d056..4d2db27a4a840 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -20,37 +20,36 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
-public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> implements ToXContentFragment {
+public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {
 
     private final String name;
 
@@ -229,7 +228,9 @@ public void writeTo(StreamOutput out) throws IOException {
     public String toString() {
         try {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            this.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.startObject();
+            IndexTemplateMetaData.Builder.toXContentWithTypes(this, builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
             return Strings.toString(builder);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -318,113 +319,185 @@ public Builder putAlias(AliasMetaData.Builder aliasMetaData) {
             return this;
         }
 
-        public void putAliases(List<AliasMetaData> aliases) {
-            for (AliasMetaData alias : aliases) {
-                putAlias(alias);
-            }
-        }
-
         public IndexTemplateMetaData build() {
             return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings.build(), aliases.build());
         }
 
-    }
-
-    public static final String INCLUDE_TYPE_NAME = "include_type_name";
-    public static final String INCLUDE_TEMPLATE_NAME = "include_template_name";
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-
-        boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME, false);
-        boolean includeTemplateName = params.paramAsBoolean(INCLUDE_TEMPLATE_NAME, true);
+        /**
+         * Serializes the template to xContent, using the legacy format where the mappings are
+         * nested under the type name.
+         *
+         * This method is used for serializing templates before storing them in the cluster metadata,
+         * and also in the REST layer when returning a deprecated typed response.
+         */
+        public static void toXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
+                                               XContentBuilder builder,
+                                               ToXContent.Params params) throws IOException {
+            builder.startObject(indexTemplateMetaData.name());
+            toInnerXContent(indexTemplateMetaData, builder, params, true);
+            builder.endObject();
+        }
 
-        if (includeTemplateName) {
-            builder.startObject(this.name());
+        /**
+         * Removes the nested type in the xContent representation of {@link IndexTemplateMetaData}.
+         *
+         * This method is useful to help bridge the gap between an the internal representation which still uses (the legacy format) a
+         * nested type in the mapping, and the external representation which does not use a nested type in the mapping.
+         */
+        public static void removeType(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder) throws IOException {
+            builder.startObject();
+            toInnerXContent(indexTemplateMetaData, builder,
+                new ToXContent.MapParams(Collections.singletonMap("reduce_mappings", "true")), false);
+            builder.endObject();
         }
 
-        builder.field("order", this.order());
-        if (this.version() != null) {
-            builder.field("version", this.version());
+        /**
+         * Serializes the template to xContent, making sure not to nest mappings under the
+         * type name.
+         *
+         * Note that this method should currently only be used for creating REST responses,
+         * and not when directly updating stored templates. Index templates are still stored
+         * in the old, typed format, and have yet to be migrated to be typeless.
+         */
+        public static void toXContent(IndexTemplateMetaData indexTemplateMetaData,
+                                      XContentBuilder builder,
+                                      ToXContent.Params params) throws IOException {
+            builder.startObject(indexTemplateMetaData.name());
+            toInnerXContent(indexTemplateMetaData, builder, params, false);
+            builder.endObject();
         }
-        builder.field("index_patterns", this.patterns());
 
-        builder.startObject("settings");
-        this.settings().toXContent(builder, params);
-        builder.endObject();
 
-        CompressedXContent m = this.mappings();
-        if (m != null) {
-            Map<String, Object> documentMapping = XContentHelper.convertToMap(new BytesArray(m.uncompressed()), true).v2();
-            if (includeTypeName == false) {
-                documentMapping = reduceMapping(documentMapping);
-            }
-            builder.field("mappings");
-            builder.map(documentMapping);
-        } else {
-            builder.startObject("mappings").endObject();
+        static void toInnerXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
+                                             XContentBuilder builder,
+                                             ToXContent.Params params) throws IOException {
+            toInnerXContent(indexTemplateMetaData, builder, params, true);
         }
 
-        builder.startObject("aliases");
-        for (ObjectCursor<AliasMetaData> cursor : this.aliases().values()) {
-            AliasMetaData.Builder.toXContent(cursor.value, builder, params);
-        }
-        builder.endObject();
+        private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
+                                            XContentBuilder builder,
+                                            ToXContent.Params params,
+                                            boolean includeTypeName) throws IOException {
 
-        if (includeTemplateName) {
-            builder.endObject();
-        }
+            builder.field("order", indexTemplateMetaData.order());
+            if (indexTemplateMetaData.version() != null) {
+                builder.field("version", indexTemplateMetaData.version());
+            }
+            builder.field("index_patterns", indexTemplateMetaData.patterns());
 
-        return builder;
-    }
+            builder.startObject("settings");
+            indexTemplateMetaData.settings().toXContent(builder, params);
+            builder.endObject();
 
-    @SuppressWarnings("unchecked")
-    private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
-        assert mapping.keySet().size() == 1;
-        return (Map<String, Object>) mapping.values().iterator().next();
-    }
+            includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false);
 
-    private static final ObjectParser<Builder, String> PARSER = ObjectParser.fromBuilder("index_template", Builder::new);
+            CompressedXContent m = indexTemplateMetaData.mappings();
+            if (m != null) {
+                Map<String, Object> documentMapping = XContentHelper.convertToMap(new BytesArray(m.uncompressed()), true).v2();
+                if (includeTypeName == false) {
+                    documentMapping = reduceMapping(documentMapping);
+                }
+                builder.field("mappings");
+                builder.map(documentMapping);
+            } else {
+                builder.startObject("mappings").endObject();
+            }
 
-    static {
-        PARSER.declareObject(Builder::settings,
-            (p, c) -> Settings.builder().put(Settings.fromXContent(p)).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(),
-            new ParseField("settings"));
-        PARSER.declareObject((builder, map) -> {
-            if (map.isEmpty() == false) {
-                String type = map.keySet().iterator().next();
-                builder.putMapping(type, map.get(type));
+            builder.startObject("aliases");
+            for (ObjectCursor<AliasMetaData> cursor : indexTemplateMetaData.aliases().values()) {
+                AliasMetaData.Builder.toXContent(cursor.value, builder, params);
             }
-        }, (p, c) -> parseMappings(p), new ParseField("mappings"));
-        PARSER.declareNamedObjects(Builder::putAliases, (p, c, n) -> AliasMetaData.Builder.fromXContent(p), new ParseField("aliases"));
-        PARSER.declareStringArray(Builder::patterns, new ParseField("index_patterns"));
-        PARSER.declareInt(Builder::order, new ParseField("order"));
-        PARSER.declareInt(Builder::version, new ParseField("version"));
-    }
+            builder.endObject();
+        }
 
-    // TODO it would be really nice if we could just copy bytes here, there's no need to parse and then
-    // re-stream the mappings
-    private static Map<String, CompressedXContent> parseMappings(XContentParser parser) throws IOException {
-        String mappingType = null;
-        String mappings = null;
-        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
-                mappingType = parser.currentName();
-            } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-                if (mappingType == null) {
-                    throw new XContentParseException(parser.getTokenLocation(), "Expected a mapping type");
+        @SuppressWarnings("unchecked")
+        private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
+            assert mapping.keySet().size() == 1;
+            return (Map<String, Object>) mapping.values().iterator().next();
+        }
+
+        public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
+            Builder builder = new Builder(templateName);
+
+            String currentFieldName = skipTemplateName(parser);
+            XContentParser.Token token;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    if ("settings".equals(currentFieldName)) {
+                        Settings.Builder templateSettingsBuilder = Settings.builder();
+                        templateSettingsBuilder.put(Settings.fromXContent(parser));
+                        templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
+                        builder.settings(templateSettingsBuilder.build());
+                    } else if ("mappings".equals(currentFieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                            if (token == XContentParser.Token.FIELD_NAME) {
+                                currentFieldName = parser.currentName();
+                            } else if (token == XContentParser.Token.START_OBJECT) {
+                                String mappingType = currentFieldName;
+                                Map<String, Object> mappingSource =
+                                    MapBuilder.<String, Object>newMapBuilder().put(mappingType, parser.mapOrdered()).map();
+                                builder.putMapping(mappingType, Strings.toString(XContentFactory.jsonBuilder().map(mappingSource)));
+                            }
+                        }
+                    } else if ("aliases".equals(currentFieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                            builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
+                        }
+                    } else {
+                        throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName);
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    if ("mappings".equals(currentFieldName)) {
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            Map<String, Object> mapping = parser.mapOrdered();
+                            if (mapping.size() == 1) {
+                                String mappingType = mapping.keySet().iterator().next();
+                                String mappingSource = Strings.toString(XContentFactory.jsonBuilder().map(mapping));
+
+                                if (mappingSource == null) {
+                                    // crap, no mapping source, warn?
+                                } else {
+                                    builder.putMapping(mappingType, mappingSource);
+                                }
+                            }
+                        }
+                    } else if ("index_patterns".equals(currentFieldName)) {
+                        List<String> index_patterns = new ArrayList<>();
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            index_patterns.add(parser.text());
+                        }
+                        builder.patterns(index_patterns);
+                    }
+                } else if (token.isValue()) {
+                    if ("order".equals(currentFieldName)) {
+                        builder.order(parser.intValue());
+                    } else if ("version".equals(currentFieldName)) {
+                        builder.version(parser.intValue());
+                    }
                 }
-                mappings = Strings.toString(XContentFactory.jsonBuilder().map(Map.of(mappingType, parser.mapOrdered())));
             }
+            return builder.build();
         }
-        if (mappings == null) {
-            return Collections.emptyMap();
-        }
-        return Map.of(mappingType, new CompressedXContent(mappings));
-    }
 
-    public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
-        return PARSER.parse(parser, templateName).build();
+        private static String skipTemplateName(XContentParser parser) throws IOException {
+            XContentParser.Token token = parser.nextToken();
+            if (token == XContentParser.Token.START_OBJECT) {
+                token = parser.nextToken();
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    String currentFieldName = parser.currentName();
+                    if (VALID_FIELDS.contains(currentFieldName)) {
+                        return currentFieldName;
+                    } else {
+                        // we just hit the template name, which should be ignored and we move on
+                        parser.nextToken();
+                    }
+                }
+            }
+
+            return null;
+        }
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index e8b607b65b20e..73ab3ccab7176 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -1404,11 +1404,9 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
                 builder.endObject();
             }
 
-            ToXContent.Params typedParams
-                = new ToXContent.DelegatingMapParams(Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true"), params);
             builder.startObject("templates");
             for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
-                cursor.value.toXContent(builder, typedParams);
+                IndexTemplateMetaData.Builder.toXContentWithTypes(cursor.value, builder, params);
             }
             builder.endObject();
 
@@ -1473,7 +1471,7 @@ public static MetaData fromXContent(XContentParser parser, boolean preserveUnkno
                         builder.hashesOfConsistentSettings(parser.mapStrings());
                     } else if ("templates".equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.put(IndexTemplateMetaData.fromXContent(parser, parser.currentName()));
+                            builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
                         }
                     } else {
                         try {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
index 4b595ad14fc2f..deb04067acb9a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
@@ -57,6 +57,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.UnaryOperator;
 
+import static java.util.Collections.singletonMap;
+
 /**
  * Upgrades Templates on behalf of installed {@link Plugin}s when a node joins the cluster
  */
@@ -246,12 +248,14 @@ Optional<Tuple<Map<String, BytesReference>, Set<String>>> calculateTemplateChang
         return Optional.empty();
     }
 
-    private static final ToXContent.Params PARAMS
-        = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
+    private static final ToXContent.Params PARAMS = new ToXContent.MapParams(singletonMap("reduce_mappings", "true"));
 
     private BytesReference toBytesReference(IndexTemplateMetaData templateMetaData) {
         try {
-            return XContentHelper.toXContent(templateMetaData, XContentType.JSON, PARAMS, false);
+            return XContentHelper.toXContent((builder, params) -> {
+                IndexTemplateMetaData.Builder.toInnerXContentWithTypes(templateMetaData, builder, params);
+                return builder;
+            }, XContentType.JSON, PARAMS, false);
         } catch (IOException ex) {
             throw new IllegalStateException("Cannot serialize template [" + templateMetaData.getName() + "]", ex);
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
index b4410550912e3..49e886c1d4745 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
@@ -34,7 +34,6 @@
 
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.contains;
@@ -55,15 +54,13 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
         final IndexTemplateMetaData indexTemplateMetaData;
         try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
             DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytes, XContentType.JSON)) {
-            indexTemplateMetaData = IndexTemplateMetaData.fromXContent(parser, "test");
+            indexTemplateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
         }
 
         final BytesReference templateBytesRoundTrip;
-        final ToXContent.Params params = new ToXContent.MapParams(
-            Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true", IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
         try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) {
             builder.startObject();
-            indexTemplateMetaData.toXContent(builder, params);
+            IndexTemplateMetaData.Builder.toXContentWithTypes(indexTemplateMetaData, builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
             templateBytesRoundTrip = BytesReference.bytes(builder);
         }
@@ -71,7 +68,7 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
         final IndexTemplateMetaData indexTemplateMetaDataRoundTrip;
         try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
             DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytesRoundTrip, XContentType.JSON)) {
-            indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.fromXContent(parser, "test");
+            indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
         }
         assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip));
     }
@@ -100,7 +97,7 @@ public void testValidateInvalidIndexPatterns() throws Exception {
                  XContentHelper.createParser(NamedXContentRegistry.EMPTY,
                      DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmptyPattern), XContentType.JSON)) {
             final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
+                () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
             assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got []"));
         }
 
@@ -115,7 +112,7 @@ DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmpty
                  XContentHelper.createParser(NamedXContentRegistry.EMPTY,
                      DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithoutPattern), XContentType.JSON)) {
             final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
+                () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
             assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got null"));
         }
     }
@@ -125,7 +122,7 @@ public void testParseTemplateWithAliases() throws Exception {
         try (XContentParser parser =
                  XContentHelper.createParser(NamedXContentRegistry.EMPTY,
                      DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateInJSON), XContentType.JSON)) {
-            IndexTemplateMetaData template = IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100));
+            IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100));
             assertThat(template.aliases().containsKey("log"), equalTo(true));
             assertThat(template.patterns(), contains("pattern-1"));
         }
@@ -160,14 +157,12 @@ public void testFromToXContent() throws Exception {
             templateBuilder.putMapping("doc", "{\"doc\":{\"properties\":{\"type\":\"text\"}}}");
         }
         IndexTemplateMetaData template = templateBuilder.build();
-        final ToXContent.Params params = new ToXContent.MapParams(
-            Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true", IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
         XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.JSON.xContent()));
         builder.startObject();
-        template.toXContent(builder, params);
+        IndexTemplateMetaData.Builder.toXContentWithTypes(template, builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         try (XContentParser parser = createParser(shuffleXContent(builder))) {
-            IndexTemplateMetaData parsed = IndexTemplateMetaData.fromXContent(parser, templateName);
+            IndexTemplateMetaData parsed = IndexTemplateMetaData.Builder.fromXContent(parser, templateName);
             assertThat(parsed, equalTo(template));
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
index cccad052c16b3..19140d9c011c4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
@@ -47,7 +47,7 @@ public static void loadTemplateIntoMap(String resource, Map<String, IndexTemplat
         final String template = loadTemplate(resource, version, versionProperty);
         try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template)) {
-            map.put(templateName, IndexTemplateMetaData.fromXContent(parser, templateName));
+            map.put(templateName, IndexTemplateMetaData.Builder.fromXContent(parser, templateName));
         } catch (IOException e) {
             // TODO: should we handle this with a thrown exception?
             logger.error("Error loading template [{}] as part of metadata upgrading", templateName);
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
index f4801a64f2fcd..d523864c6715b 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -108,13 +107,10 @@ protected void doPublish(final RestClient client, final ActionListener<Boolean>
      HttpEntity templateToHttpEntity() {
         // the internal representation of a template has type nested under mappings.
         // this uses xContent to help remove the type before sending to the remote cluster
-        ToXContent.Params params = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
         try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
             .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template.get())) {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            builder.startObject();
-            IndexTemplateMetaData.fromXContent(parser, templateName).toXContent(builder, params);
-            builder.endObject();
+            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, templateName), builder);
             return new StringEntity(BytesReference.bytes(builder).utf8ToString(), ContentType.APPLICATION_JSON);
         } catch (IOException ex) {
             throw new IllegalStateException("Cannot serialize template [" + templateName + "] for monitoring export", ex);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
index d21dc2ac62995..f1a3501487c06 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -972,10 +971,7 @@ private String getExternalTemplateRepresentation(String internalRepresentation)
         try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
             .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, internalRepresentation)) {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            ToXContent.Params params = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
-            builder.startObject();
-            IndexTemplateMetaData.fromXContent(parser, "").toXContent(builder, params);
-            builder.endObject();
+            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, ""), builder);
             return BytesReference.bytes(builder).utf8ToString();
         }
     }

From a371113d206c686304409d95e5c62273fb71a3cb Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@apache.org>
Date: Tue, 24 Mar 2020 13:44:28 +0000
Subject: [PATCH 282/285] Upgrade to lucene 8.5.0 release (#54077)

Upgrades our lucene dependency to the released 8.5.0 version.
---
 buildSrc/version.properties                                     | 2 +-
 .../lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1      | 1 -
 .../lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1  | 1 +
 .../lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1    | 1 -
 .../analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1   | 1 +
 ...ucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 .../licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1           | 1 +
 .../lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1   | 1 -
 .../analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1 | 1 +
 ...ucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 .../licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1           | 1 +
 ...lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 .../licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1            | 1 +
 ...lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 .../licenses/lucene-analyzers-stempel-8.5.0.jar.sha1            | 1 +
 ...ene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 .../licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1         | 1 +
 .../lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-analyzers-common-8.5.0.jar.sha1          | 1 +
 .../lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1  | 1 -
 server/licenses/lucene-backward-codecs-8.5.0.jar.sha1           | 1 +
 server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-core-8.5.0.jar.sha1                      | 1 +
 .../lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1         | 1 -
 server/licenses/lucene-grouping-8.5.0.jar.sha1                  | 1 +
 .../lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1      | 1 -
 server/licenses/lucene-highlighter-8.5.0.jar.sha1               | 1 +
 server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-join-8.5.0.jar.sha1                      | 1 +
 .../licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1  | 1 -
 server/licenses/lucene-memory-8.5.0.jar.sha1                    | 1 +
 server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-misc-8.5.0.jar.sha1                      | 1 +
 .../licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-queries-8.5.0.jar.sha1                   | 1 +
 .../lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1      | 1 -
 server/licenses/lucene-queryparser-8.5.0.jar.sha1               | 1 +
 .../licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-sandbox-8.5.0.jar.sha1                   | 1 +
 .../lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1   | 1 -
 server/licenses/lucene-spatial-extras-8.5.0.jar.sha1            | 1 +
 .../lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1        | 1 -
 server/licenses/lucene-spatial3d-8.5.0.jar.sha1                 | 1 +
 .../licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1 | 1 -
 server/licenses/lucene-suggest-8.5.0.jar.sha1                   | 1 +
 .../licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1    | 1 -
 .../plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1   | 1 +
 47 files changed, 24 insertions(+), 24 deletions(-)
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-analyzers-common-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-core-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-join-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-memory-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-misc-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-queries-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-8.5.0.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index ee2ce1b8d67b3..2ffc9101b9d80 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.5.0-snapshot-7f057455901
+lucene            = 8.5.0
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 14+36
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 19c383c96f0a0..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1219c9aca51a37ea3e22cf88ad2e8745d1a6e02f
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..ea1af2536868e
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1
@@ -0,0 +1 @@
+41fcbae8104c54487c83c002cf3da6a13065b7a4
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index f639c17a35f5a..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b6f880fa08a44fcb2d50808f9eeb6189a293ce27
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..c2f0eb182c26e
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1
@@ -0,0 +1 @@
+0697a7b06e4447be330f093e62d863deaadabc8c
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 5c1e0b2f6fa43..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9c5b8619795f69c225b5ec37b87cb34de0feccd4
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..9f20b12df3341
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1
@@ -0,0 +1 @@
+b269efbdd16c28525942a27f5738c9d1348b6301
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 656f63c68d194..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-421e13b9fe09523e094ac708204d62d4ea5b6618
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..4ea0c218177ec
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1
@@ -0,0 +1 @@
+056eef8f0a64a70cd9af070ecd8e17d33e55cb75
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 951d97032f9fb..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ff4ae9f3f3b0bc497f98c9bc47e943525669fc99
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..b0e0f037d5571
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1
@@ -0,0 +1 @@
+356e39f7b4b0cf8bc6a766a54ab6b93b11c89f6d
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 19846e43636c3..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd6430c037566cd3852b73b2ec31e59de24cfe58
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..faa11d7a87fb3
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1
@@ -0,0 +1 @@
+6139d1cc3f51d6f5fc5ab9976dc5376b682d8332
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 147bbb3192d24..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd4ca22b151a98a21e255bc1c54f0fadfee5ca4d
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..62fe533fde854
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1
@@ -0,0 +1 @@
+f6c1757e23f0cbad2e4d2eb39a12d9deff2e802b
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 86cee0ecdd039..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e85f94d2747ddb560af0bc4d15f0cde45cf3ff30
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..52115dc4ff819
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1
@@ -0,0 +1 @@
+e647c3158a092df07ba4ac1b827623f45176ef48
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 242bbfc9bd604..0000000000000
--- a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0365c37a03123ee8e30f75e44a1cb7d5ddd2fc52
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-8.5.0.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..398344daaa992
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-8.5.0.jar.sha1
@@ -0,0 +1 @@
+7156f2e545fd6e7faaee4781d15eb60cf5f07646
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 056fe4a00cbb5..0000000000000
--- a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d56b30f75b2df92da8c6c0965ce72e7abb86347b
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..1f926add1ffa2
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-8.5.0.jar.sha1
@@ -0,0 +1 @@
+5837d9ec231b998d9eb75a99f3bf1dc9748c8f46
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index fe30b9975cab4..0000000000000
--- a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0.jar.sha1 b/server/licenses/lucene-core-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..588b450154d09
--- /dev/null
+++ b/server/licenses/lucene-core-8.5.0.jar.sha1
@@ -0,0 +1 @@
+3f9ea85fff4fc3f7c83869dddb9b0ef7818c0cae
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index c3ef488e826f8..0000000000000
--- a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e56cc12d2f77d82946299b66f3416f9e621b2f3
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0.jar.sha1 b/server/licenses/lucene-grouping-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..89c60926a4a82
--- /dev/null
+++ b/server/licenses/lucene-grouping-8.5.0.jar.sha1
@@ -0,0 +1 @@
+08d26d94f32b38d15eaf68b17bef52158e4bbc87
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index cfa7fa961780b..0000000000000
--- a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5cddb5b65e7ead641483dcc2ffb0e50ad8d26eb7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..9ec3d4b53fef0
--- /dev/null
+++ b/server/licenses/lucene-highlighter-8.5.0.jar.sha1
@@ -0,0 +1 @@
+8c653f47ea042dec2920bab83b039774b567eb9f
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 94a18fd3c3913..0000000000000
--- a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cfabaedd80fe600cc7fda5ee12d90927fa96d87c
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0.jar.sha1 b/server/licenses/lucene-join-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..fa153682841ab
--- /dev/null
+++ b/server/licenses/lucene-join-8.5.0.jar.sha1
@@ -0,0 +1 @@
+98713495a3f48558a5acd7aaa22bfa7da394e78a
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index ae7da5c259f69..0000000000000
--- a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e90fbcc53531978fc03ef847ba396d4cdd89c7e4
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0.jar.sha1 b/server/licenses/lucene-memory-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..402e037eb1a20
--- /dev/null
+++ b/server/licenses/lucene-memory-8.5.0.jar.sha1
@@ -0,0 +1 @@
+18e3ed87c7f29bb0fe4b5db244a4f31018a9e518
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 8f6c2fe6d63c1..0000000000000
--- a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd0b4cef132a50b3fa919f214a5316fcc78c46ea
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0.jar.sha1 b/server/licenses/lucene-misc-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..fa96c7ab7e70c
--- /dev/null
+++ b/server/licenses/lucene-misc-8.5.0.jar.sha1
@@ -0,0 +1 @@
+9786e13764f55dbf28e71fe7e0a90d1e94bea0bc
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 4b2a346309f3f..0000000000000
--- a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c90cc35089afc3f7802668c3969b5e7391b6d15a
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0.jar.sha1 b/server/licenses/lucene-queries-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..ec92e3918b4d5
--- /dev/null
+++ b/server/licenses/lucene-queries-8.5.0.jar.sha1
@@ -0,0 +1 @@
+708c1f850ed70c506822b021a722e42f29c397a1
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 3b5ec91e4f938..0000000000000
--- a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e324233cb8f069e4f6abcbab47368a83c3696f36
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..3fe3e5fc193b0
--- /dev/null
+++ b/server/licenses/lucene-queryparser-8.5.0.jar.sha1
@@ -0,0 +1 @@
+13c38f39b1a7d10c4749ba789fa95da5868d4885
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 58d7518e7aefc..0000000000000
--- a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-985a451e5f564c84271419a446e044ab589d6f22
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..f36b8b978bede
--- /dev/null
+++ b/server/licenses/lucene-sandbox-8.5.0.jar.sha1
@@ -0,0 +1 @@
+2b275921f2fd92b15b4f1a2a565467c3fa221ef9
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index f51b876ad5fdf..0000000000000
--- a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-beff7cafe0fa5330b9b915825b69321faf0fcaa9
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..02c7de2cd466f
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-8.5.0.jar.sha1
@@ -0,0 +1 @@
+a8603576227b03fa94c2cde81b877f711c8b4c3f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 736c206ba1b48..0000000000000
--- a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b9256d3a2a64d79435a4c726af8a3c28c2b77d7f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..df28d9295c4a0
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-8.5.0.jar.sha1
@@ -0,0 +1 @@
+6f05de532ca9497e4d8364addf123441b01372a8
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 586728768c5b4..0000000000000
--- a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f38949db273a910e94a57229db2d8f3e4aef5e1f
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0.jar.sha1 b/server/licenses/lucene-suggest-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..b0abc5dc57319
--- /dev/null
+++ b/server/licenses/lucene-suggest-8.5.0.jar.sha1
@@ -0,0 +1 @@
+4ae6bea433acecbbaf7ae8fa3d56941ec2ad004d
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index fe30b9975cab4..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..588b450154d09
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1
@@ -0,0 +1 @@
+3f9ea85fff4fc3f7c83869dddb9b0ef7818c0cae
\ No newline at end of file

From c5764882a68db6a3a80a292a3d502dbc53286e95 Mon Sep 17 00:00:00 2001
From: Karen Metts <35154725+karenzone@users.noreply.github.com>
Date: Tue, 24 Mar 2020 10:01:58 -0400
Subject: [PATCH 283/285] [DOCS] Replace outdated Logstash monitoring link
 (#54032)

Replaces a link to Logstash OSS-only content with a link to the general Logstash monitoring topic.
---
 docs/reference/monitoring/how-monitoring-works.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/monitoring/how-monitoring-works.asciidoc b/docs/reference/monitoring/how-monitoring-works.asciidoc
index dc1c54ce9b865..2a89f0aec6524 100644
--- a/docs/reference/monitoring/how-monitoring-works.asciidoc
+++ b/docs/reference/monitoring/how-monitoring-works.asciidoc
@@ -22,7 +22,7 @@ To learn how to collect monitoring data, see:
 * <<collecting-monitoring-data>>
 * <<configuring-metricbeat>>
 * {kibana-ref}/xpack-monitoring.html[Monitoring {kib}]
-* {logstash-ref}/monitoring-logstash.html[Monitoring {ls}]
+* {logstash-ref}/configuring-logstash.html[Monitoring {ls}]
 * Monitoring Beats:
 ** {auditbeat-ref}/monitoring.html[{auditbeat}]
 ** {filebeat-ref}/monitoring.html[{filebeat}]

From 0eea6e237133913a39522b2a878a11fcb7a3ccf4 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Tue, 24 Mar 2020 14:51:31 -0400
Subject: [PATCH 284/285] add new VSTs to tests

---
 .../bucket/significant/SignificantTermsAggregatorTests.java | 5 ++++-
 .../bucket/significant/SignificantTextAggregatorTests.java  | 3 +++
 .../search/aggregations/metrics/AvgAggregatorTests.java     | 4 +++-
 .../search/aggregations/metrics/MinAggregatorTests.java     | 5 ++++-
 .../search/aggregations/metrics/StatsAggregatorTests.java   | 6 ++++--
 5 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
index 9b97b0e1ee8e6..9540f52703bab 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
@@ -86,7 +86,10 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy
     @Override
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
         return List.of(CoreValuesSourceType.NUMERIC,
-            CoreValuesSourceType.BYTES);
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN,
+            CoreValuesSourceType.BYTES,
+            CoreValuesSourceType.IP);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
index 6a354b0a6a496..01a865ffc5ea1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
@@ -77,7 +77,10 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
         // TODO it is likely accidental that SigText supports anything other than Bytes, and then only text fields
         return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN,
             CoreValuesSourceType.BYTES,
+            CoreValuesSourceType.IP,
             CoreValuesSourceType.RANGE,
             CoreValuesSourceType.GEOPOINT);
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
index 2b4af02a7cddd..f538bb922c9f2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
@@ -715,7 +715,9 @@ public void testScriptCaching() throws IOException {
     @Override
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
         return List.of(
-            CoreValuesSourceType.NUMERIC
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
index b9a649f627ce5..17ad8913a99a3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
@@ -954,7 +954,10 @@ private <T extends AggregationBuilder, V extends InternalAggregation> void testC
 
     @Override
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
-        return Collections.singletonList(CoreValuesSourceType.NUMERIC);
+        return List.of(
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
index 16fd9211a9ee4..8dbb671ac8b8e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
@@ -60,7 +60,6 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
-import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
@@ -475,7 +474,10 @@ void add(double value) {
 
     @Override
     protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
-        return singletonList(CoreValuesSourceType.NUMERIC);
+        return List.of(
+            CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.DATE,
+            CoreValuesSourceType.BOOLEAN);
     }
 
     @Override

From 7560542aec9f1f3379febd6eb18aff14d28a2085 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Tue, 24 Mar 2020 15:54:02 -0400
Subject: [PATCH 285/285] add VST for Wildcard field

---
 .../wildcard/mapper/WildcardFieldMapper.java  | 162 +++++++++---------
 1 file changed, 84 insertions(+), 78 deletions(-)

diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index 568e4f502dbe0..0c4b24e87cedf 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -59,6 +59,8 @@
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -76,15 +78,15 @@
 public class WildcardFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "wildcard";
-    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; 
-    public static final int NGRAM_SIZE = 3;        
+    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10;
+    public static final int NGRAM_SIZE = 3;
     static final NamedAnalyzer WILDCARD_ANALYZER = new NamedAnalyzer("_wildcard", AnalyzerScope.GLOBAL, new Analyzer() {
         @Override
         public TokenStreamComponents createComponents(String fieldName) {
             Tokenizer tokenizer = new NGramTokenizer(NGRAM_SIZE, NGRAM_SIZE);
             return new TokenStreamComponents(tokenizer);
         }
-    });    
+    });
 
     public static class Defaults {
         public static final MappedFieldType FIELD_TYPE = new WildcardFieldType();
@@ -98,14 +100,14 @@ public static class Defaults {
             FIELD_TYPE.setOmitNorms(true);
             FIELD_TYPE.freeze();
         }
-        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;        
+        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;
     }
 
     public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
         protected int ignoreAbove = Defaults.IGNORE_ABOVE;
         private IndexAnalyzers indexAnalyzers;
         private String normalizerName;
-        
+
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -115,7 +117,7 @@ public Builder(String name) {
         @Override
         public Builder docValues(boolean docValues) {
             if (docValues == false) {
-                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");                
+                throw new MapperParsingException("The field [" + name + "] cannot have doc values = false");
             }
             return this;
         }
@@ -131,64 +133,64 @@ public Builder indexOptions(IndexOptions indexOptions) {
         @Override
         public Builder store(boolean store) {
             if (store) {
-                throw new MapperParsingException("The field [" + name + "] cannot have store = true");                
+                throw new MapperParsingException("The field [" + name + "] cannot have store = true");
             }
             return this;
         }
 
         @Override
         public Builder similarity(SimilarityProvider similarity) {
-            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");                
+            throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");
         }
 
         @Override
         public Builder index(boolean index) {
             if (index == false) {
-                throw new MapperParsingException("The field [" + name + "] cannot have index = false");                
+                throw new MapperParsingException("The field [" + name + "] cannot have index = false");
             }
             return this;
         }
-        
+
         public Builder ignoreAbove(int ignoreAbove) {
             if (ignoreAbove < 0) {
                 throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
             }
             this.ignoreAbove = ignoreAbove;
             return this;
-        }        
-          
-        
+        }
+
+
         @Override
         protected void setupFieldType(BuilderContext context) {
             super.setupFieldType(context);
             fieldType().setHasDocValues(true);
             fieldType().setTokenized(false);
-            fieldType().setIndexOptions(IndexOptions.DOCS);                
+            fieldType().setIndexOptions(IndexOptions.DOCS);
         }
 
         @Override
         public WildcardFieldType fieldType() {
             return (WildcardFieldType) super.fieldType();
         }
-        
+
         public Builder normalizer(IndexAnalyzers indexAnalyzers, String name) {
             this.indexAnalyzers = indexAnalyzers;
             this.normalizerName = name;
             return builder;
-        }        
+        }
 
         @Override
         public WildcardFieldMapper build(BuilderContext context) {
-            setupFieldType(context);   
+            setupFieldType(context);
             if (normalizerName != null) {
                 NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
                 if (normalizer == null) {
                     throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
                 }
                 fieldType().setNormalizer(normalizer);
-            }            
+            }
             return new WildcardFieldMapper(
-                    name, fieldType, defaultFieldType, ignoreAbove, 
+                    name, fieldType, defaultFieldType, ignoreAbove,
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
         }
     }
@@ -199,7 +201,7 @@ public static class TypeParser implements Mapper.TypeParser {
                 throws MapperParsingException {
             WildcardFieldMapper.Builder builder = new WildcardFieldMapper.Builder(name);
             parseField(builder, name, node, parserContext);
-            
+
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String propName = entry.getKey();
@@ -213,21 +215,21 @@ public static class TypeParser implements Mapper.TypeParser {
                     }
                     iterator.remove();
                 }
-            }            
-            
+            }
+
             return builder;
         }
     }
-    
-     public static final char TOKEN_START_OR_END_CHAR = 0;    
-    
+
+     public static final char TOKEN_START_OR_END_CHAR = 0;
+
      public static final class WildcardFieldType extends MappedFieldType {
-         
+
         private NamedAnalyzer normalizer = null;
 
-        public WildcardFieldType() {            
+        public WildcardFieldType() {
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
-            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);            
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
         }
 
         protected WildcardFieldType(WildcardFieldType ref) {
@@ -239,7 +241,7 @@ public WildcardFieldType clone() {
             WildcardFieldType result = new WildcardFieldType(this);
             return result;
         }
-        
+
 
         @Override
         public boolean equals(Object o) {
@@ -253,8 +255,8 @@ public boolean equals(Object o) {
         @Override
         public int hashCode() {
             return 31 * super.hashCode() + Objects.hash(normalizer);
-        }       
-        
+        }
+
         private NamedAnalyzer normalizer() {
             return normalizer;
         }
@@ -262,7 +264,7 @@ private NamedAnalyzer normalizer() {
         public void setNormalizer(NamedAnalyzer normalizer) {
             checkIfFrozen();
             this.normalizer = normalizer;
-        }                
+        }
 
         @Override
         public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts) {
@@ -272,22 +274,22 @@ public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts)
                 conflicts.add("mapper [" + name() + "] has different [normalizer]");
             }
         }
-        
+
         // Holds parsed information about the wildcard pattern
         static class PatternStructure {
-            boolean openStart, openEnd, hasSymbols;            
+            boolean openStart, openEnd, hasSymbols;
             int lastGap =0;
             int wildcardCharCount, wildcardStringCount;
             String[] fragments;
             Integer []  precedingGapSizes;
             final String pattern;
-            
+
             @SuppressWarnings("fallthrough") // Intentionally uses fallthrough mirroring implementation in Lucene's WildcardQuery
             PatternStructure (String wildcardText) {
                 this.pattern = wildcardText;
                 ArrayList<String> fragmentList = new ArrayList<>();
                 ArrayList<Integer> precedingGapSizeList = new ArrayList<>();
-                StringBuilder sb = new StringBuilder();               
+                StringBuilder sb = new StringBuilder();
                 for (int i = 0; i < wildcardText.length();) {
                     final int c = wildcardText.codePointAt(i);
                     int length = Character.charCount(c);
@@ -296,10 +298,10 @@ static class PatternStructure {
                         if (i == 0) {
                             openStart = true;
                         }
-                        openEnd = true;                        
+                        openEnd = true;
                         hasSymbols = true;
                         wildcardStringCount++;
-                        
+
                         if (sb.length() > 0) {
                             precedingGapSizeList.add(lastGap);
                             fragmentList.add(sb.toString());
@@ -317,10 +319,10 @@ static class PatternStructure {
                         if (sb.length() > 0) {
                             precedingGapSizeList.add(lastGap);
                             fragmentList.add(sb.toString());
-                            sb = new StringBuilder();       
+                            sb = new StringBuilder();
                             lastGap = 0;
                         }
-                        
+
                         if (lastGap != Integer.MAX_VALUE) {
                             lastGap++;
                         }
@@ -347,19 +349,19 @@ static class PatternStructure {
                 }
                 fragments = fragmentList.toArray(new String[0]);
                 precedingGapSizes = precedingGapSizeList.toArray(new Integer[0]);
-                
+
             }
-            
+
             public boolean needsVerification() {
                 // Return true if term queries are not enough evidence
                 if (fragments.length == 1 && wildcardCharCount == 0) {
-                    // The one case where we don't need verification is when 
+                    // The one case where we don't need verification is when
                     // we have a single fragment and no ? characters
                     return false;
                 }
                 return true;
             }
-            
+
             // Returns number of positions for last gap (Integer.MAX means unlimited gap)
             public int getPrecedingGapSize(int fragmentNum) {
                 return precedingGapSizes[fragmentNum];
@@ -379,17 +381,17 @@ public boolean equals(Object obj) {
                 PatternStructure other = (PatternStructure) obj;
                 return pattern.equals(other.pattern);
             }
-            
-            
+
+
         }
-        
+
 
         @Override
         public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
             if (normalizer != null) {
                 wildcardPattern = StringFieldType.normalizeWildcardPattern(name(), wildcardPattern, normalizer);
             }
-            PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
+            PatternStructure patternStructure = new PatternStructure(wildcardPattern);
             ArrayList<String> tokens = new ArrayList<>();
 
             for (int i = 0; i < patternStructure.fragments.length; i++) {
@@ -398,7 +400,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                 if (fLength == 0) {
                     continue;
                 }
-                
+
                 // Add any start/end of string character
                 if (i == 0 && patternStructure.openStart == false) {
                     // Start-of-string anchored (is not a leading wildcard)
@@ -411,7 +413,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                 if (fragment.codePointCount(0, fragment.length()) <= NGRAM_SIZE) {
                     tokens.add(fragment);
                 } else {
-                    // Break fragment into multiple Ngrams                
+                    // Break fragment into multiple Ngrams
                     TokenStream tokenizer = WILDCARD_ANALYZER.tokenStream(name(), fragment);
                     CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
                     String lastUnusedToken = null;
@@ -425,7 +427,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                             if (takeThis) {
                                 tokens.add(tokenValue);
                             } else {
-                                lastUnusedToken = tokenValue;                                
+                                lastUnusedToken = tokenValue;
                             }
                             // alternate
                             takeThis = !takeThis;
@@ -445,7 +447,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
 
             if (patternStructure.isMatchAll()) {
                 return new MatchAllDocsQuery();
-            } 
+            }
             BooleanQuery approximation = createApproximationQuery(tokens);
             if (approximation.clauses().size() > 1 || patternStructure.needsVerification()) {
                 BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder();
@@ -455,7 +457,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh
                 return verifyingBuilder.build();
             }
             return approximation;
-        }                
+        }
 
         private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
             BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
@@ -473,7 +475,7 @@ private BooleanQuery createApproximationQuery(ArrayList<String> tokens) {
             // TODO we can be smarter about pruning here. e.g.
             // * Avoid wildcard queries if there are sufficient numbers of other terms that are full 3grams that are cheaper term queries
             // * We can select terms on their scarcity rather than even spreads across the search string.
-            
+
             return bqBuilder.build();
         }
 
@@ -504,11 +506,11 @@ public Query existsQuery(QueryShardContext context) {
         public Query termQuery(Object value, QueryShardContext context) {
             return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, context);
         }
-        
+
         @Override
         public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
             return wildcardQuery(value + "*", method, context);
-        }        
+        }
 
         @Override
         public Query termsQuery(List<?> values, QueryShardContext context) {
@@ -517,8 +519,8 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
                 bq.add(termQuery(value, context), Occur.SHOULD);
             }
             return new ConstantScoreQuery(bq.build());
-        }     
-                
+        }
+
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
             failIfNoDocValues();
@@ -531,8 +533,12 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
                 }};
         }
 
+         @Override
+         public ValuesSourceType getValuesSourceType() {
+            return CoreValuesSourceType.BYTES;
+         }
 
-        String normalize(String value) throws IOException {
+         String normalize(String value) throws IOException {
             if (normalizer != null) {
                 try (TokenStream ts = normalizer.tokenStream(name(), value)) {
                     final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
@@ -553,10 +559,10 @@ String normalize(String value) throws IOException {
                 }
             }
             return value;
-        }       
-        
+        }
+
     }
-     
+
     static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
 
         WildcardBytesBinaryDVIndexFieldData(Index index, String fieldName) {
@@ -569,7 +575,7 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested
                     sortMode, nested);
             return new SortField(getFieldName(), source, reverse);
         }
-    
+
     }
 
     private int ignoreAbove;
@@ -579,9 +585,9 @@ private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, Mapped
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         this.ignoreAbove = ignoreAbove;
         assert fieldType.indexOptions() == IndexOptions.DOCS;
-        
+
         ngramFieldType = fieldType.clone();
-        ngramFieldType.setTokenized(true);                    
+        ngramFieldType.setTokenized(true);
         ngramFieldType.freeze();
     }
 
@@ -590,8 +596,8 @@ private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, Mapped
     // pkg-private for testing
     int ignoreAbove() {
         return ignoreAbove;
-    }    
-    
+    }
+
     @Override
     protected WildcardFieldMapper clone() {
         return (WildcardFieldMapper) super.clone();
@@ -612,9 +618,9 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
             builder.field("normalizer", fieldType().normalizer().name());
         } else if (includeDefaults) {
             builder.nullField("normalizer");
-        }        
+        }
     }
-    
+
     @Override
     protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         final String value;
@@ -629,13 +635,13 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
             }
         }
         ParseContext.Document parseDoc = context.doc();
-        
-        createFields(value, parseDoc, fields);        
-    }   
-    
+
+        createFields(value, parseDoc, fields);
+    }
+
     // For internal use by Lucene only - used to define ngram index
     final MappedFieldType ngramFieldType;
-    
+
     void createFields(String value, Document parseDoc, List<IndexableField>fields) throws IOException {
         if (value == null || value.length() > ignoreAbove) {
             return;
@@ -644,25 +650,25 @@ void createFields(String value, Document parseDoc, List<IndexableField>fields) t
         String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
         Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
         fields.add(ngramField);
-        
+
         CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name());
         if (dvField == null) {
             dvField = new CustomBinaryDocValuesField(fieldType().name(), value.getBytes(StandardCharsets.UTF_8));
             parseDoc.addWithKey(fieldType().name(), dvField);
         } else {
             dvField.add(value.getBytes(StandardCharsets.UTF_8));
-        }        
+        }
     }
 
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
     }
-    
+
 
     @Override
     protected void doMerge(Mapper mergeWith) {
         super.doMerge(mergeWith);
         this.ignoreAbove = ((WildcardFieldMapper) mergeWith).ignoreAbove;
-    }    
+    }
 }