Merge branch 'master' into feature/extensible-values-source

 Conflicts:
	modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
	server/src/main/java/org/elasticsearch/index/IndexModule.java
	server/src/main/java/org/elasticsearch/index/IndexService.java
	server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
	server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
	server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
	server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
	server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java
	server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java
	server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
	server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
	server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
	server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
	server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
	server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
	server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java
	server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
	server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
	server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
	server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
	test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
	test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
	test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
	x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java
	x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java
	x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
	x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java
	x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
	x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
	x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
	x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java

Author: not-napoleon

TESTING.asciidoc

@@ -638,3 +638,9 @@ which you can use to measure the performance impact. It comes with a set of
 default benchmarks that we also
 https://elasticsearch-benchmarks.elastic.co/[run every night]. To get started,
 please see https://esrally.readthedocs.io/en/stable/[Rally's documentation].
+
+== Test doc builds
+
+The Elasticsearch docs are in AsciiDoc format. You can test and build the docs
+locally using the Elasticsearch documentation build process. See
+https://github.com/elastic/docs.

buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java

@@ -170,13 +170,23 @@ public List<T> subList(int fromIndex, int toIndex) {
     }
 
     @Override
-    public List<? extends Object> getNormalizedCollection() {
+    public List<? extends PropertyListEntry<T>> getNormalizedCollection() {
         return delegate.stream()
             .peek(this::validate)
             .filter(entry -> entry.getNormalization() != PropertyNormalization.IGNORE_VALUE)
             .collect(Collectors.toList());
     }
 
+    /**
+     * Return a "flattened" collection. This should be used when the collection type is itself a complex type with properties
+     * annotated as Gradle inputs rather than a simple type like {@link String}.
+     *
+     * @return a flattened collection filtered according to normalization strategy
+     */
+    public List<? extends T> getFlatNormalizedCollection() {
+        return getNormalizedCollection().stream().map(PropertyListEntry::getValue).collect(Collectors.toList());
+    }
+
     private void validate(PropertyListEntry<T> entry) {
         assertNotNull(entry.getValue(), "entry");
     }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1163,7 +1163,7 @@ public List<?> getKeystoreFiles() {
 
     @Nested
     public List<?> getCliSetup() {
-        return cliSetup.getNormalizedCollection();
+        return cliSetup.getFlatNormalizedCollection();
     }
 
     @Nested

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-6.1.1
+6.2

buildSrc/version.properties

@@ -29,7 +29,7 @@ joda              = 2.10.4
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 #  - x-pack/plugin/security
-bouncycastle      = 1.61
+bouncycastle=1.64
 # test dependencies
 randomizedrunner  = 2.7.6
 junit             = 4.12

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

@@ -39,8 +39,6 @@
 import java.io.IOException;
 import java.util.List;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestStatus.OK;
@@ -49,11 +47,11 @@ public class RestNoopBulkAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return unmodifiableList(asList(
+        return List.of(
             new Route(POST, "/_noop_bulk"),
             new Route(PUT, "/_noop_bulk"),
             new Route(POST, "/{index}/_noop_bulk"),
-            new Route(PUT, "/{index}/_noop_bulk")));
+            new Route(PUT, "/{index}/_noop_bulk"));
     }
 
     @Override

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java

@@ -26,20 +26,18 @@
 
 import java.util.List;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestNoopSearchAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return unmodifiableList(asList(
+        return List.of(
             new Route(GET, "/_noop_search"),
             new Route(POST, "/_noop_search"),
             new Route(GET, "/{index}/_noop_search"),
-            new Route(POST, "/{index}/_noop_search")));
+            new Route(POST, "/{index}/_noop_search"));
     }
 
     @Override

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -54,6 +54,10 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.analytics.ParsedStringStats;
+import org.elasticsearch.client.analytics.ParsedTopMetrics;
+import org.elasticsearch.client.analytics.StringStatsAggregationBuilder;
+import org.elasticsearch.client.analytics.TopMetricsAggregationBuilder;
 import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.client.core.CountResponse;
 import org.elasticsearch.client.core.GetSourceRequest;
@@ -1926,6 +1930,8 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
         map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
         map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
+        map.put(StringStatsAggregationBuilder.NAME, (p, c) -> ParsedStringStats.PARSER.parse(p, (String) c));
+        map.put(TopMetricsAggregationBuilder.NAME, (p, c) -> ParsedTopMetrics.PARSER.parse(p, (String) c));
         List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
                 .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
                 .collect(Collectors.toList());

client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedStringStats.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.analytics;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Results from the {@code string_stats} aggregation.
+ */
+public class ParsedStringStats extends ParsedAggregation {
+    private static final ParseField COUNT_FIELD = new ParseField("count");
+    private static final ParseField MIN_LENGTH_FIELD = new ParseField("min_length");
+    private static final ParseField MAX_LENGTH_FIELD = new ParseField("max_length");
+    private static final ParseField AVG_LENGTH_FIELD = new ParseField("avg_length");
+    private static final ParseField ENTROPY_FIELD = new ParseField("entropy");
+    private static final ParseField DISTRIBUTION_FIELD = new ParseField("distribution");
+
+    private final long count;
+    private final int minLength;
+    private final int maxLength;
+    private final double avgLength;
+    private final double entropy;
+    private final boolean showDistribution;
+    private final Map<String, Double> distribution;
+
+    private ParsedStringStats(String name, long count, int minLength, int maxLength, double avgLength, double entropy,
+            boolean showDistribution, Map<String, Double> distribution) {
+        setName(name);
+        this.count = count;
+        this.minLength = minLength;
+        this.maxLength = maxLength;
+        this.avgLength = avgLength;
+        this.entropy = entropy;
+        this.showDistribution = showDistribution;
+        this.distribution = distribution;
+    }
+
+    /**
+     * The number of non-empty fields counted.
+     */
+    public long getCount() {
+        return count;
+    }
+
+    /**
+     * The length of the shortest term.
+     */
+    public int getMinLength() {
+        return minLength;
+    }
+
+    /**
+     * The length of the longest term.
+     */
+    public int getMaxLength() {
+        return maxLength;
+    }
+
+    /**
+     * The average length computed over all terms.
+     */
+    public double getAvgLength() {
+        return avgLength;
+    }
+
+    /**
+     * The <a href="https://en.wikipedia.org/wiki/Entropy_(information_theory)">Shannon Entropy</a>
+     * value computed over all terms collected by the aggregation.
+     * Shannon entropy quantifies the amount of information contained in
+     * the field. It is a very useful metric for measuring a wide range of
+     * properties of a data set, such as diversity, similarity,
+     * randomness etc.
+     */
+    public double getEntropy() {
+        return entropy;
+    }
+
+    /**
+     * The probability distribution for all characters. {@code null} unless
+     * explicitly requested with {@link StringStatsAggregationBuilder#showDistribution(boolean)}.
+     */
+    public Map<String, Double> getDistribution() {
+        return distribution;
+    }
+
+    @Override
+    public String getType() {
+        return StringStatsAggregationBuilder.NAME;
+    }
+
+    private static final Object NULL_DISTRIBUTION_MARKER = new Object();
+    public static final ConstructingObjectParser<ParsedStringStats, String> PARSER = new ConstructingObjectParser<>(
+            StringStatsAggregationBuilder.NAME, true, (args, name) -> {
+                long count = (long) args[0];
+                boolean disributionWasExplicitNull = args[5] == NULL_DISTRIBUTION_MARKER;
+                if (count == 0) {
+                    return new ParsedStringStats(name, count, 0, 0, 0, 0, disributionWasExplicitNull, null);
+                }
+                int minLength = (int) args[1];
+                int maxLength = (int) args[2];
+                double averageLength = (double) args[3];
+                double entropy = (double) args[4];
+                if (disributionWasExplicitNull) {
+                    return new ParsedStringStats(name, count, minLength, maxLength, averageLength, entropy,
+                            disributionWasExplicitNull, null);
+                } else {
+                    @SuppressWarnings("unchecked")
+                    Map<String, Double> distribution = (Map<String, Double>) args[5];
+                    return new ParsedStringStats(name, count, minLength, maxLength, averageLength, entropy,
+                            distribution != null, distribution);
+                }
+            });
+    static {
+        PARSER.declareLong(constructorArg(), COUNT_FIELD);
+        PARSER.declareIntOrNull(constructorArg(), 0, MIN_LENGTH_FIELD);
+        PARSER.declareIntOrNull(constructorArg(), 0, MAX_LENGTH_FIELD);
+        PARSER.declareDoubleOrNull(constructorArg(), 0, AVG_LENGTH_FIELD);
+        PARSER.declareDoubleOrNull(constructorArg(), 0, ENTROPY_FIELD);
+        PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> unmodifiableMap(p.map(HashMap::new, XContentParser::doubleValue)),
+                NULL_DISTRIBUTION_MARKER, DISTRIBUTION_FIELD);
+        ParsedAggregation.declareAggregationFields(PARSER);
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(COUNT_FIELD.getPreferredName(), count);
+        if (count == 0) {
+            builder.nullField(MIN_LENGTH_FIELD.getPreferredName());
+            builder.nullField(MAX_LENGTH_FIELD.getPreferredName());
+            builder.nullField(AVG_LENGTH_FIELD.getPreferredName());
+            builder.field(ENTROPY_FIELD.getPreferredName(), 0.0);
+        } else {
+            builder.field(MIN_LENGTH_FIELD.getPreferredName(), minLength);
+            builder.field(MAX_LENGTH_FIELD.getPreferredName(), maxLength);
+            builder.field(AVG_LENGTH_FIELD.getPreferredName(), avgLength);
+            builder.field(ENTROPY_FIELD.getPreferredName(), entropy);
+        }
+        if (showDistribution) {
+            builder.field(DISTRIBUTION_FIELD.getPreferredName(), distribution);
+        }
+        return builder;
+    }
+}