Merge remote-tracking branch 'es/master' into ccr

* es/master:
  Add remote cluster client (#29495)
  Ensure flush happens on shard idle
  Adds SpanGapQueryBuilder in the query DSL (#28636)
  Control max size and count of warning headers (#28427)
  Make index APIs work without types. (#29479)
  Deprecate filtering on `_type`. (#29468)
  Fix auto-generated ID example format (#29461)
  Fix typo in max number of threads check docs (#29469)
  Add primary term to translog header (#29227)
  Add a helper method to get a random java.util.TimeZone (#29487)
  Move TimeValue into elasticsearch-core project (#29486)
  Fix NPE in InternalGeoCentroidTests#testReduceRandom (#29481)
  Build: introduce keystoreFile for cluster config (#29491)
  test: Index more docs, so that it is less likely the search request does not time out.

Author: martijnvg

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -141,6 +141,8 @@ class ClusterConfiguration {
 
     Map<String, String> keystoreSettings = new HashMap<>()
 
+    Map<String, Object> keystoreFiles = new HashMap<>()
+
     // map from destination path, to source file
     Map<String, Object> extraConfigFiles = new HashMap<>()
 
@@ -167,6 +169,15 @@ class ClusterConfiguration {
         keystoreSettings.put(name, value)
     }
 
+    /**
+     * Adds a file to the keystore. The name is the secure setting name, and the sourceFile
+     * is anything accepted by project.file()
+     */
+    @Input
+    void keystoreFile(String name, Object sourceFile) {
+        keystoreFiles.put(name, sourceFile)
+    }
+
     @Input
     void plugin(String path) {
         Project pluginProject = project.project(path)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -180,6 +180,7 @@ class ClusterFormationTasks {
         setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node, seedNode)
         setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node)
         setup = configureAddKeystoreSettingTasks(prefix, project, setup, node)
+        setup = configureAddKeystoreFileTasks(prefix, project, setup, node)
 
         if (node.config.plugins.isEmpty() == false) {
             if (node.nodeVersion == VersionProperties.elasticsearch) {
@@ -323,7 +324,7 @@ class ClusterFormationTasks {
 
     /** Adds a task to create keystore */
     static Task configureCreateKeystoreTask(String name, Project project, Task setup, NodeInfo node) {
-        if (node.config.keystoreSettings.isEmpty()) {
+        if (node.config.keystoreSettings.isEmpty() && node.config.keystoreFiles.isEmpty()) {
             return setup
         } else {
             /*
@@ -357,6 +358,37 @@ class ClusterFormationTasks {
         return parentTask
     }
 
+    /** Adds tasks to add files to the keystore */
+    static Task configureAddKeystoreFileTasks(String parent, Project project, Task setup, NodeInfo node) {
+        Map<String, Object> kvs = node.config.keystoreFiles
+        if (kvs.isEmpty()) {
+            return setup
+        }
+        Task parentTask = setup
+        /*
+         * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
+         * the short name requiring the path to already exist.
+         */
+        final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
+        for (Map.Entry<String, Object> entry in kvs) {
+            String key = entry.getKey()
+            String name = taskName(parent, node, 'addToKeystore#' + key)
+            String srcFileName = entry.getValue()
+            Task t = configureExecTask(name, project, parentTask, node, esKeystoreUtil, 'add-file', key, srcFileName)
+            t.doFirst {
+                File srcFile = project.file(srcFileName)
+                if (srcFile.isDirectory()) {
+                    throw new GradleException("Source for keystoreFile must be a file: ${srcFile}")
+                }
+                if (srcFile.exists() == false) {
+                    throw new GradleException("Source file for keystoreFile does not exist: ${srcFile}")
+                }
+            }
+            parentTask = t
+        }
+        return parentTask
+    }
+
     static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) {
         if (node.config.extraConfigFiles.isEmpty()) {
             return setup

docs/painless/painless-getting-started.asciidoc

@@ -239,7 +239,7 @@ their last name:
 
 [source,js]
 ----------------------------------------------------------------
-POST hockey/player/_update_by_query
+POST hockey/_update_by_query
 {
   "script": {
     "lang": "painless",
@@ -260,7 +260,7 @@ names start with a consonant and end with a vowel:
 
 [source,js]
 ----------------------------------------------------------------
-POST hockey/player/_update_by_query
+POST hockey/_update_by_query
 {
   "script": {
     "lang": "painless",
@@ -281,7 +281,7 @@ remove all of the vowels in all of their last names:
 
 [source,js]
 ----------------------------------------------------------------
-POST hockey/player/_update_by_query
+POST hockey/_update_by_query
 {
   "script": {
     "lang": "painless",
@@ -297,7 +297,7 @@ method so it supports `$1` and `\1` for replacements:
 
 [source,js]
 ----------------------------------------------------------------
-POST hockey/player/_update_by_query
+POST hockey/_update_by_query
 {
   "script": {
     "lang": "painless",
@@ -319,7 +319,7 @@ This will make all of the vowels in the hockey player's last names upper case:
 
 [source,js]
 ----------------------------------------------------------------
-POST hockey/player/_update_by_query
+POST hockey/_update_by_query
 {
   "script": {
     "lang": "painless",
@@ -337,7 +337,7 @@ last names upper case:
 
 [source,js]
 ----------------------------------------------------------------
-POST hockey/player/_update_by_query
+POST hockey/_update_by_query
 {
   "script": {
     "lang": "painless",

docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc

@@ -38,7 +38,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
-GET news/article/_search
+GET news/_search
 {
     "query" : {
         "match" : {"content" : "Bird flu"}
@@ -153,7 +153,7 @@ We can drill down into examples of these documents to see why pozmantier is conn
 
 [source,js]
 --------------------------------------------------
-GET news/article/_search
+GET news/_search
 {
   "query": {
     "simple_query_string": {
@@ -221,7 +221,7 @@ with the `filter_duplicate_text` setting turned on:
 
 [source,js]
 --------------------------------------------------
-GET news/article/_search
+GET news/_search
 {
   "query": {
     "match": {
@@ -424,7 +424,7 @@ context:
 
 [source,js]
 --------------------------------------------------
-GET news/article/_search
+GET news/_search
 {
     "query" : {
         "match" : {
@@ -463,7 +463,7 @@ will be analyzed using the `source_fields` parameter:
 
 [source,js]
 --------------------------------------------------
-GET news/article/_search
+GET news/_search
 {
     "query" : {
         "match" : {

docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc

@@ -217,7 +217,7 @@ had a value.
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {

docs/reference/docs/delete-by-query.asciidoc

@@ -75,7 +75,7 @@ Back to the API format, this will delete tweets from the `twitter` index:
 
 [source,js]
 --------------------------------------------------
-POST twitter/_doc/_delete_by_query?conflicts=proceed
+POST twitter/_delete_by_query?conflicts=proceed
 {
   "query": {
     "match_all": {}
@@ -85,12 +85,12 @@ POST twitter/_doc/_delete_by_query?conflicts=proceed
 // CONSOLE
 // TEST[setup:twitter]
 
-It's also possible to delete documents of multiple indexes and multiple
-types at once, just like the search API:
+It's also possible to delete documents of multiple indexes at once, just like
+the search API:
 
 [source,js]
 --------------------------------------------------
-POST twitter,blog/_docs,post/_delete_by_query
+POST twitter,blog/_delete_by_query
 {
   "query": {
     "match_all": {}

docs/reference/docs/index_.asciidoc

@@ -229,14 +229,14 @@ The result of the above index operation is:
     },
     "_index" : "twitter",
     "_type" : "_doc",
-    "_id" : "6a8ca01c-7896-48e9-81cc-9f70661fcb32",
+    "_id" : "W0tpsmIBdwcYyG50zbta",
     "_version" : 1,
     "_seq_no" : 0,
     "_primary_term" : 1,
     "result": "created"
 }
 --------------------------------------------------
-// TESTRESPONSE[s/6a8ca01c-7896-48e9-81cc-9f70661fcb32/$body._id/ s/"successful" : 2/"successful" : 1/]
+// TESTRESPONSE[s/W0tpsmIBdwcYyG50zbta/$body._id/ s/"successful" : 2/"successful" : 1/]
 
 [float]
 [[index-routing]]

docs/reference/docs/update-by-query.asciidoc

@@ -67,7 +67,7 @@ Back to the API format, this will update tweets from the `twitter` index:
 
 [source,js]
 --------------------------------------------------
-POST twitter/_doc/_update_by_query?conflicts=proceed
+POST twitter/_update_by_query?conflicts=proceed
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -145,12 +145,12 @@ This API doesn't allow you to move the documents it touches, just modify their
 source. This is intentional! We've made no provisions for removing the document
 from its original location.
 
-It's also possible to do this whole thing on multiple indexes and multiple
-types at once, just like the search API:
+It's also possible to do this whole thing on multiple indexes at once, just
+like the search API:
 
 [source,js]
 --------------------------------------------------
-POST twitter,blog/_doc,post/_update_by_query
+POST twitter,blog/_update_by_query
 --------------------------------------------------
 // CONSOLE
 // TEST[s/^/PUT twitter\nPUT blog\n/]

docs/reference/modules/cluster/misc.asciidoc

@@ -82,4 +82,4 @@ Enable or disable allocation for persistent tasks:
 This setting does not affect the persistent tasks that are already being executed.
 Only newly created persistent tasks, or tasks that must be reassigned (after a node
 left the cluster, for example), are impacted by this setting.
---
+--

docs/reference/modules/http.asciidoc

@@ -20,7 +20,7 @@ http://en.wikipedia.org/wiki/Chunked_transfer_encoding[HTTP chunking].
 
 The settings in the table below can be configured for HTTP. Note that none of
 them are dynamically updatable so for them to take effect they should be set in
-`elasticsearch.yml`.
+the Elasticsearch <<settings, configuration file>>.
 
 [cols="<,<",options="header",]
 |=======================================================================
@@ -100,6 +100,12 @@ simple message will be returned. Defaults to `true`
 
 |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`.
 
+|`http.max_warning_header_count` |The maximum number of warning headers in
+ client HTTP responses, defaults to unbounded.
+
+|`http.max_warning_header_size` |The maximum total size of warning headers in
+client HTTP responses, defaults to unbounded.
+
 |=======================================================================
 
 It also uses the common