Merge branch '6.x' into ccr-6.x

* 6.x:
  [DOCS] Omit shard failures assertion for incompatible responses  (#31430)
  [DOCS] Move licensing APIs to docs (#31445)
  backport of: add is-write-index flag to aliases (#30942) (#31412)
  backport of: Add rollover-creation-date setting to rolled over index (#31144) (#31413)
  [Docs] Extend Homebrew installation instructions (#28902)
  [Docs] Mention ip_range datatypes on ip type page (#31416)
  Multiplexing token filter (#31208)
  Fix use of time zone in date_histogram rewrite (#31407)
  Revert "Mute DefaultShardsIT#testDefaultShards test"
  [DOCS] Fixes code snippet testing for machine learning (#31189)
  Security: fix joining cluster with production license (#31341)
  [DOCS] Updated version in Info API example
  [DOCS] Moves the info API to docs (#31121)
  Revert "Increasing skip version for failing test on 6.x"
  Preserve response headers on cluster update task (#31421)
  [DOCS] Add code snippet testing for more ML APIs (#31404)
  Docs: Advice for reindexing many indices (#31279)

Author: dnhatn

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -27,7 +27,6 @@ import org.gradle.api.tasks.OutputDirectory
 
 import java.nio.file.Files
 import java.nio.file.Path
-import java.util.regex.Matcher
 
 /**
  * Generates REST tests for each snippet marked // TEST.
@@ -100,6 +99,14 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         return snippet.language == 'js' || snippet.curl
     }
 
+    /**
+     * Certain requests should not have the shard failure check because the
+     * format of the response is incompatible i.e. it is not a JSON object.
+     */
+    static shouldAddShardFailureCheck(String path) {
+        return path.startsWith('_cat') == false &&  path.startsWith('_xpack/ml/datafeeds/') == false
+    }
+
     /**
      * Converts Kibana's block quoted strings into standard JSON. These
      * {@code """} delimited strings can be embedded in CONSOLE and can
@@ -308,13 +315,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
              * no shard succeeds. But we need to fail the tests on all of these
              * because they mean invalid syntax or broken queries or something
              * else that we don't want to teach people to do. The REST test
-             * framework doesn't allow us to has assertions in the setup
-             * section so we have to skip it there. We also have to skip _cat
-             * actions because they don't return json so we can't is_false
-             * them. That is ok because they don't have this
-             * partial-success-is-success thing.
+             * framework doesn't allow us to have assertions in the setup
+             * section so we have to skip it there. We also omit the assertion
+             * from APIs that don't return a JSON object
              */
-            if (false == inSetup && false == path.startsWith('_cat')) {
+            if (false == inSetup && shouldAddShardFailureCheck(path)) {
                 current.println("  - is_false: _shards.failures")
             }
         }

buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy

@@ -19,9 +19,7 @@
 
 package org.elasticsearch.gradle.doc
 
-import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
-import org.gradle.api.InvalidUserDataException
-
+import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.shouldAddShardFailureCheck
 import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote
 
 class RestTestFromSnippetsTaskTest extends GroovyTestCase {
@@ -47,4 +45,10 @@ class RestTestFromSnippetsTaskTest extends GroovyTestCase {
         assertEquals("\"foo\": \"bort\\n baz\"",
             replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\""));
     }
+
+    void testIsDocWriteRequest() {
+        assertTrue(shouldAddShardFailureCheck("doc-index/_search"));
+        assertFalse(shouldAddShardFailureCheck("_cat"))
+        assertFalse(shouldAddShardFailureCheck("_xpack/ml/datafeeds/datafeed-id/_preview"));
+    }
 }

distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/DefaultShardsIT.java

@@ -31,7 +31,6 @@
 
 public class DefaultShardsIT extends ESRestTestCase {
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31408")
     public void testDefaultShards() throws IOException {
         final Response response = client().performRequest(new Request("PUT", "/index"));
         final String warning = response.getHeader("Warning");

docs/reference/analysis/tokenfilters.asciidoc

@@ -35,6 +35,8 @@ include::tokenfilters/word-delimiter-tokenfilter.asciidoc[]
 
 include::tokenfilters/word-delimiter-graph-tokenfilter.asciidoc[]
 
+include::tokenfilters/multiplexer-tokenfilter.asciidoc[]
+
 include::tokenfilters/stemmer-tokenfilter.asciidoc[]
 
 include::tokenfilters/stemmer-override-tokenfilter.asciidoc[]

docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc

@@ -0,0 +1,116 @@
+[[analysis-multiplexer-tokenfilter]]
+=== Multiplexer Token Filter
+
+A token filter of type `multiplexer` will emit multiple tokens at the same position,
+each version of the token having been run through a different filter.  Identical
+output tokens at the same position will be removed.
+
+WARNING: If the incoming token stream has duplicate tokens, then these will also be
+removed by the multiplexer
+
+[float]
+=== Options
+[horizontal]
+filters:: a list of token filters to apply to incoming tokens.  These can be any
+  token filters defined elsewhere in the index mappings.  Filters can be chained
+  using a comma-delimited string, so for example `"lowercase, porter_stem"` would
+  apply the `lowercase` filter and then the `porter_stem` filter to a single token.
+
+WARNING: Shingle or multi-word synonym token filters will not function normally
+  when they are declared in the filters array because they read ahead internally
+  which is unsupported by the multiplexer
+
+preserve_original:: if `true` (the default) then emit the original token in
+  addition to the filtered tokens
+
+
+[float]
+=== Settings example
+
+You can set it up like:
+
+[source,js]
+--------------------------------------------------
+PUT /multiplexer_example
+{
+    "settings" : {
+        "analysis" : {
+            "analyzer" : {
+                "my_analyzer" : {
+                    "tokenizer" : "standard",
+                    "filter" : [ "my_multiplexer" ]
+                }
+            },
+            "filter" : {
+                "my_multiplexer" : {
+                    "type" : "multiplexer",
+                    "filters" : [ "lowercase", "lowercase, porter_stem" ]
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+And test it like:
+
+[source,js]
+--------------------------------------------------
+POST /multiplexer_example/_analyze
+{
+  "analyzer" : "my_analyzer",
+  "text" : "Going HOME"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+And it'd respond:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens": [
+    {
+      "token": "Going",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "going",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "go",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "HOME",
+      "start_offset": 6,
+      "end_offset": 10,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "home",          <1>
+      "start_offset": 6,
+      "end_offset": 10,
+      "type": "<ALPHANUM>",
+      "position": 1
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+<1> The stemmer has also emitted a token `home` at position 1, but because it is a
+duplicate of this token it has been removed from the token stream

docs/reference/docs/reindex.asciidoc

@@ -1028,11 +1028,38 @@ number of slices.
 Whether query or indexing performance dominates the runtime depends on the
 documents being reindexed and cluster resources.
 
+[float]
+=== Reindexing many indices
+If you have many indices to reindex it is generally better to reindex them
+one at a time rather than using a glob pattern to pick up many indices. That
+way you can resume the process if there are any errors by removing the
+partially completed index and starting over at that index. It also makes
+parallelizing the process fairly simple: split the list of indices to reindex
+and run each list in parallel.
+
+One off bash scripts seem to work nicely for this:
+
+[source,bash]
+----------------------------------------------------------------
+for index in i1 i2 i3 i4 i5; do
+  curl -HContent-Type:application/json -XPOST localhost:9200/_reindex?pretty -d'{
+    "source": {
+      "index": "'$index'"
+    },
+    "dest": {
+      "index": "'$index'-reindexed"
+    }
+  }'
+done
+----------------------------------------------------------------
+// NOTCONSOLE
+
 [float]
 === Reindex daily indices
 
-You can use `_reindex` in combination with <<modules-scripting-painless, Painless>>
-to reindex daily indices to apply a new template to the existing documents.
+Notwithstanding the above advice, you can use `_reindex` in combination with
+<<modules-scripting-painless, Painless>> to reindex daily indices to apply
+a new template to the existing documents.
 
 Assuming you have indices consisting of documents as follows:
 

docs/reference/getting-started.asciidoc

@@ -158,6 +158,9 @@ On macOS, Elasticsearch can also be installed via https://brew.sh[Homebrew]:
 brew install elasticsearch
 --------------------------------------------------
 
+If installation succeeds, Homebrew will finish by saying that you can start Elasticsearch by entering
+`elasticsearch`. Do that now. The expected response is described below, under <<successfully-running-node>>.
+
 [float]
 === Installation example with MSI Windows Installer
 
@@ -216,6 +219,7 @@ And now we are ready to start our node and single cluster:
 --------------------------------------------------
 
 [float]
+[[successfully-running-node]]
 === Successfully running node
 
 If everything goes well with installation, you should see a bunch of messages that look like below:

docs/reference/indices/aliases.asciidoc

@@ -244,6 +244,94 @@ GET /alias2/_search?q=user:kimchy&routing=2,3
 // CONSOLE
 // TEST[continued]
 
+[float]
+[[aliases-write-index]]
+==== Write Index
+
+It is possible to associate the index pointed to by an alias as the write index.
+When specified, all index and update requests against an alias that point to multiple
+indices will attempt to resolve to the one index that is the write index.
+Only one index per alias can be assigned to be the write index at a time. If no write index is specified
+and there are multiple indices referenced by an alias, then writes will not be allowed.
+
+It is possible to specify an index associated with an alias as a write index using both the aliases API
+and index creation API.
+
+[source,js]
+--------------------------------------------------
+POST /_aliases
+{
+    "actions" : [
+        {
+            "add" : {
+                 "index" : "test",
+                 "alias" : "alias1",
+                 "is_write_index" : true
+            }
+        }
+    ]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test\n/]
+
+In this example, we associate the alias `alias1` to both `test` and `test2`, where
+`test` will be the index chosen for writing to.
+
+[source,js]
+--------------------------------------------------
+PUT /alias1/_doc/1
+{
+    "foo": "bar"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+The new document that was indexed to `/alias1/_doc/1` will be indexed as if it were
+`/test/_doc/1`.
+
+[source,js]
+--------------------------------------------------
+GET /test/_doc/1
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+To swap which index is the write index for an alias, the Aliases API can be leveraged to
+do an atomic swap. The swap is not dependent on the ordering of the actions.
+
+[source,js]
+--------------------------------------------------
+POST /_aliases
+{
+    "actions" : [
+        {
+            "add" : {
+                 "index" : "test",
+                 "alias" : "alias1",
+                 "is_write_index" : true
+            }
+        }, {
+            "add" : {
+                 "index" : "test2",
+                 "alias" : "alias1",
+                 "is_write_index" : false
+            }
+        }
+    ]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test\nPUT test2\n/]
+
+[IMPORTANT]
+=====================================
+Aliases that do not explicitly set `is_write_index: true` for an index, and
+only reference one index, will have that referenced index behave as if it is the write index
+until an additional index is referenced. At that point, there will be no write index and
+writes will be rejected.
+=====================================
 
 [float]
 [[alias-adding]]

x-pack/docs/en/rest-api/license/delete-license.asciidoc renamed to docs/reference/licensing/delete-license.asciidoc

@@ -1,4 +1,5 @@
 [role="xpack"]
+[testenv="basic"]
 [[delete-license]]
 === Delete License API
 
@@ -41,3 +42,4 @@ When the license is successfully deleted, the API returns the following response
   "acknowledged": true
 }
 ------------------------------------------------------------
+// NOTCONSOLE

x-pack/docs/en/rest-api/license/get-basic-status.asciidoc renamed to docs/reference/licensing/get-basic-status.asciidoc

@@ -1,4 +1,5 @@
 [role="xpack"]
+[testenv="basic"]
 [[get-basic-status]]
 === Get Basic Status API