Merge branch '6.2' of github.com:elastic/elasticsearch into 6.2

Author: Sue-Gallagher

buildSrc/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 6.2.4
+elasticsearch     = 6.2.5
 lucene            = 7.2.1
 
 # optional dependencies

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -707,10 +707,13 @@ private void installPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
         final PluginInfo info = loadPluginInfo(terminal, tmpRoot, isBatch, env);
         // read optional security policy (extra permissions), if it exists, confirm or warn the user
         Path policy = tmpRoot.resolve(PluginInfo.ES_PLUGIN_POLICY);
+        final Set<String> permissions;
         if (Files.exists(policy)) {
-            Set<String> permissions = PluginSecurity.parsePermissions(policy, env.tmpFile());
-            PluginSecurity.confirmPolicyExceptions(terminal, permissions, info.hasNativeController(), isBatch);
+            permissions = PluginSecurity.parsePermissions(policy, env.tmpFile());
+        } else {
+            permissions = Collections.emptySet();
         }
+        PluginSecurity.confirmPolicyExceptions(terminal, permissions, info.hasNativeController(), isBatch);
 
         final Path destination = env.pluginsFile().resolve(info.getName());
         deleteOnFailure.add(destination);

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -1191,6 +1191,59 @@ private Function<byte[], String> checksumAndString(final MessageDigest digest, f
         return bytes -> MessageDigests.toHexString(digest.digest(bytes)) + s;
     }
 
+    // checks the plugin requires a policy confirmation, and does not install when that is rejected by the user
+    // the plugin is installed after this method completes
+    private void assertPolicyConfirmation(Tuple<Path, Environment> env, String pluginZip, String... warnings) throws Exception {
+        for (int i = 0; i < warnings.length; ++i) {
+            String warning = warnings[i];
+            for (int j = 0; j < i; ++j) {
+                terminal.addTextInput("y"); // accept warnings we have already tested
+            }
+            // default answer, does not install
+            terminal.addTextInput("");
+            UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+            assertEquals("installation aborted by user", e.getMessage());
+
+            assertThat(terminal.getOutput(), containsString("WARNING: " + warning));
+            try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
+                assertThat(fileStream.collect(Collectors.toList()), empty());
+            }
+
+            // explicitly do not install
+            terminal.reset();
+            for (int j = 0; j < i; ++j) {
+                terminal.addTextInput("y"); // accept warnings we have already tested
+            }
+            terminal.addTextInput("n");
+            e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+            assertEquals("installation aborted by user", e.getMessage());
+            assertThat(terminal.getOutput(), containsString("WARNING: " + warning));
+            try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
+                assertThat(fileStream.collect(Collectors.toList()), empty());
+            }
+        }
+
+        // allow installation
+        terminal.reset();
+        for (int j = 0; j < warnings.length; ++j) {
+            terminal.addTextInput("y");
+        }
+        installPlugin(pluginZip, env.v1());
+        for (String warning : warnings) {
+            assertThat(terminal.getOutput(), containsString("WARNING: " + warning));
+        }
+    }
+
+    public void testPolicyConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        writePluginSecurityPolicy(pluginDir, "setAccessible", "setFactory");
+        String pluginZip = createPluginUrl("fake", pluginDir);
+
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions");
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
     public void testMetaPluginPolicyConfirmation() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path metaDir = createPluginDir(temp);
@@ -1204,32 +1257,60 @@ public void testMetaPluginPolicyConfirmation() throws Exception {
         writePlugin("fake2", fake2Dir);
         String pluginZip = createMetaPluginUrl("meta-plugin", metaDir);
 
-        // default answer, does not install
-        terminal.addTextInput("");
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertEquals("installation aborted by user", e.getMessage());
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
-        try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
-            assertThat(fileStream.collect(Collectors.toList()), empty());
-        }
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions");
+        assertMetaPlugin("meta-plugin", "fake1", metaDir, env.v2());
+        assertMetaPlugin("meta-plugin", "fake2", metaDir, env.v2());
+    }
 
-        // explicitly do not install
-        terminal.reset();
-        terminal.addTextInput("n");
-        e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertEquals("installation aborted by user", e.getMessage());
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
-        try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
-            assertThat(fileStream.collect(Collectors.toList()), empty());
-        }
+    public void testNativeControllerConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir, "has.native.controller", "true");
 
-        // allow installation
-        terminal.reset();
-        terminal.addTextInput("y");
-        installPlugin(pluginZip, env.v1());
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
+        assertPolicyConfirmation(env, pluginZip, "plugin forks a native controller");
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testMetaPluginNativeControllerConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path fake1Dir = metaDir.resolve("fake1");
+        Files.createDirectory(fake1Dir);
+        writePlugin("fake1", fake1Dir, "has.native.controller", "true");
+        Path fake2Dir = metaDir.resolve("fake2");
+        Files.createDirectory(fake2Dir);
+        writePlugin("fake2", fake2Dir);
+        String pluginZip = createMetaPluginUrl("meta-plugin", metaDir);
+
+        assertPolicyConfirmation(env, pluginZip, "plugin forks a native controller");
         assertMetaPlugin("meta-plugin", "fake1", metaDir, env.v2());
         assertMetaPlugin("meta-plugin", "fake2", metaDir, env.v2());
     }
 
+    public void testNativeControllerAndPolicyConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        writePluginSecurityPolicy(pluginDir, "setAccessible", "setFactory");
+        String pluginZip = createPluginUrl("fake", pluginDir, "has.native.controller", "true");
+
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions", "plugin forks a native controller");
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testMetaPluginNativeControllerAndPolicyConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path fake1Dir = metaDir.resolve("fake1");
+        Files.createDirectory(fake1Dir);
+        writePluginSecurityPolicy(fake1Dir, "setAccessible", "setFactory");
+        writePlugin("fake1", fake1Dir);
+        Path fake2Dir = metaDir.resolve("fake2");
+        Files.createDirectory(fake2Dir);
+        writePlugin("fake2", fake2Dir, "has.native.controller", "true");
+        String pluginZip = createMetaPluginUrl("meta-plugin", metaDir);
+
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions", "plugin forks a native controller");
+        assertMetaPlugin("meta-plugin", "fake1", metaDir, env.v2());
+        assertMetaPlugin("meta-plugin", "fake2", metaDir, env.v2());
+    }
 }

docs/Versions.asciidoc

@@ -1,4 +1,4 @@
-:version:               6.2.3
+:version:               6.2.4
 :major-version:         6.x
 :lucene_version:        7.2.1
 :lucene_version_path:   7_2_0

docs/plugins/analysis.asciidoc

@@ -53,6 +53,7 @@ A number of analysis plugins have been contributed by our community:
 * https://github.com/duydo/elasticsearch-analysis-vietnamese[Vietnamese Analysis Plugin] (by Duy Do)
 * https://github.com/ofir123/elasticsearch-network-analysis[Network Addresses Analysis Plugin] (by Ofir123)
 * https://github.com/medcl/elasticsearch-analysis-string2int[String2Integer Analysis Plugin] (by Medcl)
+* https://github.com/ZarHenry96/elasticsearch-dandelion-plugin[Dandelion Analysis Plugin] (by ZarHenry96)
 
 include::analysis-icu.asciidoc[]
 

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -27,11 +27,13 @@ POST /sales/_search?size=0
 // CONSOLE
 // TEST[setup:sales]
 
-Available expressions for interval: `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, `second`
+Available expressions for interval: `year` (`1y`), `quarter` (`1q`), `month` (`1M`), `week` (`1w`),
+`day` (`1d`), `hour` (`1h`), `minute` (`1m`), `second` (`1s`)
 
 Time values can also be specified via abbreviations supported by <<time-units,time units>> parsing.
 Note that fractional time values are not supported, but you can address this by shifting to another
-time unit (e.g., `1.5h` could instead be specified as `90m`).
+time unit (e.g., `1.5h` could instead be specified as `90m`). Also note that time intervals larger than
+than days do not support arbitrary values but can only be one unit large (e.g. `1y` is valid, `2y` is not).
 
 [source,js]
 --------------------------------------------------

docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc

@@ -5,8 +5,7 @@ A `single-value` metrics aggregation that calculates an approximate count of
 distinct values. Values can be extracted either from specific fields in the
 document or generated by a script.
 
-Assume you are indexing books and would like to count the unique authors that
-match a query:
+Assume you are indexing store sales and would like to count the unique number of sold products that match a query:
 
 [source,js]
 --------------------------------------------------

docs/reference/cluster/tasks.asciidoc

@@ -64,7 +64,7 @@ It is also possible to retrieve information for a particular task:
 
 [source,js]
 --------------------------------------------------
-GET _tasks/task_id:1 <1>
+GET _tasks/node_id:1 <1>
 --------------------------------------------------
 // CONSOLE
 // TEST[catch:missing]

docs/reference/docs/delete-by-query.asciidoc

@@ -284,9 +284,12 @@ executed again in order to conform to `requests_per_second`.
 
 `failures`::
 
-Array of all indexing failures. If this is non-empty then the request aborted
-because of those failures. See `conflicts` for how to prevent version conflicts
-from aborting the operation.
+Array of failures if there were any unrecoverable errors during the process. If
+this is non-empty then the request aborted because of those failures.
+Delete-by-query is implemented using batches and any failure causes the entire
+process to abort but all failures in the current batch are collected into the
+array. You can use the `conflicts` option to prevent reindex from aborting on
+version conflicts.
 
 
 [float]

docs/reference/docs/delete.asciidoc

@@ -39,11 +39,14 @@ The result of the above delete operation is:
 [[delete-versioning]]
 === Versioning
 
-Each document indexed is versioned. When deleting a document, the
-`version` can be specified to make sure the relevant document we are
-trying to delete is actually being deleted and it has not changed in the
-meantime. Every write operation executed on a document, deletes included,
-causes its version to be incremented.
+Each document indexed is versioned. When deleting a document, the `version` can
+be specified to make sure the relevant document we are trying to delete is
+actually being deleted and it has not changed in the meantime. Every write
+operation executed on a document, deletes included, causes its version to be
+incremented. The version number of a deleted document remains available for a
+short time after deletion to allow for control of concurrent operations. The
+length of time for which a deleted document's version remains available is
+determined by the `index.gc_deletes` index setting and defaults to 60 seconds.
 
 [float]
 [[delete-routing]]