Merge branch 'master' into compile-with-jdk-9

* master:
  Set watermarks in single-node test cases
  Add the ability to bundle multiple plugins into a meta plugin (elastic#28022)
  Declare empty package dirs as output dirs
  Consistent updates of IndexShardSnapshotStatus (elastic#28130)

Author: jasontedor

buildSrc/src/main/resources/meta-plugin-descriptor.properties

@@ -0,0 +1,21 @@
+# Elasticsearch meta plugin descriptor file
+# This file must exist as 'meta-plugin-descriptor.properties' in a folder named `elasticsearch`.
+#
+### example meta plugin for "meta-foo"
+#
+# meta-foo.zip <-- zip file for the meta plugin, with this structure:
+#|____elasticsearch/
+#| |____   <bundled_plugin_1> <-- The plugin files for bundled_plugin_1 (the content of the elastisearch directory)
+#| |____   <bundled_plugin_2>  <-- The plugin files for bundled_plugin_2
+#| |____   meta-plugin-descriptor.properties <-- example contents below:
+#
+# description=My meta plugin
+# name=meta-foo
+#
+### mandatory elements for all meta plugins:
+#
+# 'description': simple summary of the meta plugin
+description=${description}
+#
+# 'name': the meta plugin name
+name=${name}

core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java

@@ -30,6 +30,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 
 /**
@@ -60,23 +61,23 @@ public void writeTo(StreamOutput out) throws IOException {
      */
     public List<PluginInfo> getPluginInfos() {
         List<PluginInfo> plugins = new ArrayList<>(this.plugins);
-        Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        Collections.sort(plugins, Comparator.comparing(PluginInfo::getName));
         return plugins;
     }
-    
+
     /**
      * Returns an ordered list based on modules name
      */
     public List<PluginInfo> getModuleInfos() {
         List<PluginInfo> modules = new ArrayList<>(this.modules);
-        Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        Collections.sort(modules, Comparator.comparing(PluginInfo::getName));
         return modules;
     }
 
     public void addPlugin(PluginInfo info) {
         plugins.add(info);
     }
-    
+
     public void addModule(PluginInfo info) {
         modules.add(info);
     }

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java

@@ -22,7 +22,6 @@
 import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
@@ -49,13 +48,13 @@ private SnapshotIndexShardStatus() {
         this.stats = new SnapshotStats();
     }
 
-    SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus indexShardStatus) {
+    SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus.Copy indexShardStatus) {
         this(shardId, indexShardStatus, null);
     }
 
-    SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus indexShardStatus, String nodeId) {
+    SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus.Copy indexShardStatus, String nodeId) {
         super(shardId);
-        switch (indexShardStatus.stage()) {
+        switch (indexShardStatus.getStage()) {
             case INIT:
                 stage = SnapshotIndexShardStage.INIT;
                 break;
@@ -72,10 +71,12 @@ private SnapshotIndexShardStatus() {
                 stage = SnapshotIndexShardStage.FAILURE;
                 break;
             default:
-                throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.stage());
+                throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.getStage());
         }
-        stats = new SnapshotStats(indexShardStatus);
-        failure = indexShardStatus.failure();
+        this.stats = new SnapshotStats(indexShardStatus.getStartTime(), indexShardStatus.getTotalTime(),
+                                        indexShardStatus.getNumberOfFiles(), indexShardStatus.getProcessedFiles(),
+                                        indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize());
+        this.failure = indexShardStatus.getFailure();
         this.nodeId = nodeId;
     }
 

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java

@@ -25,33 +25,28 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 
 import java.io.IOException;
 
 public class SnapshotStats implements Streamable, ToXContentFragment {
-    private long startTime;
 
+    private long startTime;
     private long time;
-
     private int numberOfFiles;
-
     private int processedFiles;
-
     private long totalSize;
-
     private long processedSize;
 
     SnapshotStats() {
     }
 
-    SnapshotStats(IndexShardSnapshotStatus indexShardStatus) {
-        startTime = indexShardStatus.startTime();
-        time = indexShardStatus.time();
-        numberOfFiles = indexShardStatus.numberOfFiles();
-        processedFiles = indexShardStatus.processedFiles();
-        totalSize = indexShardStatus.totalSize();
-        processedSize = indexShardStatus.processedSize();
+    SnapshotStats(long startTime, long time, int numberOfFiles, int processedFiles, long totalSize, long processedSize) {
+        this.startTime = startTime;
+        this.time = time;
+        this.numberOfFiles = numberOfFiles;
+        this.processedFiles = processedFiles;
+        this.totalSize = totalSize;
+        this.processedSize = processedSize;
     }
 
     /**

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java

@@ -96,23 +96,25 @@ protected NodesSnapshotStatus newResponse(Request request, List<NodeSnapshotStat
     protected NodeSnapshotStatus nodeOperation(NodeRequest request) {
         Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>();
         try {
-            String nodeId = clusterService.localNode().getId();
+            final String nodeId = clusterService.localNode().getId();
             for (Snapshot snapshot : request.snapshots) {
                 Map<ShardId, IndexShardSnapshotStatus> shardsStatus = snapshotShardsService.currentSnapshotShards(snapshot);
                 if (shardsStatus == null) {
                     continue;
                 }
                 Map<ShardId, SnapshotIndexShardStatus> shardMapBuilder = new HashMap<>();
                 for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : shardsStatus.entrySet()) {
-                    SnapshotIndexShardStatus shardStatus;
-                    IndexShardSnapshotStatus.Stage stage = shardEntry.getValue().stage();
+                    final ShardId shardId = shardEntry.getKey();
+
+                    final IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardEntry.getValue().asCopy();
+                    final IndexShardSnapshotStatus.Stage stage = lastSnapshotStatus.getStage();
+
+                    String shardNodeId = null;
                     if (stage != IndexShardSnapshotStatus.Stage.DONE && stage != IndexShardSnapshotStatus.Stage.FAILURE) {
                         // Store node id for the snapshots that are currently running.
-                        shardStatus = new SnapshotIndexShardStatus(shardEntry.getKey(), shardEntry.getValue(), nodeId);
-                    } else {
-                        shardStatus = new SnapshotIndexShardStatus(shardEntry.getKey(), shardEntry.getValue());
+                        shardNodeId = nodeId;
                     }
-                    shardMapBuilder.put(shardEntry.getKey(), shardStatus);
+                    shardMapBuilder.put(shardEntry.getKey(), new SnapshotIndexShardStatus(shardId, lastSnapshotStatus, shardNodeId));
                 }
                 snapshotMapBuilder.put(snapshot, unmodifiableMap(shardMapBuilder));
             }

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java

@@ -233,7 +233,8 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li
                     Map<ShardId, IndexShardSnapshotStatus> shardStatues =
                         snapshotsService.snapshotShards(request.repository(), snapshotInfo);
                     for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatues.entrySet()) {
-                        shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue()));
+                        IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue().asCopy();
+                        shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus));
                     }
                     final SnapshotsInProgress.State state;
                     switch (snapshotInfo.state()) {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -163,16 +163,8 @@ static Map<String, URL> getCodebaseJarMap(Set<URL> urls) {
     static Map<String,Policy> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
         Map<String,Policy> map = new HashMap<>();
         // collect up set of plugins and modules by listing directories.
-        Set<Path> pluginsAndModules = new LinkedHashSet<>(); // order is already lost, but some filesystems have it
-        if (Files.exists(environment.pluginsFile())) {
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
-                for (Path plugin : stream) {
-                    if (pluginsAndModules.add(plugin) == false) {
-                        throw new IllegalStateException("duplicate plugin: " + plugin);
-                    }
-                }
-            }
-        }
+        Set<Path> pluginsAndModules = new LinkedHashSet<>(PluginInfo.extractAllPlugins(environment.pluginsFile()));
+
         if (Files.exists(environment.modulesFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.modulesFile())) {
                 for (Path module : stream) {

core/src/main/java/org/elasticsearch/bootstrap/Spawner.java

@@ -21,14 +21,12 @@
 
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -72,27 +70,23 @@ void spawnNativePluginControllers(final Environment environment) throws IOExcept
          * For each plugin, attempt to spawn the controller daemon. Silently ignore any plugin that
          * don't include a controller for the correct platform.
          */
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsFile)) {
-            for (final Path plugin : stream) {
-                if (FileSystemUtils.isDesktopServicesStore(plugin)) {
-                    continue;
-                }
-                final PluginInfo info = PluginInfo.readFromProperties(plugin);
-                final Path spawnPath = Platforms.nativeControllerPath(plugin);
-                if (!Files.isRegularFile(spawnPath)) {
-                    continue;
-                }
-                if (!info.hasNativeController()) {
-                    final String message = String.format(
-                            Locale.ROOT,
-                            "plugin [%s] does not have permission to fork native controller",
-                            plugin.getFileName());
-                    throw new IllegalArgumentException(message);
-                }
-                final Process process =
-                        spawnNativePluginController(spawnPath, environment.tmpFile());
-                processes.add(process);
+        List<Path> paths = PluginInfo.extractAllPlugins(pluginsFile);
+        for (Path plugin : paths) {
+            final PluginInfo info = PluginInfo.readFromProperties(plugin);
+            final Path spawnPath = Platforms.nativeControllerPath(plugin);
+            if (!Files.isRegularFile(spawnPath)) {
+                continue;
             }
+            if (!info.hasNativeController()) {
+                final String message = String.format(
+                    Locale.ROOT,
+                    "plugin [%s] does not have permission to fork native controller",
+                    plugin.getFileName());
+                throw new IllegalArgumentException(message);
+            }
+            final Process process =
+                spawnNativePluginController(spawnPath, environment.tmpFile());
+            processes.add(process);
         }
     }