Add the ability to bundle multiple plugins into a meta plugin

buildSrc/src/main/resources/meta-plugin-descriptor.properties

@@ -0,0 +1,21 @@
+# Elasticsearch meta plugin descriptor file
+# This file must exist as 'meta-plugin-descriptor.properties' in a folder named `elasticsearch`.
+#
+### example meta plugin for "meta-foo"
+#
+# meta-foo.zip <-- zip file for the meta plugin, with this structure:
+#|____elasticsearch/
+#| |____   <bundled_plugin_1> <-- The plugin files for bundled_plugin_1 (the content of the elastisearch directory)
+#| |____   <bundled_plugin_2>  <-- The plugin files for bundled_plugin_2
+#| |____   meta-plugin-descriptor.properties <-- example contents below:
+#
+# description=My meta plugin
+# name=meta-foo
+#
+### mandatory elements for all meta plugins:
+#
+# 'description': simple summary of the meta plugin
+description=${description}
+#
+# 'name': the meta plugin name
+name=${name}

core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java

@@ -30,6 +30,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 
 /**
@@ -60,23 +61,23 @@ public void writeTo(StreamOutput out) throws IOException {
      */
     public List<PluginInfo> getPluginInfos() {
         List<PluginInfo> plugins = new ArrayList<>(this.plugins);
-        Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        Collections.sort(plugins, Comparator.comparing(PluginInfo::getName));
         return plugins;
     }
-    
+
     /**
      * Returns an ordered list based on modules name
      */
     public List<PluginInfo> getModuleInfos() {
         List<PluginInfo> modules = new ArrayList<>(this.modules);
-        Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        Collections.sort(modules, Comparator.comparing(PluginInfo::getName));
         return modules;
     }
 
     public void addPlugin(PluginInfo info) {
         plugins.add(info);
     }
-    
+
     public void addModule(PluginInfo info) {
         modules.add(info);
     }

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -163,16 +163,8 @@ static Map<String, URL> getCodebaseJarMap(Set<URL> urls) {
     static Map<String,Policy> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
         Map<String,Policy> map = new HashMap<>();
         // collect up set of plugins and modules by listing directories.
-        Set<Path> pluginsAndModules = new LinkedHashSet<>(); // order is already lost, but some filesystems have it
-        if (Files.exists(environment.pluginsFile())) {
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
-                for (Path plugin : stream) {
-                    if (pluginsAndModules.add(plugin) == false) {
-                        throw new IllegalStateException("duplicate plugin: " + plugin);
-                    }
-                }
-            }
-        }
+        Set<Path> pluginsAndModules = new LinkedHashSet<>(PluginInfo.extractAllPlugins(environment.pluginsFile()));
+
         if (Files.exists(environment.modulesFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.modulesFile())) {
                 for (Path module : stream) {

core/src/main/java/org/elasticsearch/bootstrap/Spawner.java

@@ -21,14 +21,12 @@
 
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -72,27 +70,23 @@ void spawnNativePluginControllers(final Environment environment) throws IOExcept
          * For each plugin, attempt to spawn the controller daemon. Silently ignore any plugin that
          * don't include a controller for the correct platform.
          */
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsFile)) {
-            for (final Path plugin : stream) {
-                if (FileSystemUtils.isDesktopServicesStore(plugin)) {
-                    continue;
-                }
-                final PluginInfo info = PluginInfo.readFromProperties(plugin);
-                final Path spawnPath = Platforms.nativeControllerPath(plugin);
-                if (!Files.isRegularFile(spawnPath)) {
-                    continue;
-                }
-                if (!info.hasNativeController()) {
-                    final String message = String.format(
-                            Locale.ROOT,
-                            "plugin [%s] does not have permission to fork native controller",
-                            plugin.getFileName());
-                    throw new IllegalArgumentException(message);
-                }
-                final Process process =
-                        spawnNativePluginController(spawnPath, environment.tmpFile());
-                processes.add(process);
+        List<Path> paths = PluginInfo.extractAllPlugins(pluginsFile);
+        for (Path plugin : paths) {
+            final PluginInfo info = PluginInfo.readFromProperties(plugin);
+            final Path spawnPath = Platforms.nativeControllerPath(plugin);
+            if (!Files.isRegularFile(spawnPath)) {
+                continue;
             }
+            if (!info.hasNativeController()) {
+                final String message = String.format(
+                    Locale.ROOT,
+                    "plugin [%s] does not have permission to fork native controller",
+                    plugin.getFileName());
+                throw new IllegalArgumentException(message);
+            }
+            final Process process =
+                spawnNativePluginController(spawnPath, environment.tmpFile());
+            processes.add(process);
         }
     }
 

core/src/main/java/org/elasticsearch/plugins/MetaPluginInfo.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Properties;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * An in-memory representation of the meta plugin descriptor.
+ */
+public class MetaPluginInfo {
+    static final String ES_META_PLUGIN_PROPERTIES = "meta-plugin-descriptor.properties";
+
+    private final String name;
+    private final String description;
+
+    /**
+     * Construct plugin info.
+     *
+     * @param name                the name of the plugin
+     * @param description         a description of the plugin
+     */
+    private MetaPluginInfo(String name, String description) {
+        this.name = name;
+        this.description = description;
+    }
+
+    /**
+     * @return Whether the provided {@code path} is a meta plugin.
+     */
+    public static boolean isMetaPlugin(final Path path) {
+        return Files.exists(path.resolve(ES_META_PLUGIN_PROPERTIES));
+    }
+
+    /**
+     * @return Whether the provided {@code path} is a meta properties file.
+     */
+    public static boolean isPropertiesFile(final Path path) {
+        return ES_META_PLUGIN_PROPERTIES.equals(path.getFileName().toString());
+    }
+
+    /** reads (and validates) meta plugin metadata descriptor file */
+
+    /**
+     * Reads and validates the meta plugin descriptor file.
+     *
+     * @param path the path to the root directory for the meta plugin
+     * @return the meta plugin info
+     * @throws IOException if an I/O exception occurred reading the meta plugin descriptor
+     */
+    public static MetaPluginInfo readFromProperties(final Path path) throws IOException {
+        final Path descriptor = path.resolve(ES_META_PLUGIN_PROPERTIES);
+
+        final Map<String, String> propsMap;
+        {
+            final Properties props = new Properties();
+            try (InputStream stream = Files.newInputStream(descriptor)) {
+                props.load(stream);
+            }
+            propsMap = props.stringPropertyNames().stream().collect(Collectors.toMap(Function.identity(), props::getProperty));
+        }
+
+        final String name = propsMap.remove("name");
+        if (name == null || name.isEmpty()) {
+            throw new IllegalArgumentException(
+                    "property [name] is missing for meta plugin in [" + descriptor + "]");
+        }
+        final String description = propsMap.remove("description");
+        if (description == null) {
+            throw new IllegalArgumentException(
+                    "property [description] is missing for meta plugin [" + name + "]");
+        }
+
+        if (propsMap.isEmpty() == false) {
+            throw new IllegalArgumentException("Unknown properties in meta plugin descriptor: " + propsMap.keySet());
+        }
+
+        return new MetaPluginInfo(name, description);
+    }
+
+    /**
+     * The name of the meta plugin.
+     *
+     * @return the meta plugin name
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * The description of the meta plugin.
+     *
+     * @return the meta plugin description
+     */
+    public String getDescription() {
+        return description;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        MetaPluginInfo that = (MetaPluginInfo) o;
+
+        if (!name.equals(that.name)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return name.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder information = new StringBuilder()
+                .append("- Plugin information:\n")
+                .append("Name: ").append(name).append("\n")
+                .append("Description: ").append(description);
+        return information.toString();
+    }
+
+}

core/src/main/java/org/elasticsearch/plugins/PluginInfo.java

@@ -22,7 +22,9 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -31,14 +33,19 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -125,7 +132,46 @@ public void writeTo(final StreamOutput out) throws IOException {
         }
     }
 
-    /** reads (and validates) plugin metadata descriptor file */
+    /**
+     * Extracts all {@link PluginInfo} from the provided {@code rootPath} expanding meta plugins if needed.
+     * @param rootPath the path where the plugins are installed
+     * @return A list of all plugin paths installed in the {@code rootPath}
+     * @throws IOException if an I/O exception occurred reading the plugin descriptors
+     */
+    public static List<Path> extractAllPlugins(final Path rootPath) throws IOException {
+        final List<Path> plugins = new LinkedList<>();  // order is already lost, but some filesystems have it
+        final Set<String> seen = new HashSet<>();
+        if (Files.exists(rootPath)) {
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(rootPath)) {
+                for (Path plugin : stream) {
+                    if (FileSystemUtils.isDesktopServicesStore(plugin) ||
+                            plugin.getFileName().toString().startsWith(".removing-")) {
+                        continue;
+                    }
+                    if (seen.add(plugin.getFileName().toString()) == false) {
+                        throw new IllegalStateException("duplicate plugin: " + plugin);
+                    }
+                    if (MetaPluginInfo.isMetaPlugin(plugin)) {
+                        try (DirectoryStream<Path> subStream = Files.newDirectoryStream(plugin)) {
+                            for (Path subPlugin : subStream) {
+                                if (MetaPluginInfo.isPropertiesFile(subPlugin) ||
+                                        FileSystemUtils.isDesktopServicesStore(subPlugin)) {
+                                    continue;
+                                }
+                                if (seen.add(subPlugin.getFileName().toString()) == false) {
+                                    throw new IllegalStateException("duplicate plugin: " + subPlugin);
+                                }
+                                plugins.add(subPlugin);
+                            }
+                        }
+                    } else {
+                        plugins.add(plugin);
+                    }
+                }
+            }
+        }
+        return plugins;
+    }
 
     /**
      * Reads and validates the plugin descriptor file.
@@ -341,16 +387,19 @@ public int hashCode() {
 
     @Override
     public String toString() {
+        return toString("");
+    }
+
+    public String toString(String prefix) {
         final StringBuilder information = new StringBuilder()
-                .append("- Plugin information:\n")
-                .append("Name: ").append(name).append("\n")
-                .append("Description: ").append(description).append("\n")
-                .append("Version: ").append(version).append("\n")
-                .append("Native Controller: ").append(hasNativeController).append("\n")
-                .append("Requires Keystore: ").append(requiresKeystore).append("\n")
-                .append("Extended Plugins: ").append(extendedPlugins).append("\n")
-                .append(" * Classname: ").append(classname);
+            .append(prefix).append("- Plugin information:\n")
+            .append(prefix).append("Name: ").append(name).append("\n")
+            .append(prefix).append("Description: ").append(description).append("\n")
+            .append(prefix).append("Version: ").append(version).append("\n")
+            .append(prefix).append("Native Controller: ").append(hasNativeController).append("\n")
+            .append(prefix).append("Requires Keystore: ").append(requiresKeystore).append("\n")
+            .append(prefix).append("Extended Plugins: ").append(extendedPlugins).append("\n")
+            .append(prefix).append(" * Classname: ").append(classname);
         return information.toString();
     }
-
 }