Add modules to distributions, and move lang-expression and lang-groovy to them

build.gradle

@@ -109,6 +109,7 @@ subprojects {
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
     "org.elasticsearch:test-framework:${version}": ':test-framework',
+    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
     "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -41,10 +41,16 @@ public class PluginBuildPlugin extends BuildPlugin {
             String name = project.pluginProperties.extension.name
             project.jar.baseName = name
             project.bundlePlugin.baseName = name
+
             project.integTest.dependsOn(project.bundlePlugin)
-            project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
             project.tasks.run.dependsOn(project.bundlePlugin)
-            project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+            if (project.path.startsWith(':modules:')) {
+                project.integTest.clusterConfig.module(project)
+                project.tasks.run.clusterConfig.module(project)
+            } else {
+                project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+                project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+            }
         }
         createIntegTestTask(project)
         createBundleTask(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -27,7 +27,7 @@ import org.gradle.api.tasks.Input
 class ClusterConfiguration {
 
     @Input
-    String distribution = 'zip'
+    String distribution = 'integ-test-zip'
 
     @Input
     int numNodes = 1
@@ -71,6 +71,8 @@ class ClusterConfiguration {
 
     LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()
 
+    List<Project> modules = new ArrayList<>()
+
     LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
 
     @Input
@@ -93,6 +95,12 @@ class ClusterConfiguration {
         plugins.put(name, pluginProject)
     }
 
+    /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */
+    @Input
+    void module(Project moduleProject) {
+        modules.add(moduleProject)
+    }
+
     @Input
     void setupCommand(String name, Object... args) {
         setupCommands.put(name, args)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -60,7 +60,12 @@ class ClusterFormationTasks {
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro) {
         String elasticsearchVersion = VersionProperties.elasticsearch
-        String packaging = distro == 'tar' ? 'tar.gz' : distro
+        String packaging = distro
+        if (distro == 'tar') {
+            packaging = 'tar.gz'
+        } else if (distro == 'integ-test-zip') {
+            packaging = 'zip'
+        }
         project.configurations {
             elasticsearchDistro
         }
@@ -103,6 +108,12 @@ class ClusterFormationTasks {
         setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node)
         setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node)
 
+        // install modules
+        for (Project module : node.config.modules) {
+            String actionName = pluginTaskName('install', module.name, 'Module')
+            setup = configureInstallModuleTask(taskName(task, node, actionName), project, setup, node, module)
+        }
+
         // install plugins
         for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
             String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
@@ -138,6 +149,7 @@ class ClusterFormationTasks {
           by the source tree. If it isn't then Bad Things(TM) will happen. */
         Task extract
         switch (node.config.distribution) {
+            case 'integ-test-zip':
             case 'zip':
                 extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
                     from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) }
@@ -286,6 +298,20 @@ class ClusterFormationTasks {
         return copyPlugins
     }
 
+    static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) {
+        if (node.config.distribution != 'integ-test-zip') {
+            throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!")
+        }
+        if (module.plugins.hasPlugin(PluginBuildPlugin) == false) {
+            throw new GradleException("Task ${name} cannot include module ${module.path} which is not an esplugin")
+        }
+        Copy installModule = project.tasks.create(name, Copy.class)
+        installModule.dependsOn(setup)
+        installModule.into(new File(node.homeDir, "modules/${module.name}"))
+        installModule.from({ project.zipTree(module.tasks.bundlePlugin.outputs.files.singleFile) })
+        return installModule
+    }
+
     static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Object plugin) {
         FileCollection pluginZip
         if (plugin instanceof Project) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -173,6 +173,7 @@ class NodeInfo {
     static File homeDir(File baseDir, String distro) {
         String path
         switch (distro) {
+            case 'integ-test-zip':
             case 'zip':
             case 'tar':
                 path = "elasticsearch-${VersionProperties.elasticsearch}"
@@ -188,8 +189,8 @@ class NodeInfo {
     }
 
     static File confDir(File baseDir, String distro) {
-        String Path
         switch (distro) {
+            case 'integ-test-zip':
             case 'zip':
             case 'tar':
                 return new File(homeDir(baseDir, distro), 'config')

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy

@@ -29,6 +29,7 @@ public class RestTestPlugin implements Plugin<Project> {
         project.pluginManager.apply(StandaloneTestBasePlugin)
 
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
+        integTest.cluster.distribution = 'zip' // rest tests should run with the real zip
         integTest.mustRunAfter(project.precommit)
         project.check.dependsOn(integTest)
     }

core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java

@@ -72,14 +72,14 @@ public class NodeInfo extends BaseNodeResponse {
     private HttpInfo http;
 
     @Nullable
-    private PluginsInfo plugins;
+    private PluginsAndModules plugins;
 
     NodeInfo() {
     }
 
     public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Map<String, String> serviceAttributes, @Nullable Settings settings,
                     @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool,
-                    @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsInfo plugins) {
+                    @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins) {
         super(node);
         this.version = version;
         this.build = build;
@@ -172,7 +172,7 @@ public HttpInfo getHttp() {
     }
 
     @Nullable
-    public PluginsInfo getPlugins() {
+    public PluginsAndModules getPlugins() {
         return this.plugins;
     }
 
@@ -217,7 +217,8 @@ public void readFrom(StreamInput in) throws IOException {
             http = HttpInfo.readHttpInfo(in);
         }
         if (in.readBoolean()) {
-            plugins = PluginsInfo.readPluginsInfo(in);
+            plugins = new PluginsAndModules();
+            plugins.readFrom(in);
         }
     }
 

core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.info;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.plugins.PluginInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Information about plugins and modules
+ */
+public class PluginsAndModules implements Streamable, ToXContent {
+    private List<PluginInfo> plugins;
+    private List<PluginInfo> modules;
+
+    public PluginsAndModules() {
+        plugins = new ArrayList<>();
+        modules = new ArrayList<>();
+    }
+
+    /**
+     * Returns an ordered list based on plugins name
+     */
+    public List<PluginInfo> getPluginInfos() {
+        List<PluginInfo> plugins = new ArrayList<>(this.plugins);
+        Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        return plugins;
+    }
+
+    /**
+     * Returns an ordered list based on modules name
+     */
+    public List<PluginInfo> getModuleInfos() {
+        List<PluginInfo> modules = new ArrayList<>(this.modules);
+        Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        return modules;
+    }
+
+    public void addPlugin(PluginInfo info) {
+        plugins.add(info);
+    }
+
+    public void addModule(PluginInfo info) {
+        modules.add(info);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        if (plugins.isEmpty() == false || modules.isEmpty() == false) {
+            throw new IllegalStateException("instance is already populated");
+        }
+        int plugins_size = in.readInt();
+        for (int i = 0; i < plugins_size; i++) {
+            plugins.add(PluginInfo.readFromStream(in));
+        }
+        int modules_size = in.readInt();
+        for (int i = 0; i < modules_size; i++) {
+            modules.add(PluginInfo.readFromStream(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeInt(plugins.size());
+        for (PluginInfo plugin : getPluginInfos()) {
+            plugin.writeTo(out);
+        }
+        out.writeInt(modules.size());
+        for (PluginInfo module : getModuleInfos()) {
+            module.writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray("plugins");
+        for (PluginInfo pluginInfo : getPluginInfos()) {
+            pluginInfo.toXContent(builder, params);
+        }
+        builder.endArray();
+        // TODO: not ideal, make a better api for this (e.g. with jar metadata, and so on)
+        builder.startArray("modules");
+        for (PluginInfo moduleInfo : getModuleInfos()) {
+            moduleInfo.toXContent(builder, params);
+        }
+        builder.endArray();
+
+        return builder;
+    }
+}