Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x:
  Fix link to parent join (#28085)
  Introduce Gradle wrapper
  Ignore GIT_COMMIT when calculating commit hash
  Plugins: Add plugin extension capabilities (#27881)

Author: martijnvg

.gitignore

@@ -20,11 +20,6 @@ nbactions.xml
 .gradle/
 build/
 
-# gradle wrapper 
-/gradle/
-gradlew
-gradlew.bat
-
 # maven stuff (to be removed when trunk becomes 4.x)
 *-execution-hints.log
 target/

CONTRIBUTING.md

@@ -92,9 +92,8 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-Make sure you have [Gradle](http://gradle.org) installed, as
-Elasticsearch uses it as its build system. Gradle must be at least
-version 4.3 in order to build successfully.
+Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
+using the wrapper via the `gradlew` script in the root of the repository.
 
 Eclipse users can automatically configure their IDE: `gradle eclipse`
 then `File: Import: Existing Projects into Workspace`. Select the

build.gradle

@@ -25,7 +25,13 @@ import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
+import org.gradle.api.tasks.wrapper.Wrapper
+import org.gradle.api.tasks.wrapper.Wrapper.DistributionType
+import org.gradle.util.GradleVersion
+import org.gradle.util.DistributionLocator
+
 import java.nio.file.Path
+import java.security.MessageDigest
 
 // common maven publishing configuration
 subprojects {
@@ -401,6 +407,27 @@ task run(type: Run) {
   impliesSubProjects = true
 }
 
+task wrapper(type: Wrapper)
+
+gradle.projectsEvaluated {
+
+  allprojects {
+    tasks.withType(Wrapper) { Wrapper wrapper ->
+      wrapper.distributionType = DistributionType.ALL
+
+      wrapper.doLast {
+        final DistributionLocator locator = new DistributionLocator()
+        final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
+        final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH))
+        final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
+        final String sha256Sum = new String(sha256Uri.toURL().bytes)
+        wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
+      }
+    }
+  }
+
+}
+
 /* Remove assemble on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -21,6 +21,8 @@ package org.elasticsearch.gradle
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import nebula.plugin.extraconfigurations.ProvidedBasePlugin
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.eclipse.jgit.lib.Constants
+import org.eclipse.jgit.lib.RepositoryBuilder
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.GradleException
 import org.gradle.api.InvalidUserDataException
@@ -35,7 +37,6 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.file.CopySpec
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
@@ -509,6 +510,17 @@ class BuildPlugin implements Plugin<Project> {
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {
                     logger.warn('Building without git revision id.')
                     jarTask.manifest.attributes('Change': 'Unknown')
+                } else {
+                    /*
+                     * The info-scm plugin assumes that if GIT_COMMIT is set it was set by Jenkins to the commit hash for this build.
+                     * However, that assumption is wrong as this build could be a sub-build of another Jenkins build for which GIT_COMMIT
+                     * is the commit hash for that build. Therefore, if GIT_COMMIT is set we calculate the commit hash ourselves.
+                     */
+                    if (System.getenv("GIT_COMMIT") != null) {
+                        final String hash = new RepositoryBuilder().findGitDir(project.buildDir).build().resolve(Constants.HEAD).name
+                        final String shortHash = hash?.substring(0, 7)
+                        jarTask.manifest.attributes('Change': shortHash)
+                    }
                 }
             }
             // add license/notice files

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -39,6 +39,10 @@ class PluginPropertiesExtension {
     @Input
     String classname
 
+    /** Other plugins this plugin extends through SPI */
+    @Input
+    List<String> extendedPlugins = []
+
     @Input
     boolean hasNativeController = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -80,6 +80,7 @@ class PluginPropertiesTask extends Copy {
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
+            'extendedPlugins': extension.extendedPlugins.join(','),
             'hasNativeController': extension.hasNativeController,
             'requiresKeystore': extension.requiresKeystore
         ]

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -40,6 +40,9 @@ java.version=${javaVersion}
 elasticsearch.version=${elasticsearchVersion}
 ### optional elements for plugins:
 #
+#  'extended.plugins': other plugins this plugin extends through SPI
+extended.plugins=${extendedPlugins}
+#
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
 #

core/build.gradle

@@ -38,6 +38,9 @@ archivesBaseName = 'elasticsearch'
 
 dependencies {
 
+  compileOnly project(':libs:plugin-classloader')
+  testRuntime project(':libs:plugin-classloader')
+
   // lucene
   compile "org.apache.lucene:lucene-core:${versions.lucene}"
   compile "org.apache.lucene:lucene-analyzers-common:${versions.lucene}"

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -33,6 +33,7 @@
 import java.security.ProtectionDomain;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Predicate;
 
 /** custom policy for union of static and dynamic permissions */
@@ -49,9 +50,9 @@ final class ESPolicy extends Policy {
     final PermissionCollection dynamic;
     final Map<String,Policy> plugins;
 
-    ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
-        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath());
-        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptySet());
+    ESPolicy(Map<String, URL> codebases, PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
+        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), codebases);
+        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptyMap());
         if (filterBadDefaults) {
             this.system = new SystemPolicy(Policy.getPolicy());
         } else {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -48,10 +48,13 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addSingleFilePath;
@@ -116,7 +119,8 @@ private Security() {}
     static void configure(Environment environment, boolean filterBadDefaults) throws IOException, NoSuchAlgorithmException {
 
         // enable security policy: union of template and environment-based paths, and possibly plugin permissions
-        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
+        Map<String, URL> codebases = getCodebaseJarMap(JarHell.parseClassPath());
+        Policy.setPolicy(new ESPolicy(codebases, createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
 
         // enable security manager
         final String[] classesThatCanExit =
@@ -130,6 +134,27 @@ static void configure(Environment environment, boolean filterBadDefaults) throws
         selfTest();
     }
 
+    /**
+     * Return a map from codebase name to codebase url of jar codebases used by ES core.
+     */
+    @SuppressForbidden(reason = "find URL path")
+    static Map<String, URL> getCodebaseJarMap(Set<URL> urls) {
+        Map<String, URL> codebases = new LinkedHashMap<>(); // maintain order
+        for (URL url : urls) {
+            try {
+                String fileName = PathUtils.get(url.toURI()).getFileName().toString();
+                if (fileName.endsWith(".jar") == false) {
+                    // tests :(
+                    continue;
+                }
+                codebases.put(fileName, url);
+            } catch (URISyntaxException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return codebases;
+    }
+
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
@@ -174,7 +199,7 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
                 }
 
                 // parse the plugin's policy file into a set of permissions
-                Policy policy = readPolicy(policyFile.toUri().toURL(), codebases);
+                Policy policy = readPolicy(policyFile.toUri().toURL(), getCodebaseJarMap(codebases));
 
                 // consult this policy for each of the plugin's jars:
                 for (URL url : codebases) {
@@ -197,21 +222,20 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
      * would map to full URL.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
-    static Policy readPolicy(URL policyFile, Set<URL> codebases) {
+    static Policy readPolicy(URL policyFile, Map<String, URL> codebases) {
         try {
             List<String> propertiesSet = new ArrayList<>();
             try {
                 // set codebase properties
-                for (URL url : codebases) {
-                    String fileName = PathUtils.get(url.toURI()).getFileName().toString();
-                    if (fileName.endsWith(".jar") == false) {
-                        continue; // tests :(
-                    }
+                for (Map.Entry<String,URL> codebase : codebases.entrySet()) {
+                    String name = codebase.getKey();
+                    URL url = codebase.getValue();
+
                     // We attempt to use a versionless identifier for each codebase. This assumes a specific version
                     // format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity
                     // only means policy grants would need to include the entire jar filename as they always have before.
-                    String property = "codebase." + fileName;
-                    String aliasProperty = "codebase." + fileName.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
+                    String property = "codebase." + name;
+                    String aliasProperty = "codebase." + name.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
                     if (aliasProperty.equals(property) == false) {
                         propertiesSet.add(aliasProperty);
                         String previous = System.setProperty(aliasProperty, url.toString());