Merge branch 'master' into security_authz_engine

Author: jaymode

TESTING.asciidoc

@@ -631,3 +631,13 @@ inside `/etc/hosts`, e.g.:
 255.255.255.255 broadcasthost
 ::1             localhost ElasticMBP.local`
 ....
+
+== Benchmarking
+
+For changes that might affect the performance characteristics of Elasticsearch
+you should also run macrobenchmarks. We maintain a macrobenchmarking tool
+called https://github.com/elastic/rally[Rally]
+which you can use to measure the performance impact. It comes with a set of
+default benchmarks that we also
+https://elasticsearch-benchmarks.elastic.co/[run every night]. To get started,
+please see https://esrally.readthedocs.io/en/stable/[Rally's documentation].

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

@@ -55,4 +55,3 @@ public TemporalAccessor parseJodaDate() {
         return jodaFormatter.parse("1234567890");
     }
 }
-

build.gradle

@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "backporting https://github.com/elastic/elasticsearch/pull/37639" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
@@ -221,14 +221,6 @@ allprojects {
     "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
     // for transport client
@@ -240,19 +232,6 @@ allprojects {
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
-  // substitute unreleased versions with projects that check out and build locally
-  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
-    Version unreleased = unreleasedVersion.version
-    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
-    if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
-    }
-  }
 
   /*
    * Gradle only resolve project substitutions during dependency resolution but

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -392,7 +392,7 @@ class BuildPlugin implements Plugin<Project> {
     static void requireJavaHome(Task task, int version) {
         Project rootProject = task.project.rootProject // use root project for global accounting
         if (rootProject.hasProperty('requiredJavaVersions') == false) {
-            rootProject.rootProject.ext.requiredJavaVersions = [:].withDefault{key -> return []}
+            rootProject.rootProject.ext.requiredJavaVersions = [:]
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
                 List<String> messages = []
                 for (entry in rootProject.requiredJavaVersions) {
@@ -415,7 +415,7 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException("JAVA${version}_HOME required to run task:\n${task}")
             }
         } else {
-            rootProject.requiredJavaVersions.get(version).add(task)
+            rootProject.requiredJavaVersions.getOrDefault(version, []).add(task)
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
@@ -171,6 +172,12 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        if (distro.equals("integ-test-zip")) {
+            // short circuit integ test so it doesn't complicate the rest of the distribution setup below
+            project.dependencies.add(configuration.name,
+                    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
+            return
+        }
         // TEMP HACK
         // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
         if (distro.equals('oss-zip')) {
@@ -180,22 +187,31 @@ class ClusterFormationTasks {
             distro = 'default'
         }
         // END TEMP HACK
-        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+        if (['oss', 'default'].contains(distro) == false) {
             throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
         }
         Version version = Version.fromString(elasticsearchVersion)
-        if (version.before('6.3.0') && distro.startsWith('oss-')) {
-            distro = distro.substring('oss-'.length())
-        }
-        String group = "downloads.zip"
-        if (distro.equals("integ-test-zip")) {
-            group = "org.elasticsearch.distribution.integ-test-zip"
-        }
+        String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
         String artifactName = 'elasticsearch'
         if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
+        String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
+        Object dependency
+        boolean internalBuild = project.hasProperty('bwcVersions')
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
+        if (project.hasProperty('bwcVersions')) {
+            // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
+            unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
+        }
+        if (unreleasedInfo != null) {
+            dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+        } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
+            dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
+        } else {
+            dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
+        }
+        project.dependencies.add(configuration.name, dependency)
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.FileContentsTask
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.gradle.api.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionAdapter
@@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin<Project> {
             upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
         }
 
+        List<Object> dependencies = new ArrayList<>()
         DISTRIBUTIONS.each {
             // Adds a dependency for the current version
-            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
+            dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }
 
-        UPGRADE_FROM_ARCHIVES.each {
-            // The version of elasticsearch that we upgrade *from*
-            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
-            if (upgradeFromVersion.onOrAfter('6.3.0')) {
-                project.dependencies.add(PACKAGING_CONFIGURATION,
-                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+        // The version of elasticsearch that we upgrade *from*
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
+        if (unreleasedInfo != null) {
+            // handle snapshots pointing to bwc build
+            UPGRADE_FROM_ARCHIVES.each {
+                dependencies.add(project.dependencies.project(
+                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add(project.dependencies.project(
+                            path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
+                }
+            }
+        } else {
+            UPGRADE_FROM_ARCHIVES.each {
+                // The version of elasticsearch that we upgrade *from*
+                dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                }
             }
         }
 
+        for (Object dependency : dependencies) {
+            project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
+        }
+
         project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -20,6 +20,8 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -86,6 +88,7 @@ public class VersionCollection {
 
     private final Version currentVersion;
     private final Map<Integer, List<Version>> groupByMajor;
+    private final Map<Version, UnreleasedVersionInfo> unreleased;
 
     public class UnreleasedVersionInfo {
         public final Version version;
@@ -129,6 +132,16 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
         assertCurrentVersionMatchesParsed(currentVersionProperty);
 
         assertNoOlderThanTwoMajors();
+
+        Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
+        for (Version unreleasedVersion : getUnreleased()) {
+            if (unreleasedVersion.equals(currentVersion)) {
+                continue;
+            }
+            unreleased.put(unreleasedVersion,
+                new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
+        }
+        this.unreleased = Collections.unmodifiableMap(unreleased);
     }
 
     private void assertNoOlderThanTwoMajors() {
@@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
         }
     }
 
+    /**
+      * Returns info about the unreleased version, or {@code null} if the version is released.
+      */
+    public UnreleasedVersionInfo unreleasedInfo(Version version) {
+        return unreleased.get(version);
+    }
+
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
         getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -96,7 +96,7 @@ public void apply(Project project) {
             if (dockerComposeSupported(project) == false) {
                 project.getLogger().warn(
                     "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
-                        "but none could not be found so these will be skipped", project.getPath()
+                        "but none could be found so these will be skipped", project.getPath()
                 );
                 tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
                     task.setEnabled(false)

buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java

@@ -40,13 +40,15 @@
 
 public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
 
-    private static List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
+    private static final List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
         Arrays.stream(
             Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples"))
                 .split(File.pathSeparator)
         ).map(File::new).collect(Collectors.toList())
     );
 
+    private static final String BUILD_TOOLS_VERSION = Objects.requireNonNull(System.getProperty("test.version_under_test"));
+
     @Rule
     public TemporaryFolder tmpDir = new TemporaryFolder();
 
@@ -96,7 +98,8 @@ public void testCurrentExamplePlugin() throws IOException {
 
     private void adaptBuildScriptForTest() throws IOException {
         // Add the local repo as a build script URL so we can pull in build-tools and apply the plugin under test
-        // + is ok because we have no other repo and just want to pick up latest
+        // we need to specify the exact version of build-tools because gradle automatically adds its plugin portal
+        // which appears to mirror jcenter, opening us up to pulling a "later" version of build-tools
         writeBuildScript(
             "buildscript {\n" +
                 "    repositories {\n" +
@@ -105,7 +108,7 @@ private void adaptBuildScriptForTest() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "    dependencies {\n" +
-                "        classpath \"org.elasticsearch.gradle:build-tools:+\"\n" +
+                "        classpath \"org.elasticsearch.gradle:build-tools:" + BUILD_TOOLS_VERSION + "\"\n" +
                 "    }\n" +
                 "}\n"
         );