Merge branch 'master' into feature/extensible-values-source

 Conflicts:
	server/src/main/java/org/elasticsearch/index/IndexModule.java
	server/src/main/java/org/elasticsearch/index/IndexService.java
	server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
	server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
	server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
	server/src/main/java/org/elasticsearch/node/Node.java
	test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
	x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java

Author: not-napoleon

benchmarks/build.gradle

@@ -41,7 +41,6 @@ dependencies {
   runtime 'org.apache.commons:commons-math3:3.2'
 }
 
-compileJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked,-processing"
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])

build.gradle

@@ -34,6 +34,7 @@ import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 plugins {
   id 'lifecycle-base'
+  id 'elasticsearch.docker-support'
   id 'elasticsearch.global-build-info'
   id "com.diffplug.gradle.spotless" version "3.24.2" apply false
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -32,7 +32,6 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
-import org.elasticsearch.gradle.testclusters.TestDistribution
 import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.Action
 import org.gradle.api.GradleException
@@ -83,8 +82,6 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
-import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable
-import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -186,51 +183,6 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    static void requireDocker(final Task task) {
-        final Project rootProject = task.project.rootProject
-        ExtraPropertiesExtension ext = rootProject.extensions.getByType(ExtraPropertiesExtension)
-
-        if (rootProject.hasProperty('requiresDocker') == false) {
-            /*
-             * This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
-             * that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
-             * in the task graph, then we check two things:
-             *  - the Docker binary is available
-             *  - we can execute a Docker command that requires privileges
-             *
-             *  If either of these fail, we fail the build.
-             */
-
-            // check if the Docker binary exists and record its path
-            final String dockerBinary = getDockerPath().orElse(null)
-
-            final boolean buildDocker
-            final String buildDockerProperty = System.getProperty("build.docker")
-            if (buildDockerProperty == null) {
-                buildDocker = dockerBinary != null
-            } else if (buildDockerProperty == "true") {
-                buildDocker = true
-            } else if (buildDockerProperty == "false") {
-                buildDocker = false
-            } else {
-                throw new IllegalArgumentException(
-                        "expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
-            }
-
-            ext.set('buildDocker', buildDocker)
-            ext.set('requiresDocker', [])
-            rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
-                final List<String> tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List<Task>).collect { "  ${it.path}".toString()}
-
-                if (tasks.isEmpty() == false) {
-                    assertDockerIsAvailable(task.project, tasks)
-                }
-            }
-        }
-
-        (ext.get('requiresDocker') as List<Task>).add(task)
-    }
-
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
     static void requireJavaHome(Task task, int version) {
         // use root project for global accounting

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -22,8 +22,11 @@
 import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.elasticsearch.gradle.docker.DockerSupportPlugin;
+import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.tool.Boilerplate;
 import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
@@ -37,6 +40,7 @@
 import org.gradle.api.credentials.HttpHeaderCredentials;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.authentication.http.HttpHeaderAuthentication;
@@ -71,11 +75,17 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     public void apply(Project project) {
         // this is needed for isInternal
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
+
+        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+            project.getGradle().getSharedServices(),
+            DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+        );
 
         distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
             Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
             Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
-            return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration);
+            return new ElasticsearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration);
         });
         project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
 

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -19,13 +19,16 @@
 
 package org.elasticsearch.gradle;
 
+import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.gradle.api.Buildable;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskDependency;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Locale;
 
@@ -110,6 +113,7 @@ public String toString() {
     }
 
     private final String name;
+    private final Provider<DockerSupportService> dockerSupport;
     // pkg private so plugin can configure
     final Configuration configuration;
     private final Extracted extracted;
@@ -119,21 +123,25 @@ public String toString() {
     private final Property<Platform> platform;
     private final Property<Flavor> flavor;
     private final Property<Boolean> bundledJdk;
+    private final Property<Boolean> failIfUnavailable;
 
     ElasticsearchDistribution(
         String name,
         ObjectFactory objectFactory,
+        Provider<DockerSupportService> dockerSupport,
         Configuration fileConfiguration,
         Configuration extractedConfiguration
     ) {
         this.name = name;
+        this.dockerSupport = dockerSupport;
         this.configuration = fileConfiguration;
         this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
         this.type = objectFactory.property(Type.class);
         this.type.convention(Type.ARCHIVE);
         this.platform = objectFactory.property(Platform.class);
         this.flavor = objectFactory.property(Flavor.class);
         this.bundledJdk = objectFactory.property(Boolean.class);
+        this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
         this.extracted = new Extracted(extractedConfiguration);
     }
 
@@ -182,6 +190,14 @@ public void setBundledJdk(Boolean bundledJdk) {
         this.bundledJdk.set(bundledJdk);
     }
 
+    public boolean getFailIfUnavailable() {
+        return this.failIfUnavailable.get();
+    }
+
+    public void setFailIfUnavailable(boolean failIfUnavailable) {
+        this.failIfUnavailable.set(failIfUnavailable);
+    }
+
     @Override
     public String toString() {
         return configuration.getSingleFile().toString();
@@ -203,6 +219,13 @@ public Extracted getExtracted() {
 
     @Override
     public TaskDependency getBuildDependencies() {
+        // For non-required Docker distributions, skip building the distribution is Docker is unavailable
+        if (getType() == Type.DOCKER
+            && getFailIfUnavailable() == false
+            && dockerSupport.get().getDockerAvailability().isAvailable == false) {
+            return task -> Collections.emptySet();
+        }
+
         return configuration.getBuildDependencies();
     }
 
@@ -222,7 +245,7 @@ void finalizeValues() {
         if (getType() == Type.INTEG_TEST_ZIP) {
             if (platform.getOrNull() != null) {
                 throw new IllegalArgumentException(
-                    "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"
+                    "platform cannot be set on elasticsearch distribution [" + name + "] of type [integ_test_zip]"
                 );
             }
             if (flavor.getOrNull() != null) {
@@ -232,12 +255,18 @@ void finalizeValues() {
             }
             if (bundledJdk.getOrNull() != null) {
                 throw new IllegalArgumentException(
-                    "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"
+                    "bundledJdk cannot be set on elasticsearch distribution [" + name + "] of type [integ_test_zip]"
                 );
             }
             return;
         }
 
+        if (getType() != Type.DOCKER && failIfUnavailable.get() == false) {
+            throw new IllegalArgumentException(
+                "failIfUnavailable cannot be 'false' on elasticsearch distribution [" + name + "] of type [" + getType() + "]"
+            );
+        }
+
         if (getType() == Type.ARCHIVE) {
             // defaults for archive, set here instead of via convention so integ-test-zip can verify they are not set
             if (platform.isPresent() == false) {
@@ -246,7 +275,12 @@ void finalizeValues() {
         } else { // rpm, deb or docker
             if (platform.isPresent()) {
                 throw new IllegalArgumentException(
-                    "platform not allowed for elasticsearch distribution [" + name + "] of type [" + getType() + "]"
+                    "platform cannot be set on elasticsearch distribution [" + name + "] of type [" + getType() + "]"
+                );
+            }
+            if (getType() == Type.DOCKER && bundledJdk.isPresent()) {
+                throw new IllegalArgumentException(
+                    "bundledJdk cannot be set on elasticsearch distribution [" + name + "] of type [docker]"
                 );
             }
         }

buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportPlugin.java

@@ -0,0 +1,61 @@
+package org.elasticsearch.gradle.docker;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.provider.Provider;
+
+import java.io.File;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Plugin providing {@link DockerSupportService} for detecting Docker installations and determining requirements for Docker-based
+ * Elasticsearch build tasks.
+ * <p>
+ * Additionally registers a task graph listener used to assert a compatible Docker installation exists when task requiring Docker are
+ * scheduled for execution. Tasks may declare a Docker requirement via an extra property. If a compatible Docker installation is not
+ * available on the build system an exception will be thrown prior to task execution.
+ *
+ * <pre>
+ *     task myDockerTask {
+ *         ext.requiresDocker = true
+ *     }
+ * </pre>
+ */
+public class DockerSupportPlugin implements Plugin<Project> {
+    public static final String DOCKER_SUPPORT_SERVICE_NAME = "dockerSupportService";
+    public static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions";
+    public static final String REQUIRES_DOCKER_ATTRIBUTE = "requiresDocker";
+
+    @Override
+    public void apply(Project project) {
+        if (project != project.getRootProject()) {
+            throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
+        }
+
+        Provider<DockerSupportService> dockerSupportServiceProvider = project.getGradle()
+            .getSharedServices()
+            .registerIfAbsent(
+                DOCKER_SUPPORT_SERVICE_NAME,
+                DockerSupportService.class,
+                spec -> spec.parameters(
+                    params -> { params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE)); }
+                )
+            );
+
+        // Ensure that if any tasks declare they require docker, we assert an available Docker installation exists
+        project.getGradle().getTaskGraph().whenReady(graph -> {
+            List<String> dockerTasks = graph.getAllTasks().stream().filter(task -> {
+                ExtraPropertiesExtension ext = task.getExtensions().getExtraProperties();
+                return ext.has(REQUIRES_DOCKER_ATTRIBUTE) && (boolean) ext.get(REQUIRES_DOCKER_ATTRIBUTE);
+            }).map(Task::getPath).collect(Collectors.toList());
+
+            if (dockerTasks.isEmpty() == false) {
+                dockerSupportServiceProvider.get().failIfDockerUnavailable(dockerTasks);
+            }
+        });
+    }
+
+}