Improve Docker image caching and testing (#78552)

Firstly: we tag our Docker images with various pieces of information,
including a timestamp for when the image was built. However, this makes
it impossible completely cache the image. When developing the Docker
images, it's very tedious to completely rebuild an image for every
single change. Therefore, provided we're not building a proper release
build, we fix the build time to midnight so that the Docker build cache
is usable.

Secondly: the `DockerBuildTask` outputs a marker file to indicate that
an image has been built, but that isn't enough for a meaningful
up-to-date check by Gradle. Improve this by fetching the newly-built
image's hash, and writing that to the output file.

Thirdly: improve the Docker tests to make them more ergonomic, and also
disable `ingest.geoip.downloader.enabled` by default.

Fourthly: add missing test coverage for sourcing settings from env vars.

Author: pugnascotia

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -29,10 +29,17 @@
 import org.gradle.workers.WorkParameters;
 import org.gradle.workers.WorkerExecutor;
 
-import javax.inject.Inject;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.Arrays;
+import java.util.List;
+import javax.inject.Inject;
 
+/**
+ * This task wraps up the details of building a Docker image, including adding a pull
+ * mechanism that can retry, and emitting the image SHA as a task output.
+ */
 public class DockerBuildTask extends DefaultTask {
     private static final Logger LOGGER = Logging.getLogger(DockerBuildTask.class);
 
@@ -167,6 +174,8 @@ public void execute() {
                 parameters.getBaseImages().get().forEach(this::pullBaseImage);
             }
 
+            final List<String> tags = parameters.getTags().get();
+
             LoggedExec.exec(execOperations, spec -> {
                 spec.executable("docker");
 
@@ -176,17 +185,32 @@ public void execute() {
                     spec.args("--no-cache");
                 }
 
-                parameters.getTags().get().forEach(tag -> spec.args("--tag", tag));
+                tags.forEach(tag -> spec.args("--tag", tag));
 
                 parameters.getBuildArgs().get().forEach((k, v) -> spec.args("--build-arg", k + "=" + v));
             });
 
+            // Fetch the Docker image's hash, and write it to desk as the task's output. Doing this allows us
+            // to do proper up-to-date checks in Gradle.
             try {
-                parameters.getMarkerFile().getAsFile().get().createNewFile();
+                final String checksum = getImageChecksum(tags.get(0));
+                Files.writeString(parameters.getMarkerFile().getAsFile().get().toPath(), checksum + "\n");
             } catch (IOException e) {
-                throw new RuntimeException("Failed to create marker file", e);
+                throw new RuntimeException("Failed to write marker file", e);
             }
         }
+
+        private String getImageChecksum(String imageTag) {
+            final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
+
+            execOperations.exec(spec -> {
+                spec.setCommandLine("docker", "inspect", "--format", "{{ .Id }}", imageTag);
+                spec.setStandardOutput(stdout);
+                spec.setIgnoreExitValue(false);
+            });
+
+            return stdout.toString().trim();
+        }
     }
 
     interface Parameters extends WorkParameters {

distribution/docker/build.gradle

@@ -9,6 +9,7 @@ import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 import java.nio.file.Path
+import java.time.temporal.ChronoUnit
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
@@ -81,10 +82,17 @@ dependencies {
 ext.expansions = { Architecture architecture, DockerBase base ->
   def (major,minor) = VersionProperties.elasticsearch.split("\\.")
 
+  // We tag our Docker images with various pieces of information, including a timestamp
+  // for when the image was built. However, this makes it impossible completely cache
+  // the image. When developing the Docker images, it's very tedious to completely rebuild
+  // an image for every single change. Therefore, outside of CI, we fix the
+  // build time to midnight so that the Docker build cache is usable.
+  def buildDate = BuildParams.isCi() ? BuildParams.buildDate : BuildParams.buildDate.truncatedTo(ChronoUnit.DAYS)
+
   return [
     'base_image'         : base.image,
     'bin_dir'            : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
-    'build_date'         : BuildParams.buildDate,
+    'build_date'         : buildDate,
     'config_dir'         : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
     'git_revision'       : BuildParams.gitRevision,
     'license'            : base == DockerBase.IRON_BANK ? 'Elastic License 1.0' : 'Elastic-License-2.0',
@@ -341,6 +349,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
 
       dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() })
 
+      noCache = BuildParams.isCi
       tags = generateTags(base)
 
       if (base == DockerBase.IRON_BANK) {
@@ -402,11 +411,11 @@ void addBuildEssDockerImageTask(Architecture architecture) {
     tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) {
 
       TaskProvider<DockerBuildTask> buildCloudTask = tasks.named(taskName("build", architecture, DockerBase.CLOUD, "DockerImage"))
-      dependsOn(buildCloudTask)
-      dependsOn(buildContextTask)
+      inputs.files(buildCloudTask)
 
       dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() })
 
+      noCache = BuildParams.isCi
       baseImages = []
       tags = generateTags(base)