Merge remote-tracking branch 'es/master' into enrich

Author: martijnvg

buildSrc/build.gradle

@@ -66,22 +66,30 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
   throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
 }
 
-// Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11
-targetCompatibility = '11'
-sourceCompatibility = '11'
+allprojects {
+  targetCompatibility = '11'
+  sourceCompatibility = '11'
+}
 
 sourceSets {
     // We have a few classes that need to be compiled for older java versions
     minimumRuntime { }
 }
 
+configurations {
+  reaper
+}
+
 compileMinimumRuntimeJava {
     targetCompatibility = 8
     sourceCompatibility = 8
 }
 
 jar {
     from sourceSets.minimumRuntime.output
+    into('META-INF') {
+        from configurations.reaper
+    }
 }
 
 javadoc {
@@ -119,6 +127,7 @@ dependencies {
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+  reaper project('reaper')
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
   minimumRuntimeCompile localGroovy()
   minimumRuntimeCompile gradleApi()

buildSrc/reaper/build.gradle

@@ -0,0 +1,7 @@
+apply plugin: 'java'
+
+jar {
+    manifest {
+        attributes 'Main-Class': 'org.elasticsearch.gradle.reaper.Reaper'
+    }
+}

buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.reaper;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A standalone process that will reap external services after a build dies.
+ *
+ * <h2>Input</h2>
+ * Since how to reap a given service is platform and service dependent, this tool
+ * operates on system commands to execute. It takes a single argument, a directory
+ * that will contain files with reaping commands. Each line in each file will be
+ * executed with {@link Runtime#getRuntime()#exec}.
+ *
+ * The main method will wait indefinitely on the parent process (Gradle) by
+ * reading from stdin. When Gradle shuts down, whether normally or abruptly, the
+ * pipe will be broken and read will return.
+ *
+ * The reaper will then iterate over the files in the configured directory,
+ * and execute the given commands. If any commands fail, a failure message is
+ * written to stderr. Otherwise, the input file will be deleted. If no inputs
+ * produced errors, the entire input directory is deleted upon completion of reaping.
+ */
+public class Reaper implements Closeable {
+
+    private Path inputDir;
+    private boolean failed;
+
+    private Reaper(Path inputDir) {
+        this.inputDir = inputDir;
+        this.failed = false;
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 1) {
+            System.err.println("Expected one argument.\nUsage: java -jar reaper.jar <DIR_OF_REAPING_COMMANDS>");
+            System.exit(1);
+        }
+        Path inputDir = Paths.get(args[0]);
+
+        try (Reaper reaper = new Reaper(inputDir)){
+            System.in.read();
+            reaper.reap();
+        }
+    }
+
+    private void reap() {
+        try (Stream<Path> stream = Files.list(inputDir)){
+            final List<Path> inputFiles = stream.filter(p -> p.getFileName().toString().endsWith(".cmd")).collect(Collectors.toList());
+
+            for (Path inputFile : inputFiles) {
+                System.out.println("Process file: " + inputFile);
+                String line = Files.readString(inputFile);
+                System.out.println("Running command: " + line);
+                String[] command = line.split(" ");
+                Process process = Runtime.getRuntime().exec(command);
+                int ret = process.waitFor();
+
+                System.out.print("Stdout: ");
+                process.getInputStream().transferTo(System.out);
+                System.out.print("\nStderr: ");
+                process.getErrorStream().transferTo(System.out);
+                System.out.println(); // end the stream
+                if (ret != 0) {
+                    logFailure("Command [" + line + "] failed with exit code " + ret, null);
+                } else {
+                    delete(inputFile);
+                }
+            }
+        } catch (Exception e) {
+            logFailure("Failed to reap inputs", e);
+        }
+    }
+
+    private void logFailure(String message, Exception e) {
+        System.err.println(message);
+        if (e != null) {
+            e.printStackTrace(System.err);
+        }
+        failed = true;
+    }
+
+    private void delete(Path toDelete) {
+        try {
+            Files.delete(toDelete);
+        } catch (IOException e) {
+            logFailure("Failed to delete [" + toDelete + "]", e);
+        }
+    }
+
+    @Override
+    public void close() {
+        if (failed == false) {
+            try (Stream<Path> stream = Files.walk(inputDir)){
+                stream.sorted(Comparator.reverseOrder()).forEach(this::delete);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+    }
+}

buildSrc/settings.gradle

@@ -0,0 +1 @@
+include 'reaper'

buildSrc/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+
+import java.nio.file.Path;
+
+/**
+ * A plugin to handle reaping external services spawned by a build if Gradle dies.
+ */
+public class ReaperPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        if (project != project.getRootProject()) {
+            throw new IllegalArgumentException("ReaperPlugin can only be applied to the root project of a build");
+        }
+
+        Path inputDir = project.getRootDir().toPath().resolve(".gradle")
+            .resolve("reaper").resolve("build-" + ProcessHandle.current().pid());
+        ReaperService service = project.getExtensions().create("reaper", ReaperService.class,
+            project.getLogger(), project.getBuildDir().toPath(), inputDir);
+
+        project.getGradle().buildFinished(result -> service.shutdown());
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.GradleException;
+import org.gradle.api.logging.Logger;
+import org.gradle.internal.jvm.Jvm;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class ReaperService {
+
+    private Logger logger;
+    private Path buildDir;
+    private Path inputDir;
+    private Path logFile;
+    private volatile Process reaperProcess;
+
+    public ReaperService(Logger logger, Path buildDir, Path inputDir) {
+        this.logger = logger;
+        this.buildDir = buildDir;
+        this.inputDir = inputDir;
+        this.logFile = inputDir.resolve("reaper.log");
+    }
+
+    /**
+     * Register a pid that will be killed by the reaper.
+     */
+    public void registerPid(String serviceId, long pid) {
+        String[] killPidCommand = OS.<String[]>conditional()
+            .onWindows(() -> new String[]{"Taskill", "/F", "/PID", String.valueOf(pid)})
+            .onUnix(() -> new String[]{"kill", "-9", String.valueOf(pid)})
+            .supply();
+        registerCommand(serviceId, killPidCommand);
+    }
+
+    /**
+     * Register a system command that will be run by the reaper.
+     */
+    public void registerCommand(String serviceId, String... command) {
+        ensureReaperStarted();
+
+        try {
+            Files.writeString(inputDir.resolve(serviceId + ".cmd"), String.join(" ", command));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public void unregister(String serviceId) {
+        try {
+            Files.deleteIfExists(inputDir.resolve(serviceId + ".cmd"));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    void shutdown() {
+        if (reaperProcess != null) {
+            ensureReaperAlive();
+            try {
+                reaperProcess.getOutputStream().close();
+                logger.info("Waiting for reaper to exit normally");
+                if (reaperProcess.waitFor() != 0) {
+                    throw new GradleException("Reaper process failed. Check log at "
+                        + inputDir.resolve("error.log") + " for details");
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+    }
+
+    private synchronized void ensureReaperStarted() {
+        if (reaperProcess == null) {
+            try {
+                // copy the reaper jar
+                Path jarPath = buildDir.resolve("reaper").resolve("reaper.jar");
+                Files.createDirectories(jarPath.getParent());
+                InputStream jarInput = ReaperPlugin.class.getResourceAsStream("/META-INF/reaper.jar");
+                try (OutputStream out = Files.newOutputStream(jarPath)) {
+                    jarInput.transferTo(out);
+                }
+
+                // ensure the input directory exists
+                Files.createDirectories(inputDir);
+
+                // start the reaper
+                ProcessBuilder builder = new ProcessBuilder(
+                    Jvm.current().getJavaExecutable().toString(), // same jvm as gradle
+                    "-Xms4m", "-Xmx16m", // no need for a big heap, just need to read some files and execute
+                    "-jar", jarPath.toString(),
+                    inputDir.toString());
+                logger.info("Launching reaper: " + String.join(" ", builder.command()));
+                // be explicit for stdin, we use closing of the pipe to signal shutdown to the reaper
+                builder.redirectInput(ProcessBuilder.Redirect.PIPE);
+                builder.redirectOutput(logFile.toFile());
+                builder.redirectError(logFile.toFile());
+                reaperProcess = builder.start();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            ensureReaperAlive();
+        }
+    }
+
+    private void ensureReaperAlive() {
+        if (reaperProcess.isAlive() == false) {
+            throw new IllegalStateException("Reaper process died unexpectedly! Check the log at " + logFile.toString());
+        }
+    }
+}