From 5fc35261eb99d0b17c7a53f84abf0873303e2fca Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 18 Apr 2023 15:10:09 +0200 Subject: [PATCH 01/61] implement containerization support for native image bundles --- .../com/oracle/svm/driver/BundleSupport.java | 218 +++++++++++++++++- .../com/oracle/svm/driver/NativeImage.java | 27 ++- 2 files changed, 229 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 0097fc926885..af2f84bf57e0 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -24,8 +24,10 @@ */ package com.oracle.svm.driver; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.net.URI; @@ -61,6 +63,8 @@ import java.util.jar.Manifest; import java.util.stream.Stream; +import org.graalvm.collections.EconomicMap; +import org.graalvm.util.json.JSONParser; import org.graalvm.util.json.JSONParserException; import com.oracle.svm.core.OS; @@ -78,7 +82,7 @@ final class BundleSupport { final NativeImage nativeImage; final Path rootDir; - + final Path inputDir; final Path stageDir; final Path classPathDir; final Path modulePathDir; @@ -112,6 +116,13 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = ".nib"; + boolean containerizedBuild; + String containerizationTool; + static final String DEFAULT_CONTAINERIZATION_TOOL = "podman"; + static final List SUPPORTED_CONTAINERIZATION_TOOLS = List.of("podman", "docker"); + String containerImage = "graalvm-container"; + Path dockerfile; + enum BundleOptionVariants { create(), apply(); @@ -121,11 +132,25 @@ String optionName() { } } + enum ExtendedBundleOptions { + containerized, + tool, + container, + dockerfile + } + static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { try { String variant = bundleArg.substring(BUNDLE_OPTION.length() + 1); String bundleFilename = null; - String[] variantParts = SubstrateUtil.split(variant, "=", 2); + String extendedOptions = null; + String[] variantParts = SubstrateUtil.split(variant, ",", 2); + if (variantParts.length == 2) { + variant = variantParts[0]; + extendedOptions = variantParts[1]; + } + + variantParts = SubstrateUtil.split(variant, "=", 2); if (variantParts.length == 2) { variant = variantParts[0]; bundleFilename = variantParts[1]; @@ -174,6 +199,54 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma default: throw new IllegalArgumentException(); } + + if(extendedOptions != null) { + try { + String[] options = SubstrateUtil.split(extendedOptions, ","); + for (int i = 0; i < options.length; i++) { + String[] optionParts = SubstrateUtil.split(options[i], "="); + switch (ExtendedBundleOptions.valueOf(optionParts[0])) { + case containerized: + if (!OS.LINUX.isCurrent()) { + nativeImage.showMessage("Containerized builds only supported for Linux, skipping containerized build."); + bundleSupport.containerizedBuild = false; + } else { + bundleSupport.containerizedBuild = true; + } + break; + case tool: + if (optionParts.length == 2 && !optionParts[1].isEmpty()) { + if (!SUPPORTED_CONTAINERIZATION_TOOLS.contains(optionParts[1])) { + throw NativeImage.showError(String.format("%s is not supported, please use one of the following tools for containerized builds: %s", optionParts[1], SUPPORTED_CONTAINERIZATION_TOOLS)); + } + bundleSupport.containerizationTool = optionParts[1]; + } + break; + case container: + if (optionParts.length == 2 && !optionParts[1].isEmpty()) + bundleSupport.containerImage = optionParts[1]; + break; + case dockerfile: + if (optionParts.length == 2 && !optionParts[1].isEmpty() && Files.exists(Path.of(optionParts[1]))) + bundleSupport.dockerfile = Path.of(optionParts[1]); + break; + default: + throw new IllegalArgumentException(); + } + } + + } catch (StringIndexOutOfBoundsException | IllegalArgumentException e) { + String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) + .map(Enum::toString) + .collect(Collectors.joining(", ")); + throw NativeImage.showError("Unknown extended option in " + extendedOptions + ". Valid options are: " + suggestedOptions + "."); + } + } + + if(bundleSupport.containerizedBuild) { + bundleSupport.initializeContainerImage(); + } + return bundleSupport; } catch (StringIndexOutOfBoundsException | IllegalArgumentException e) { @@ -182,6 +255,96 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } + private void initializeContainerImage() { + // create dockerfile if not available for writing or loading bundle + try { + if (dockerfile == null) { + dockerfile = Files.createTempFile("Dockerfile", null); + Files.write(dockerfile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + + "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y").getBytes()); + dockerfile.toFile().deleteOnExit(); + } + } catch (IOException e) { + throw NativeImage.showError(e.getMessage()); + } + + if(writeBundle && nativeImage.isDryRun()) { + nativeImage.showMessage("Skipping container creation for native-image dry-run."); + if(containerizationTool == null) { + containerizationTool = DEFAULT_CONTAINERIZATION_TOOL; + } + return; + } + + if(containerizationTool != null) { + if(!hasContainerizationSupport(containerizationTool)) { + throw NativeImage.showError("Configured containerization tool not available."); + } else if(containerizationTool.equals("docker") && !isRootlessDocker()) { + throw NativeImage.showError("Only rootless docker is supported for containerized builds."); + } + } else { + if (hasContainerizationSupport("podman")) { + containerizationTool = "podman"; + } else if (hasContainerizationSupport("docker") && isRootlessDocker()) { + containerizationTool = "docker"; + } else { + throw NativeImage.showError(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINERIZATION_TOOLS)); + } + } + + if (createContainer() != 0) { + throw NativeImage.showError("Failed to create container image for building"); + } + } + + private int createContainer() { + Process p = null; + try { + p = new ProcessBuilder(containerizationTool, "build", "-f", dockerfile.toString(), "-t", containerImage, ".").inheritIO().start(); + return p.waitFor(); + } catch (IOException | InterruptedException e) { + throw NativeImage.showError(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } + } + } + + private boolean hasContainerizationSupport(String tool) { + ProcessBuilder pb = new ProcessBuilder("which", tool); + Process p = null; + try { + p = pb.start(); + p.waitFor(); + String result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); + return result != null; + } catch (IOException | InterruptedException e) { + throw NativeImage.showError(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } + } + } + + private boolean isRootlessDocker() { + ProcessBuilder pb = new ProcessBuilder("docker", "context", "show"); + Process p = null; + try { + p = pb.start(); + p.waitFor(); + String result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); + return result.equals("rootless"); + } catch (IOException | InterruptedException e) { + throw NativeImage.showError(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } + } + } + private BundleSupport(NativeImage nativeImage) { Objects.requireNonNull(nativeImage); this.nativeImage = nativeImage; @@ -193,7 +356,7 @@ private BundleSupport(NativeImage nativeImage) { bundleProperties = new BundleProperties(); bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); - Path inputDir = rootDir.resolve("input"); + inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); Path classesDir = inputDir.resolve("classes"); @@ -262,7 +425,7 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { nativeImage.config.modulePathBuild = !forceBuilderOnClasspath; try { - Path inputDir = rootDir.resolve("input"); + inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); Path classesDir = inputDir.resolve("classes"); @@ -295,6 +458,25 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } } + Path containerFile = stageDir.resolve("container.json"); + if(Files.exists(containerFile)) { + try (Reader reader = Files.newBufferedReader(containerFile)) { + EconomicMap json = JSONParser.parseDict(reader); + containerImage = json.get("containerImage").toString(); + containerizationTool = json.get("tool").toString(); + } catch (IOException e) { + throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); + } + nativeImage.showMessage(String.format("%s %s was specified as container image on bundle creation and will be used for bundle apply. Specify other images with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.container)); + nativeImage.showMessage(String.format("%s %s was specified as containerization tool on bundle creation and will be used for bundle apply. Specify other tools with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerizationTool, ExtendedBundleOptions.tool)); + } + + dockerfile = stageDir.resolve("Dockerfile"); + if(!Files.exists(dockerfile)) { + dockerfile = null; + } + + Path buildArgsFile = stageDir.resolve("build.json"); try (Reader reader = Files.newBufferedReader(buildArgsFile)) { List buildArgsFromFile = new ArrayList<>(); @@ -330,7 +512,7 @@ Path recordCanonicalization(Path before, Path after) { return before; } if (after.startsWith(nativeImage.config.getJavaHome())) { - return after; + return containerizedBuild ? Path.of("/graalvm").resolve(nativeImage.config.getJavaHome().relativize(after)) : after; } nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordCanonicalization src: " + before + ", dst: " + after); pathCanonicalizations.put(before, after); @@ -357,7 +539,7 @@ Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) Path substituteImagePath(Path origPath) { pathSubstitutions.put(origPath, rootDir.relativize(imagePathOutputDir)); - return imagePathOutputDir; + return containerizedBuild ? Path.of("/").resolve(rootDir.relativize(imagePathOutputDir)) : imagePathOutputDir; } Path substituteClassPath(Path origPath) { @@ -398,12 +580,12 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path previousRelativeSubstitutedPath = pathSubstitutions.get(origPath); if (previousRelativeSubstitutedPath != null) { nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); - return rootDir.resolve(previousRelativeSubstitutedPath); + return (containerizedBuild ? Path.of("/") : rootDir).resolve(previousRelativeSubstitutedPath); } if (origPath.startsWith(nativeImage.config.getJavaHome())) { /* If origPath comes from native-image itself, substituting is not needed. */ - return origPath; + return containerizedBuild ? Path.of("/graalvm").resolve(nativeImage.config.getJavaHome().relativize(origPath)) : origPath; } boolean forbiddenPath = false; @@ -468,7 +650,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); pathSubstitutions.put(origPath, relativeSubstitutedPath); - return substitutedPath; + return containerizedBuild ? Path.of("/").resolve(relativeSubstitutedPath) : substitutedPath; } Path originalPath(Path substitutedPath) { @@ -615,6 +797,22 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + environmentFile, e); } + if(containerizedBuild) { + Path containerFile = stageDir.resolve("container.json"); + try (JsonWriter writer = new JsonWriter(containerFile)) { + writer.print(Map.of("containerImage", containerImage, "tool", containerizationTool)); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + containerFile, e); + } + + Path dockerfileFile = stageDir.resolve("Dockerfile"); + try { + Files.copy(dockerfile, dockerfileFile); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + dockerfileFile, e); + } + } + Path buildArgsFile = stageDir.resolve("build.json"); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { List equalsNonBundleOptions = List.of(CmdLineOptionHandler.VERBOSE_OPTION, CmdLineOptionHandler.DRY_RUN_OPTION); @@ -857,7 +1055,7 @@ private void write() { boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); if (imageBuilt) { - properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(false)); + properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(containerizedBuild)); } properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, NativeImage.platform); properties.put(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, System.getProperty("java.vm.vendor")); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index f8781b866b0e..104c07b53ebd 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -909,6 +909,9 @@ private static void consolidateListArgs(Collection args, String argPrefi private void processClasspathNativeImageMetaInf(Path classpathEntry) { try { + if (classpathEntry.startsWith(Path.of("/input"))) { + classpathEntry = bundleSupport.rootDir.resolve(Path.of("/").relativize(classpathEntry)); + } NativeImageMetaInfWalker.walkMetaInfForCPEntry(classpathEntry, metaInfProcessor); } catch (NativeImageMetaInfWalker.MetaInfWalkException e) { throw showError(e.getMessage(), e.cause); @@ -1427,7 +1430,7 @@ protected static List createImageBuilderArgs(List imageArgs, Lis return result; } - protected static String createVMInvocationArgumentFile(List arguments) { + protected Path createVMInvocationArgumentFile(List arguments) { try { Path argsFile = Files.createTempFile("vminvocation", ".args"); StringJoiner joiner = new StringJoiner("\n"); @@ -1448,19 +1451,19 @@ protected static String createVMInvocationArgumentFile(List arguments) { String joinedOptions = joiner.toString(); Files.write(argsFile, joinedOptions.getBytes()); argsFile.toFile().deleteOnExit(); - return "@" + argsFile; + return argsFile; } catch (IOException e) { throw showError(e.getMessage()); } } - protected static String createImageBuilderArgumentFile(List imageBuilderArguments) { + protected Path createImageBuilderArgumentFile(List imageBuilderArguments) { try { Path argsFile = Files.createTempFile("native-image", ".args"); String joinedOptions = String.join("\0", imageBuilderArguments); Files.write(argsFile, joinedOptions.getBytes()); argsFile.toFile().deleteOnExit(); - return NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + argsFile.toString(); + return argsFile; } catch (IOException e) { throw showError(e.getMessage()); } @@ -1553,10 +1556,22 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa List finalImageBuilderArgs = createImageBuilderArgs(finalImageArgs, finalImageClassPath, finalImageModulePath); /* Construct ProcessBuilder command from final arguments */ + Path argFile = createVMInvocationArgumentFile(arguments); + Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); List command = new ArrayList<>(); + if(useBundle() && bundleSupport.containerizedBuild) { + List containerCommand = List.of(bundleSupport.containerizationTool, "run", "--network=none", "--rm", + "--mount", "type=bind,source=" + config.getJavaHome() + ",target=/graalvm,readonly", + "--mount", "type=bind,source=" + bundleSupport.inputDir + ",target=" + Path.of("/").resolve(bundleSupport.inputDir.getFileName()) + ",readonly", + "--mount", "type=bind,source=" + bundleSupport.outputDir + ",target=" + Path.of("/").resolve(bundleSupport.outputDir.getFileName()), + "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", + "--mount", "type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly", + bundleSupport.containerImage); + command.addAll(containerCommand); + } command.add(javaExecutable); - command.add(createVMInvocationArgumentFile(arguments)); - command.add(createImageBuilderArgumentFile(finalImageBuilderArgs)); + command.add("@" + argFile); + command.add(NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + builderArgFile); ProcessBuilder pb = new ProcessBuilder(); pb.command(command); Map environment = pb.environment(); From b72e13377fe6020365400595b3c73a84dd3ee633 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 18 Apr 2023 15:10:09 +0200 Subject: [PATCH 02/61] no output if container exists already, refactoring, check tool version --- .../com/oracle/svm/driver/BundleSupport.java | 110 ++++++++++++------ .../com/oracle/svm/driver/NativeImage.java | 2 +- 2 files changed, 74 insertions(+), 38 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index af2f84bf57e0..4ba2d2e2b785 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -117,11 +117,12 @@ final class BundleSupport { static final String BUNDLE_FILE_EXTENSION = ".nib"; boolean containerizedBuild; - String containerizationTool; - static final String DEFAULT_CONTAINERIZATION_TOOL = "podman"; - static final List SUPPORTED_CONTAINERIZATION_TOOLS = List.of("podman", "docker"); + String containerTool; + String containerToolVersion; + static final String DEFAULT_CONTAINER_TOOL = "podman"; + static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); String containerImage = "graalvm-container"; - Path dockerfile; + Path containerImageFile; enum BundleOptionVariants { create(), @@ -216,10 +217,10 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma break; case tool: if (optionParts.length == 2 && !optionParts[1].isEmpty()) { - if (!SUPPORTED_CONTAINERIZATION_TOOLS.contains(optionParts[1])) { - throw NativeImage.showError(String.format("%s is not supported, please use one of the following tools for containerized builds: %s", optionParts[1], SUPPORTED_CONTAINERIZATION_TOOLS)); + if (!SUPPORTED_CONTAINER_TOOLS.contains(optionParts[1])) { + throw NativeImage.showError(String.format("%s is not supported, please use one of the following tools for containerized builds: %s", optionParts[1], SUPPORTED_CONTAINER_TOOLS)); } - bundleSupport.containerizationTool = optionParts[1]; + bundleSupport.containerTool = optionParts[1]; } break; case container: @@ -228,7 +229,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma break; case dockerfile: if (optionParts.length == 2 && !optionParts[1].isEmpty() && Files.exists(Path.of(optionParts[1]))) - bundleSupport.dockerfile = Path.of(optionParts[1]); + bundleSupport.containerImageFile = Path.of(optionParts[1]); break; default: throw new IllegalArgumentException(); @@ -258,11 +259,11 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma private void initializeContainerImage() { // create dockerfile if not available for writing or loading bundle try { - if (dockerfile == null) { - dockerfile = Files.createTempFile("Dockerfile", null); - Files.write(dockerfile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + + if (containerImageFile == null) { + containerImageFile = Files.createTempFile("Dockerfile", null); + Files.write(containerImageFile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y").getBytes()); - dockerfile.toFile().deleteOnExit(); + containerImageFile.toFile().deleteOnExit(); } } catch (IOException e) { throw NativeImage.showError(e.getMessage()); @@ -270,25 +271,27 @@ private void initializeContainerImage() { if(writeBundle && nativeImage.isDryRun()) { nativeImage.showMessage("Skipping container creation for native-image dry-run."); - if(containerizationTool == null) { - containerizationTool = DEFAULT_CONTAINERIZATION_TOOL; + if(containerTool == null) { + containerTool = DEFAULT_CONTAINER_TOOL; } return; } - if(containerizationTool != null) { - if(!hasContainerizationSupport(containerizationTool)) { + if(containerTool != null) { + containerToolVersion = getContainerToolVersion(containerTool); + if(containerToolVersion.equals("")) { throw NativeImage.showError("Configured containerization tool not available."); - } else if(containerizationTool.equals("docker") && !isRootlessDocker()) { + } else if(containerTool.equals("docker") && !isRootlessDocker()) { throw NativeImage.showError("Only rootless docker is supported for containerized builds."); } } else { - if (hasContainerizationSupport("podman")) { - containerizationTool = "podman"; - } else if (hasContainerizationSupport("docker") && isRootlessDocker()) { - containerizationTool = "docker"; - } else { - throw NativeImage.showError(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINERIZATION_TOOLS)); + for(String tool : SUPPORTED_CONTAINER_TOOLS) { + containerTool = tool; + containerToolVersion = getContainerToolVersion(tool); + if(!containerToolVersion.equals("")) break; + } + if (containerToolVersion.equals("")) { + throw NativeImage.showError(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS)); } } @@ -298,10 +301,29 @@ private void initializeContainerImage() { } private int createContainer() { + ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); + ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", containerImageFile.toString(), "-t", containerImage, "."); Process p = null; try { - p = new ProcessBuilder(containerizationTool, "build", "-f", dockerfile.toString(), "-t", containerImage, ".").inheritIO().start(); - return p.waitFor(); + p = pbCheckForImage.start(); + p.waitFor(); + String imageId = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); + if(imageId == null) { + pb.inheritIO(); + } else { + nativeImage.showMessage(String.format("Checking container image %s", containerImage)); + } + p = pb.start(); + int status = p.waitFor(); + if(status == 0) { + Stream result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).lines(); + p = pbCheckForImage.start(); + p.waitFor(); + if(imageId != null && !(new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine().equals(imageId)) { + result.forEach(System.out::println); + } + } + return status; } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); } finally { @@ -311,14 +333,28 @@ private int createContainer() { } } - private boolean hasContainerizationSupport(String tool) { - ProcessBuilder pb = new ProcessBuilder("which", tool); + private boolean isToolAvailable(String tool) { + String[] paths = SubstrateUtil.split(System.getenv("PATH"), ":"); + for (String path : paths) { + try (Stream walk = Files.walk(Path.of(path))) { + if(walk.noneMatch(p -> p.endsWith(tool))) { + return true; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return false; + } + + private String getContainerToolVersion(String tool) { + ProcessBuilder pb = new ProcessBuilder(tool, "--version"); Process p = null; try { p = pb.start(); p.waitFor(); String result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); - return result != null; + return result.contains("version") ? result : ""; } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); } finally { @@ -463,17 +499,17 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { try (Reader reader = Files.newBufferedReader(containerFile)) { EconomicMap json = JSONParser.parseDict(reader); containerImage = json.get("containerImage").toString(); - containerizationTool = json.get("tool").toString(); + containerTool = json.get("tool").toString(); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } nativeImage.showMessage(String.format("%s %s was specified as container image on bundle creation and will be used for bundle apply. Specify other images with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.container)); - nativeImage.showMessage(String.format("%s %s was specified as containerization tool on bundle creation and will be used for bundle apply. Specify other tools with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerizationTool, ExtendedBundleOptions.tool)); + nativeImage.showMessage(String.format("%s %s was specified as containerization tool on bundle creation and will be used for bundle apply. Specify other tools with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerTool, ExtendedBundleOptions.tool)); } - dockerfile = stageDir.resolve("Dockerfile"); - if(!Files.exists(dockerfile)) { - dockerfile = null; + containerImageFile = stageDir.resolve("Dockerfile"); + if(!Files.exists(containerImageFile)) { + containerImageFile = null; } @@ -800,16 +836,16 @@ private Path writeBundle() { if(containerizedBuild) { Path containerFile = stageDir.resolve("container.json"); try (JsonWriter writer = new JsonWriter(containerFile)) { - writer.print(Map.of("containerImage", containerImage, "tool", containerizationTool)); + writer.print(Map.of("containerImage", containerImage, "containerTool", containerTool, "containerToolVersion", containerToolVersion)); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + containerFile, e); } - Path dockerfileFile = stageDir.resolve("Dockerfile"); + Path bundleContainerImageFile= stageDir.resolve("Dockerfile"); try { - Files.copy(dockerfile, dockerfileFile); + Files.copy(containerImageFile, bundleContainerImageFile); } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + dockerfileFile, e); + throw NativeImage.showError("Failed to write bundle-file " + bundleContainerImageFile, e); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 104c07b53ebd..e659071dc8b2 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1560,7 +1560,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); List command = new ArrayList<>(); if(useBundle() && bundleSupport.containerizedBuild) { - List containerCommand = List.of(bundleSupport.containerizationTool, "run", "--network=none", "--rm", + List containerCommand = List.of(bundleSupport.containerTool, "run", "--network=none", "--rm", "--mount", "type=bind,source=" + config.getJavaHome() + ",target=/graalvm,readonly", "--mount", "type=bind,source=" + bundleSupport.inputDir + ",target=" + Path.of("/").resolve(bundleSupport.inputDir.getFileName()) + ",readonly", "--mount", "type=bind,source=" + bundleSupport.outputDir + ",target=" + Path.of("/").resolve(bundleSupport.outputDir.getFileName()), From 89e1ba114a04c41f428ec93bb1e13e752756ca66 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 18 Apr 2023 15:10:09 +0200 Subject: [PATCH 03/61] add and change bundle options --- .../com/oracle/svm/driver/BundleSupport.java | 171 +++++++++++------- .../com/oracle/svm/driver/NativeImage.java | 8 +- 2 files changed, 110 insertions(+), 69 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 4ba2d2e2b785..c938fcb6354b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -63,6 +63,7 @@ import java.util.jar.Manifest; import java.util.stream.Stream; +import com.oracle.svm.core.util.ExitStatus; import org.graalvm.collections.EconomicMap; import org.graalvm.util.json.JSONParser; import org.graalvm.util.json.JSONParserException; @@ -116,13 +117,14 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = ".nib"; - boolean containerizedBuild; + final Path containerRootDir = Path.of("/"); + final Path containerGraalVMHome = Path.of("/graalvm"); + boolean useContainer; String containerTool; String containerToolVersion; - static final String DEFAULT_CONTAINER_TOOL = "podman"; - static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); + private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); String containerImage = "graalvm-container"; - Path containerImageFile; + Path dockerfile; enum BundleOptionVariants { create(), @@ -134,10 +136,19 @@ String optionName() { } enum ExtendedBundleOptions { - containerized, - tool, + dry_run, container, - dockerfile + dockerfile, + image; + + static ExtendedBundleOptions get(String name) { + return ExtendedBundleOptions.valueOf(name.replace('-', '_')); + } + + @Override + public String toString() { + return super.toString().replace('_', '-'); + } } static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { @@ -204,18 +215,17 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma if(extendedOptions != null) { try { String[] options = SubstrateUtil.split(extendedOptions, ","); - for (int i = 0; i < options.length; i++) { - String[] optionParts = SubstrateUtil.split(options[i], "="); - switch (ExtendedBundleOptions.valueOf(optionParts[0])) { - case containerized: - if (!OS.LINUX.isCurrent()) { - nativeImage.showMessage("Containerized builds only supported for Linux, skipping containerized build."); - bundleSupport.containerizedBuild = false; - } else { - bundleSupport.containerizedBuild = true; - } + for (String option : options) { + String[] optionParts = SubstrateUtil.split(option, "="); + switch (ExtendedBundleOptions.get(optionParts[0])) { + case dry_run: + nativeImage.setDryRun(true); break; - case tool: + case container: + if (bundleSupport.useContainer) { + throw NativeImage.showError("native-image bundle allows container option to be specified only once."); + } + bundleSupport.useContainer = true; if (optionParts.length == 2 && !optionParts[1].isEmpty()) { if (!SUPPORTED_CONTAINER_TOOLS.contains(optionParts[1])) { throw NativeImage.showError(String.format("%s is not supported, please use one of the following tools for containerized builds: %s", optionParts[1], SUPPORTED_CONTAINER_TOOLS)); @@ -223,14 +233,21 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma bundleSupport.containerTool = optionParts[1]; } break; - case container: + case dockerfile: + if (!bundleSupport.useContainer) { + throw NativeImage.showError("native-image bundle allows option dockerfile to be specified only after container option."); + } + if (optionParts.length == 2 && !optionParts[1].isEmpty() && Files.exists(Path.of(optionParts[1]))) { + bundleSupport.dockerfile = Path.of(optionParts[1]); + } + break; + case image: + if (!bundleSupport.useContainer) { + throw NativeImage.showError("native-image bundle allows image option to be specified only after container option."); + } if (optionParts.length == 2 && !optionParts[1].isEmpty()) bundleSupport.containerImage = optionParts[1]; break; - case dockerfile: - if (optionParts.length == 2 && !optionParts[1].isEmpty() && Files.exists(Path.of(optionParts[1]))) - bundleSupport.containerImageFile = Path.of(optionParts[1]); - break; default: throw new IllegalArgumentException(); } @@ -244,8 +261,13 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } - if(bundleSupport.containerizedBuild) { - bundleSupport.initializeContainerImage(); + if(bundleSupport.useContainer) { + if (!OS.LINUX.isCurrent()) { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Containerized builds only supported for Linux, skipping containerized build."); + bundleSupport.useContainer = false; + } else { + bundleSupport.initializeContainerImage(); + } } return bundleSupport; @@ -257,52 +279,68 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } private void initializeContainerImage() { - // create dockerfile if not available for writing or loading bundle + // create Dockerfile if not available for writing or loading bundle try { - if (containerImageFile == null) { - containerImageFile = Files.createTempFile("Dockerfile", null); - Files.write(containerImageFile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + + // TODO load dockerfile by graalvm version + if (dockerfile == null) { + dockerfile = Files.createTempFile("Dockerfile", null); + Files.write(dockerfile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y").getBytes()); - containerImageFile.toFile().deleteOnExit(); + dockerfile.toFile().deleteOnExit(); } } catch (IOException e) { throw NativeImage.showError(e.getMessage()); } - if(writeBundle && nativeImage.isDryRun()) { - nativeImage.showMessage("Skipping container creation for native-image dry-run."); - if(containerTool == null) { - containerTool = DEFAULT_CONTAINER_TOOL; - } + if(nativeImage.isDryRun()) { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); return; } if(containerTool != null) { - containerToolVersion = getContainerToolVersion(containerTool); - if(containerToolVersion.equals("")) { - throw NativeImage.showError("Configured containerization tool not available."); + if(!isToolAvailable(containerTool)) { + throw NativeImage.showError("Configured container tool not available."); } else if(containerTool.equals("docker") && !isRootlessDocker()) { throw NativeImage.showError("Only rootless docker is supported for containerized builds."); } + containerToolVersion = getContainerToolVersion(containerTool); } else { for(String tool : SUPPORTED_CONTAINER_TOOLS) { - containerTool = tool; - containerToolVersion = getContainerToolVersion(tool); - if(!containerToolVersion.equals("")) break; + if(isToolAvailable(tool)) { + if(tool.equals("docker") && !isRootlessDocker()) { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); + continue; + } + containerTool = tool; + containerToolVersion = getContainerToolVersion(tool); + break; + } } - if (containerToolVersion.equals("")) { + if (containerTool == null) { throw NativeImage.showError(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS)); } } - if (createContainer() != 0) { - throw NativeImage.showError("Failed to create container image for building"); + int exitStatusCode = createContainer(); + switch (ExitStatus.of(exitStatusCode)) { + case OK: + break; + case BUILDER_ERROR: + /* Exit, builder has handled error reporting. */ + throw NativeImage.showError(null, null, exitStatusCode); + case OUT_OF_MEMORY: + nativeImage.showOutOfMemoryWarning(); + throw NativeImage.showError(null, null, exitStatusCode); + default: + String message = String.format("Container build request for '%s' failed with exit status %d", + nativeImage.imageName, exitStatusCode); + throw NativeImage.showError(message, null, exitStatusCode); } } private int createContainer() { ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); - ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", containerImageFile.toString(), "-t", containerImage, "."); + ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); Process p = null; try { p = pbCheckForImage.start(); @@ -311,16 +349,19 @@ private int createContainer() { if(imageId == null) { pb.inheritIO(); } else { - nativeImage.showMessage(String.format("Checking container image %s", containerImage)); + nativeImage.showMessage(String.format("%sChecking container image %s for reuse.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); } p = pb.start(); int status = p.waitFor(); - if(status == 0) { + if(imageId != null && status == 0) { Stream result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).lines(); p = pbCheckForImage.start(); p.waitFor(); - if(imageId != null && !(new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine().equals(imageId)) { + if(!(new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine().equals(imageId)) { + nativeImage.showMessage(String.format("%sUpdate container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); result.forEach(System.out::println); + } else { + nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); } } return status; @@ -337,7 +378,7 @@ private boolean isToolAvailable(String tool) { String[] paths = SubstrateUtil.split(System.getenv("PATH"), ":"); for (String path : paths) { try (Stream walk = Files.walk(Path.of(path))) { - if(walk.noneMatch(p -> p.endsWith(tool))) { + if(walk.anyMatch(p -> p.endsWith(tool))) { return true; } } catch (IOException e) { @@ -503,13 +544,13 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } - nativeImage.showMessage(String.format("%s %s was specified as container image on bundle creation and will be used for bundle apply. Specify other images with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.container)); - nativeImage.showMessage(String.format("%s %s was specified as containerization tool on bundle creation and will be used for bundle apply. Specify other tools with the '%s' option.", BUNDLE_INFO_MESSAGE_PREFIX, containerTool, ExtendedBundleOptions.tool)); + nativeImage.showMessage(String.format("%sUsing %s as container image. Specify other image with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.image)); + nativeImage.showMessage(String.format("%sUsing %s as container tool. Specify other tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, containerTool, ExtendedBundleOptions.container)); } - containerImageFile = stageDir.resolve("Dockerfile"); - if(!Files.exists(containerImageFile)) { - containerImageFile = null; + dockerfile = stageDir.resolve("Dockerfile"); + if(!Files.exists(dockerfile)) { + dockerfile = null; } @@ -548,7 +589,7 @@ Path recordCanonicalization(Path before, Path after) { return before; } if (after.startsWith(nativeImage.config.getJavaHome())) { - return containerizedBuild ? Path.of("/graalvm").resolve(nativeImage.config.getJavaHome().relativize(after)) : after; + return useContainer ? containerGraalVMHome.resolve(nativeImage.config.getJavaHome().relativize(after)) : after; } nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordCanonicalization src: " + before + ", dst: " + after); pathCanonicalizations.put(before, after); @@ -575,7 +616,7 @@ Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) Path substituteImagePath(Path origPath) { pathSubstitutions.put(origPath, rootDir.relativize(imagePathOutputDir)); - return containerizedBuild ? Path.of("/").resolve(rootDir.relativize(imagePathOutputDir)) : imagePathOutputDir; + return useContainer ? containerRootDir.resolve(rootDir.relativize(imagePathOutputDir)) : imagePathOutputDir; } Path substituteClassPath(Path origPath) { @@ -616,12 +657,12 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path previousRelativeSubstitutedPath = pathSubstitutions.get(origPath); if (previousRelativeSubstitutedPath != null) { nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); - return (containerizedBuild ? Path.of("/") : rootDir).resolve(previousRelativeSubstitutedPath); + return (useContainer ? containerRootDir : rootDir).resolve(previousRelativeSubstitutedPath); } if (origPath.startsWith(nativeImage.config.getJavaHome())) { /* If origPath comes from native-image itself, substituting is not needed. */ - return containerizedBuild ? Path.of("/graalvm").resolve(nativeImage.config.getJavaHome().relativize(origPath)) : origPath; + return useContainer ? containerGraalVMHome.resolve(nativeImage.config.getJavaHome().relativize(origPath)) : origPath; } boolean forbiddenPath = false; @@ -686,7 +727,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); pathSubstitutions.put(origPath, relativeSubstitutedPath); - return containerizedBuild ? Path.of("/").resolve(relativeSubstitutedPath) : substitutedPath; + return useContainer ? containerRootDir.resolve(relativeSubstitutedPath) : substitutedPath; } Path originalPath(Path substitutedPath) { @@ -833,19 +874,19 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + environmentFile, e); } - if(containerizedBuild) { + if(useContainer) { Path containerFile = stageDir.resolve("container.json"); try (JsonWriter writer = new JsonWriter(containerFile)) { - writer.print(Map.of("containerImage", containerImage, "containerTool", containerTool, "containerToolVersion", containerToolVersion)); + writer.print(Map.of("containerImage", containerImage, "containerTool", containerTool == null ? "" : containerTool, "containerToolVersion", containerToolVersion == null ? "" : containerToolVersion)); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + containerFile, e); } - Path bundleContainerImageFile= stageDir.resolve("Dockerfile"); + Path bundleDockerfile = stageDir.resolve("Dockerfile"); try { - Files.copy(containerImageFile, bundleContainerImageFile); + Files.copy(dockerfile, bundleDockerfile); } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + bundleContainerImageFile, e); + throw NativeImage.showError("Failed to write bundle-file " + bundleDockerfile, e); } } @@ -1091,7 +1132,7 @@ private void write() { boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); if (imageBuilt) { - properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(containerizedBuild)); + properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(useContainer)); } properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, NativeImage.platform); properties.put(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, System.getProperty("java.vm.vendor")); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index e659071dc8b2..5e93f1f32800 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1559,11 +1559,11 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa Path argFile = createVMInvocationArgumentFile(arguments); Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); List command = new ArrayList<>(); - if(useBundle() && bundleSupport.containerizedBuild) { + if(useBundle() && bundleSupport.useContainer) { List containerCommand = List.of(bundleSupport.containerTool, "run", "--network=none", "--rm", - "--mount", "type=bind,source=" + config.getJavaHome() + ",target=/graalvm,readonly", - "--mount", "type=bind,source=" + bundleSupport.inputDir + ",target=" + Path.of("/").resolve(bundleSupport.inputDir.getFileName()) + ",readonly", - "--mount", "type=bind,source=" + bundleSupport.outputDir + ",target=" + Path.of("/").resolve(bundleSupport.outputDir.getFileName()), + "--mount", "type=bind,source=" + config.getJavaHome() + ",target=" + bundleSupport.containerGraalVMHome + ",readonly", + "--mount", "type=bind,source=" + bundleSupport.inputDir + ",target=" + bundleSupport.containerRootDir.resolve(bundleSupport.inputDir.getFileName()) + ",readonly", + "--mount", "type=bind,source=" + bundleSupport.outputDir + ",target=" + bundleSupport.containerRootDir.resolve(bundleSupport.outputDir.getFileName()), "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", "--mount", "type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly", bundleSupport.containerImage); From d03923ce8ed923ce53f4f671c37b2f7ea34c3e34 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 20 Apr 2023 09:50:20 +0200 Subject: [PATCH 04/61] improve containerized bundle creation and writing, combine replacing bundle paths for containers in one place --- .../com/oracle/svm/driver/BundleSupport.java | 116 +++++++++++++----- .../com/oracle/svm/driver/NativeImage.java | 28 +++-- 2 files changed, 100 insertions(+), 44 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index c938fcb6354b..e284a5e04da2 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -117,14 +117,20 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = ".nib"; - final Path containerRootDir = Path.of("/"); final Path containerGraalVMHome = Path.of("/graalvm"); boolean useContainer; - String containerTool; - String containerToolVersion; + private String containerTool; + private String bundleContainerTool; + private String containerToolVersion; + private String bundleContainerToolVersion; private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); - String containerImage = "graalvm-container"; - Path dockerfile; + private String containerImage; + private static final String DEFAULT_CONTAINER_IMAGE = "graalvm-container"; + private Path dockerfile; + private final String containerToolJsonKey = "containerTool"; + private final String containerToolVersionJsonKey = "containerToolVersion"; + private final String containerImageJsonKey = "containerImage"; + enum BundleOptionVariants { create(), @@ -263,10 +269,14 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma if(bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Containerized builds only supported for Linux, skipping containerized build."); + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); bundleSupport.useContainer = false; } else { - bundleSupport.initializeContainerImage(); + if(nativeImage.isDryRun()) { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); + } else { + bundleSupport.initializeContainerImage(); + } } } @@ -281,7 +291,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma private void initializeContainerImage() { // create Dockerfile if not available for writing or loading bundle try { - // TODO load dockerfile by graalvm version + // TODO load graalvm docker base if (dockerfile == null) { dockerfile = Files.createTempFile("Dockerfile", null); Files.write(dockerfile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + @@ -292,9 +302,8 @@ private void initializeContainerImage() { throw NativeImage.showError(e.getMessage()); } - if(nativeImage.isDryRun()) { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); - return; + if(bundleContainerTool != null && containerTool == null) { + containerTool = bundleContainerTool; } if(containerTool != null) { @@ -304,6 +313,15 @@ private void initializeContainerImage() { throw NativeImage.showError("Only rootless docker is supported for containerized builds."); } containerToolVersion = getContainerToolVersion(containerTool); + + if(bundleContainerTool != null) { + String bundleFileName = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); + if (!containerTool.equals(bundleContainerTool)) { + NativeImage.showWarning(String.format("The given bundle file %s was created with container tool '%s' (using '%s').", bundleFileName, bundleContainerTool, containerTool)); + } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { + NativeImage.showWarning(String.format("The given bundle file %s was created with %s version '%s' (installed '%s').", bundleFileName, containerTool, bundleContainerToolVersion, containerToolVersion)); + } + } } else { for(String tool : SUPPORTED_CONTAINER_TOOLS) { if(isToolAvailable(tool)) { @@ -321,6 +339,10 @@ private void initializeContainerImage() { } } + if(containerImage == null) { + containerImage = DEFAULT_CONTAINER_IMAGE; + } + int exitStatusCode = createContainer(); switch (ExitStatus.of(exitStatusCode)) { case OK: @@ -395,7 +417,7 @@ private String getContainerToolVersion(String tool) { p = pb.start(); p.waitFor(); String result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); - return result.contains("version") ? result : ""; + return result.contains("version") ? result : null; } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); } finally { @@ -422,6 +444,17 @@ private boolean isRootlessDocker() { } } + List createContainerCommand(Path argFile, Path builderArgFile) { + Path containerRoot = Path.of("/"); + return List.of(containerTool, "run", "--network=none", "--rm", + "--mount", "type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + containerGraalVMHome + ",readonly", + "--mount", "type=bind,source=" + inputDir + ",target=" + containerRoot.resolve(rootDir.relativize(inputDir)) + ",readonly", + "--mount", "type=bind,source=" + outputDir + ",target=" + containerRoot.resolve(rootDir.relativize(outputDir)), + "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", + "--mount", "type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly", + containerImage); + } + private BundleSupport(NativeImage nativeImage) { Objects.requireNonNull(nativeImage); this.nativeImage = nativeImage; @@ -539,13 +572,17 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { if(Files.exists(containerFile)) { try (Reader reader = Files.newBufferedReader(containerFile)) { EconomicMap json = JSONParser.parseDict(reader); - containerImage = json.get("containerImage").toString(); - containerTool = json.get("tool").toString(); + if(json.get(containerImageJsonKey) != null) containerImage = json.get(containerImageJsonKey).toString(); + if(json.get(containerToolJsonKey) != null) bundleContainerTool = json.get(containerToolJsonKey).toString(); + if(json.get(containerToolVersionJsonKey) != null) bundleContainerToolVersion = json.get(containerToolVersionJsonKey).toString(); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } - nativeImage.showMessage(String.format("%sUsing %s as container image. Specify other image with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.image)); - nativeImage.showMessage(String.format("%sUsing %s as container tool. Specify other tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, containerTool, ExtendedBundleOptions.container)); + if(containerImage != null) nativeImage.showMessage(String.format("%sUsing container image %s. Specify other image with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.image)); + if(bundleContainerTool != null) { + String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" version: %s", bundleContainerToolVersion); + nativeImage.showMessage(String.format("%sUsing %s%s. Specify other container tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString, ExtendedBundleOptions.container)); + } } dockerfile = stageDir.resolve("Dockerfile"); @@ -589,7 +626,7 @@ Path recordCanonicalization(Path before, Path after) { return before; } if (after.startsWith(nativeImage.config.getJavaHome())) { - return useContainer ? containerGraalVMHome.resolve(nativeImage.config.getJavaHome().relativize(after)) : after; + return after; } nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordCanonicalization src: " + before + ", dst: " + after); pathCanonicalizations.put(before, after); @@ -602,6 +639,14 @@ Path restoreCanonicalization(Path before) { return after; } + void replacePathsForContainerBuild(List arguments) { + arguments.replaceAll(arg -> arg + .replace(nativeImage.config.getJavaHome().toString(), containerGraalVMHome.toString()) + .replace(rootDir.toString(), "") + ); + } + + Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) { Path destinationDir = switch (bundleMemberRole) { case Input -> auxiliaryDir; @@ -616,7 +661,7 @@ Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) Path substituteImagePath(Path origPath) { pathSubstitutions.put(origPath, rootDir.relativize(imagePathOutputDir)); - return useContainer ? containerRootDir.resolve(rootDir.relativize(imagePathOutputDir)) : imagePathOutputDir; + return imagePathOutputDir; } Path substituteClassPath(Path origPath) { @@ -657,12 +702,12 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path previousRelativeSubstitutedPath = pathSubstitutions.get(origPath); if (previousRelativeSubstitutedPath != null) { nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); - return (useContainer ? containerRootDir : rootDir).resolve(previousRelativeSubstitutedPath); + return rootDir.resolve(previousRelativeSubstitutedPath); } if (origPath.startsWith(nativeImage.config.getJavaHome())) { /* If origPath comes from native-image itself, substituting is not needed. */ - return useContainer ? containerGraalVMHome.resolve(nativeImage.config.getJavaHome().relativize(origPath)) : origPath; + return origPath; } boolean forbiddenPath = false; @@ -727,7 +772,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); pathSubstitutions.put(origPath, relativeSubstitutedPath); - return useContainer ? containerRootDir.resolve(relativeSubstitutedPath) : substitutedPath; + return substitutedPath; } Path originalPath(Path substitutedPath) { @@ -875,18 +920,27 @@ private Path writeBundle() { } if(useContainer) { - Path containerFile = stageDir.resolve("container.json"); - try (JsonWriter writer = new JsonWriter(containerFile)) { - writer.print(Map.of("containerImage", containerImage, "containerTool", containerTool == null ? "" : containerTool, "containerToolVersion", containerToolVersion == null ? "" : containerToolVersion)); - } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + containerFile, e); + Map containerInfo = new HashMap<>(); + if(containerImage != null) containerInfo.put(containerImageJsonKey, containerImage); + if(containerTool != null) containerInfo.put(containerToolJsonKey, containerTool); + if(containerToolVersion != null) containerInfo.put(containerToolVersionJsonKey, containerToolVersion); + + if(!containerInfo.isEmpty()) { + Path containerFile = stageDir.resolve("container.json"); + try (JsonWriter writer = new JsonWriter(containerFile)) { + writer.print(containerInfo); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + containerFile, e); + } } - Path bundleDockerfile = stageDir.resolve("Dockerfile"); - try { - Files.copy(dockerfile, bundleDockerfile); - } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + bundleDockerfile, e); + if(dockerfile != null) { + Path bundleDockerfile = stageDir.resolve("Dockerfile"); + try { + Files.copy(dockerfile, bundleDockerfile); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + bundleDockerfile, e); + } } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 5e93f1f32800..f601c022c83d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -909,9 +909,6 @@ private static void consolidateListArgs(Collection args, String argPrefi private void processClasspathNativeImageMetaInf(Path classpathEntry) { try { - if (classpathEntry.startsWith(Path.of("/input"))) { - classpathEntry = bundleSupport.rootDir.resolve(Path.of("/").relativize(classpathEntry)); - } NativeImageMetaInfWalker.walkMetaInfForCPEntry(classpathEntry, metaInfProcessor); } catch (NativeImageMetaInfWalker.MetaInfWalkException e) { throw showError(e.getMessage(), e.cause); @@ -1492,8 +1489,10 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa BiFunction substituteAuxiliaryPath = useBundle() ? bundleSupport::substituteAuxiliaryPath : (a, b) -> a; Function imageArgsTransformer = rawArg -> apiOptionHandler.transformBuilderArgument(rawArg, substituteAuxiliaryPath); List finalImageArgs = imageArgs.stream().map(imageArgsTransformer).collect(Collectors.toList()); + Function substituteClassPath = useBundle() ? bundleSupport::substituteClassPath : Function.identity(); List finalImageClassPath = imagecp.stream().map(substituteClassPath).collect(Collectors.toList()); + Function substituteModulePath = useBundle() ? bundleSupport::substituteModulePath : Function.identity(); List substitutedImageModulePath = imagemp.stream().map(substituteModulePath).toList(); @@ -1556,19 +1555,22 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa List finalImageBuilderArgs = createImageBuilderArgs(finalImageArgs, finalImageClassPath, finalImageModulePath); /* Construct ProcessBuilder command from final arguments */ - Path argFile = createVMInvocationArgumentFile(arguments); - Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); List command = new ArrayList<>(); + if(useBundle() && bundleSupport.useContainer) { - List containerCommand = List.of(bundleSupport.containerTool, "run", "--network=none", "--rm", - "--mount", "type=bind,source=" + config.getJavaHome() + ",target=" + bundleSupport.containerGraalVMHome + ",readonly", - "--mount", "type=bind,source=" + bundleSupport.inputDir + ",target=" + bundleSupport.containerRootDir.resolve(bundleSupport.inputDir.getFileName()) + ",readonly", - "--mount", "type=bind,source=" + bundleSupport.outputDir + ",target=" + bundleSupport.containerRootDir.resolve(bundleSupport.outputDir.getFileName()), - "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", - "--mount", "type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly", - bundleSupport.containerImage); - command.addAll(containerCommand); + bundleSupport.replacePathsForContainerBuild(arguments); + bundleSupport.replacePathsForContainerBuild(finalImageBuilderArgs); + Path binJava = Paths.get("bin", "java"); + javaExecutable = bundleSupport.containerGraalVMHome.resolve(binJava).toString(); } + + Path argFile = createVMInvocationArgumentFile(arguments); + Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); + + if(!isDryRun() && useBundle() && bundleSupport.useContainer) { + command.addAll(bundleSupport.createContainerCommand(argFile, builderArgFile)); + } + command.add(javaExecutable); command.add("@" + argFile); command.add(NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + builderArgFile); From c922a31c1bf49d06338e79215a341fd83035c02c Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 25 Apr 2023 17:24:27 +0200 Subject: [PATCH 05/61] remove image option for bundles and compute image name from hash --- .../com/oracle/svm/driver/BundleSupport.java | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index e284a5e04da2..4e4f4d3a18e9 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -36,6 +36,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -65,6 +67,7 @@ import com.oracle.svm.core.util.ExitStatus; import org.graalvm.collections.EconomicMap; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.util.json.JSONParser; import org.graalvm.util.json.JSONParserException; @@ -125,6 +128,7 @@ final class BundleSupport { private String bundleContainerToolVersion; private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private String containerImage; + private String bundleContainerImage; private static final String DEFAULT_CONTAINER_IMAGE = "graalvm-container"; private Path dockerfile; private final String containerToolJsonKey = "containerTool"; @@ -144,8 +148,7 @@ String optionName() { enum ExtendedBundleOptions { dry_run, container, - dockerfile, - image; + dockerfile; static ExtendedBundleOptions get(String name) { return ExtendedBundleOptions.valueOf(name.replace('-', '_')); @@ -247,13 +250,6 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma bundleSupport.dockerfile = Path.of(optionParts[1]); } break; - case image: - if (!bundleSupport.useContainer) { - throw NativeImage.showError("native-image bundle allows image option to be specified only after container option."); - } - if (optionParts.length == 2 && !optionParts[1].isEmpty()) - bundleSupport.containerImage = optionParts[1]; - break; default: throw new IllegalArgumentException(); } @@ -289,6 +285,8 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } private void initializeContainerImage() { + String bundleFileName = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); + // create Dockerfile if not available for writing or loading bundle try { // TODO load graalvm docker base @@ -315,7 +313,6 @@ private void initializeContainerImage() { containerToolVersion = getContainerToolVersion(containerTool); if(bundleContainerTool != null) { - String bundleFileName = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); if (!containerTool.equals(bundleContainerTool)) { NativeImage.showWarning(String.format("The given bundle file %s was created with container tool '%s' (using '%s').", bundleFileName, bundleContainerTool, containerTool)); } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { @@ -339,8 +336,10 @@ private void initializeContainerImage() { } } - if(containerImage == null) { - containerImage = DEFAULT_CONTAINER_IMAGE; + String imageName = getSignature(dockerfile); + containerImage = imageName != null ? imageName : DEFAULT_CONTAINER_IMAGE; + if(bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { + NativeImage.showWarning(String.format("The given bundle file %s was created with a different dockerfile.", bundleFileName)); } int exitStatusCode = createContainer(); @@ -360,6 +359,22 @@ private void initializeContainerImage() { } } + private String getSignature(Path f) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(f)); + StringBuilder sb = new StringBuilder(); + for (byte b : md.digest()) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (IOException e) { + return null; + } catch (NoSuchAlgorithmException e) { + throw GraalError.shouldNotReachHere(e); // ExcludeFromJacocoGeneratedReport + } + } + private int createContainer() { ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); @@ -371,7 +386,7 @@ private int createContainer() { if(imageId == null) { pb.inheritIO(); } else { - nativeImage.showMessage(String.format("%sChecking container image %s for reuse.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); + nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); } p = pb.start(); int status = p.waitFor(); @@ -380,10 +395,8 @@ private int createContainer() { p = pbCheckForImage.start(); p.waitFor(); if(!(new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine().equals(imageId)) { - nativeImage.showMessage(String.format("%sUpdate container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); + nativeImage.showMessage(String.format("%sUpdated container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); result.forEach(System.out::println); - } else { - nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); } } return status; @@ -572,13 +585,12 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { if(Files.exists(containerFile)) { try (Reader reader = Files.newBufferedReader(containerFile)) { EconomicMap json = JSONParser.parseDict(reader); - if(json.get(containerImageJsonKey) != null) containerImage = json.get(containerImageJsonKey).toString(); + if(json.get(containerImageJsonKey) != null) bundleContainerImage = json.get(containerImageJsonKey).toString(); if(json.get(containerToolJsonKey) != null) bundleContainerTool = json.get(containerToolJsonKey).toString(); if(json.get(containerToolVersionJsonKey) != null) bundleContainerToolVersion = json.get(containerToolVersionJsonKey).toString(); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } - if(containerImage != null) nativeImage.showMessage(String.format("%sUsing container image %s. Specify other image with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage, ExtendedBundleOptions.image)); if(bundleContainerTool != null) { String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" version: %s", bundleContainerToolVersion); nativeImage.showMessage(String.format("%sUsing %s%s. Specify other container tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString, ExtendedBundleOptions.container)); From 9c0f571e567844e5bfd3e8f23c504732825c899e Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 27 Apr 2023 09:44:17 +0200 Subject: [PATCH 06/61] Use SubstrateUtil.digest for generating Dockerfile hash --- .../com/oracle/svm/driver/BundleSupport.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 4e4f4d3a18e9..00c9755eed11 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -129,7 +129,8 @@ final class BundleSupport { private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private String containerImage; private String bundleContainerImage; - private static final String DEFAULT_CONTAINER_IMAGE = "graalvm-container"; + private static final String DEFAULT_DOCKERFILE = "FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + + "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y"; private Path dockerfile; private final String containerToolJsonKey = "containerTool"; private final String containerToolVersionJsonKey = "containerToolVersion"; @@ -292,14 +293,20 @@ private void initializeContainerImage() { // TODO load graalvm docker base if (dockerfile == null) { dockerfile = Files.createTempFile("Dockerfile", null); - Files.write(dockerfile, ("FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + - "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y").getBytes()); + Files.write(dockerfile, DEFAULT_DOCKERFILE.getBytes()); dockerfile.toFile().deleteOnExit(); + containerImage = SubstrateUtil.digest(DEFAULT_DOCKERFILE); + } else { + containerImage = SubstrateUtil.digest(Files.readString(dockerfile)); } } catch (IOException e) { throw NativeImage.showError(e.getMessage()); } + if(bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { + NativeImage.showWarning(String.format("The given bundle file %s was created with a different dockerfile.", bundleFileName)); + } + if(bundleContainerTool != null && containerTool == null) { containerTool = bundleContainerTool; } @@ -336,12 +343,6 @@ private void initializeContainerImage() { } } - String imageName = getSignature(dockerfile); - containerImage = imageName != null ? imageName : DEFAULT_CONTAINER_IMAGE; - if(bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { - NativeImage.showWarning(String.format("The given bundle file %s was created with a different dockerfile.", bundleFileName)); - } - int exitStatusCode = createContainer(); switch (ExitStatus.of(exitStatusCode)) { case OK: @@ -359,22 +360,6 @@ private void initializeContainerImage() { } } - private String getSignature(Path f) { - try { - MessageDigest md = MessageDigest.getInstance("MD5"); - md.update(Files.readAllBytes(f)); - StringBuilder sb = new StringBuilder(); - for (byte b : md.digest()) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); - } catch (IOException e) { - return null; - } catch (NoSuchAlgorithmException e) { - throw GraalError.shouldNotReachHere(e); // ExcludeFromJacocoGeneratedReport - } - } - private int createContainer() { ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); From bcbd2cb26df970031a99c1ce472456688aae86eb Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 27 Apr 2023 10:12:54 +0200 Subject: [PATCH 07/61] Improve tool availability check --- .../com/oracle/svm/driver/BundleSupport.java | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 00c9755eed11..062bde38a0ac 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -36,8 +36,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -67,7 +65,6 @@ import com.oracle.svm.core.util.ExitStatus; import org.graalvm.collections.EconomicMap; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.util.json.JSONParser; import org.graalvm.util.json.JSONParserException; @@ -395,17 +392,9 @@ private int createContainer() { } private boolean isToolAvailable(String tool) { - String[] paths = SubstrateUtil.split(System.getenv("PATH"), ":"); - for (String path : paths) { - try (Stream walk = Files.walk(Path.of(path))) { - if(walk.anyMatch(p -> p.endsWith(tool))) { - return true; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return false; + return Arrays.stream(SubstrateUtil.split(System.getenv("PATH"), ":")) + .map(str -> Path.of(str).resolve(tool)) + .anyMatch(Files::isExecutable); } private String getContainerToolVersion(String tool) { From 9384123cfe1fcca80d1262e9600c29f2efb63a96 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 27 Apr 2023 11:52:09 +0200 Subject: [PATCH 08/61] Rework option parsing for extended bundle options --- .../com/oracle/svm/driver/BundleSupport.java | 93 ++++++++++--------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 062bde38a0ac..de9a612b038b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -123,12 +123,13 @@ final class BundleSupport { private String bundleContainerTool; private String containerToolVersion; private String bundleContainerToolVersion; - private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private String containerImage; private String bundleContainerImage; + private Path dockerfile; + private Path bundleDockerfile; + private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private static final String DEFAULT_DOCKERFILE = "FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y"; - private Path dockerfile; private final String containerToolJsonKey = "containerTool"; private final String containerToolVersionJsonKey = "containerToolVersion"; private final String containerImageJsonKey = "containerImage"; @@ -162,14 +163,10 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma try { String variant = bundleArg.substring(BUNDLE_OPTION.length() + 1); String bundleFilename = null; - String extendedOptions = null; - String[] variantParts = SubstrateUtil.split(variant, ",", 2); - if (variantParts.length == 2) { - variant = variantParts[0]; - extendedOptions = variantParts[1]; - } + String[] options = SubstrateUtil.split(variant, ","); - variantParts = SubstrateUtil.split(variant, "=", 2); + variant = options[0]; + String[] variantParts = SubstrateUtil.split(variant, "=", 2); if (variantParts.length == 2) { variant = variantParts[0]; bundleFilename = variantParts[1]; @@ -219,47 +216,51 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma throw new IllegalArgumentException(); } - if(extendedOptions != null) { - try { - String[] options = SubstrateUtil.split(extendedOptions, ","); - for (String option : options) { - String[] optionParts = SubstrateUtil.split(option, "="); - switch (ExtendedBundleOptions.get(optionParts[0])) { - case dry_run: - nativeImage.setDryRun(true); - break; - case container: + Arrays.stream(options) + .skip(1) + .forEach(option -> { + String optionValue = null; + String[] optionParts = SubstrateUtil.split(option, "=", 2); + if (optionParts.length == 2) { + option = optionParts[0]; + optionValue = optionParts[1]; + } + switch (ExtendedBundleOptions.get(option)) { + case dry_run -> nativeImage.setDryRun(true); + case container -> { if (bundleSupport.useContainer) { - throw NativeImage.showError("native-image bundle allows container option to be specified only once."); + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); } bundleSupport.useContainer = true; - if (optionParts.length == 2 && !optionParts[1].isEmpty()) { - if (!SUPPORTED_CONTAINER_TOOLS.contains(optionParts[1])) { - throw NativeImage.showError(String.format("%s is not supported, please use one of the following tools for containerized builds: %s", optionParts[1], SUPPORTED_CONTAINER_TOOLS)); + if (optionValue != null) { + if (!SUPPORTED_CONTAINER_TOOLS.contains(optionValue)) { + throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, SUPPORTED_CONTAINER_TOOLS)); } - bundleSupport.containerTool = optionParts[1]; + bundleSupport.containerTool = optionValue; } - break; - case dockerfile: + } + case dockerfile -> { if (!bundleSupport.useContainer) { - throw NativeImage.showError("native-image bundle allows option dockerfile to be specified only after container option."); + throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", option, ExtendedBundleOptions.container)); } - if (optionParts.length == 2 && !optionParts[1].isEmpty() && Files.exists(Path.of(optionParts[1]))) { - bundleSupport.dockerfile = Path.of(optionParts[1]); + if (bundleSupport.dockerfile != null) { + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); } - break; - default: - throw new IllegalArgumentException(); + if (optionValue != null) { + bundleSupport.dockerfile = Path.of(optionValue); + if (!Files.isReadable(bundleSupport.dockerfile)) { + throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", bundleSupport.dockerfile.toAbsolutePath())); + } + } + } + default -> { + String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) + .map(Enum::toString) + .collect(Collectors.joining(", ")); + throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", option, suggestedOptions)); + } } - } - - } catch (StringIndexOutOfBoundsException | IllegalArgumentException e) { - String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) - .map(Enum::toString) - .collect(Collectors.joining(", ")); - throw NativeImage.showError("Unknown extended option in " + extendedOptions + ". Valid options are: " + suggestedOptions + "."); - } - } + }); if(bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { @@ -285,6 +286,10 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma private void initializeContainerImage() { String bundleFileName = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); + if(bundleDockerfile != null && dockerfile == null) { + dockerfile = bundleDockerfile; + } + // create Dockerfile if not available for writing or loading bundle try { // TODO load graalvm docker base @@ -571,9 +576,9 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } } - dockerfile = stageDir.resolve("Dockerfile"); - if(!Files.exists(dockerfile)) { - dockerfile = null; + bundleDockerfile = stageDir.resolve("Dockerfile"); + if(!Files.isReadable(bundleDockerfile)) { + bundleDockerfile = null; } From 5fa8b50bcd92fdfe34d3cefb7bdebcd03f7ee64e Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 27 Apr 2023 12:42:00 +0200 Subject: [PATCH 09/61] Extract common functionality for processes --- .../com/oracle/svm/driver/BundleSupport.java | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index de9a612b038b..930dfdea8119 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -365,23 +365,21 @@ private void initializeContainerImage() { private int createContainer() { ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); + + String imageId = getFirstProcessResultLine(pbCheckForImage); + if(imageId == null) { + pb.inheritIO(); + } else { + nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); + } + Process p = null; try { - p = pbCheckForImage.start(); - p.waitFor(); - String imageId = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); - if(imageId == null) { - pb.inheritIO(); - } else { - nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); - } p = pb.start(); int status = p.waitFor(); - if(imageId != null && status == 0) { + if(status == 0 && imageId != null) { Stream result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).lines(); - p = pbCheckForImage.start(); - p.waitFor(); - if(!(new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine().equals(imageId)) { + if(!imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { nativeImage.showMessage(String.format("%sUpdated container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); result.forEach(System.out::println); } @@ -404,29 +402,20 @@ private boolean isToolAvailable(String tool) { private String getContainerToolVersion(String tool) { ProcessBuilder pb = new ProcessBuilder(tool, "--version"); - Process p = null; - try { - p = pb.start(); - p.waitFor(); - String result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); - return result.contains("version") ? result : null; - } catch (IOException | InterruptedException e) { - throw NativeImage.showError(e.getMessage()); - } finally { - if (p != null) { - p.destroy(); - } - } + return getFirstProcessResultLine(pb); } private boolean isRootlessDocker() { ProcessBuilder pb = new ProcessBuilder("docker", "context", "show"); + return getFirstProcessResultLine(pb).equals("rootless"); + } + + private String getFirstProcessResultLine(ProcessBuilder pb) { Process p = null; try { p = pb.start(); p.waitFor(); - String result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); - return result.equals("rootless"); + return (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); } finally { From 3eaf9d4e2fabf333712e4302a893f9b2cd92a05d Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 27 Apr 2023 12:42:56 +0200 Subject: [PATCH 10/61] Adjust warning and info messages for container tools --- .../src/com/oracle/svm/driver/BundleSupport.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 930dfdea8119..429c53eafac4 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -325,7 +325,7 @@ private void initializeContainerImage() { if (!containerTool.equals(bundleContainerTool)) { NativeImage.showWarning(String.format("The given bundle file %s was created with container tool '%s' (using '%s').", bundleFileName, bundleContainerTool, containerTool)); } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { - NativeImage.showWarning(String.format("The given bundle file %s was created with %s version '%s' (installed '%s').", bundleFileName, containerTool, bundleContainerToolVersion, containerToolVersion)); + NativeImage.showWarning(String.format("The given bundle file %s was created with different %s version '%s' (installed '%s').", bundleFileName, containerTool, bundleContainerToolVersion, containerToolVersion)); } } } else { @@ -560,8 +560,11 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } if(bundleContainerTool != null) { - String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" version: %s", bundleContainerToolVersion); - nativeImage.showMessage(String.format("%sUsing %s%s. Specify other container tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString, ExtendedBundleOptions.container)); + String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" (%s)", bundleContainerToolVersion); + nativeImage.showMessage(String.format("%sBundled native-image was created in a container with %s%s.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString)); + if(useContainer) { + nativeImage.showMessage(String.format("%sUsing %s for native-image container build. Specify other container tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, ExtendedBundleOptions.container)); + } } } From 2bb166b9bca4f3bc0bfb3d95e2712da26585978e Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 27 Apr 2023 13:12:47 +0200 Subject: [PATCH 11/61] Update default dockerfile to ol8 based dockerfile --- .../src/com/oracle/svm/driver/BundleSupport.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 429c53eafac4..053b926b3523 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -128,8 +128,13 @@ final class BundleSupport { private Path dockerfile; private Path bundleDockerfile; private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); - private static final String DEFAULT_DOCKERFILE = "FROM registry.fedoraproject.org/fedora-minimal:latest" + System.lineSeparator() + - "RUN microdnf -y install gcc g++ zlib-static --nodocs --setopt install_weak_deps=0 && microdnf clean all -y"; + private static final String DEFAULT_DOCKERFILE = "ARG BASE_IMAGE=container-registry.oracle.com/os/oraclelinux:8-slim" + System.lineSeparator() + + "FROM ${BASE_IMAGE}" + System.lineSeparator() + + "RUN microdnf update -y oraclelinux-release-el8 \\" + System.lineSeparator() + + "&& microdnf --enablerepo ol8_codeready_builder install bzip2-devel ed gcc gcc-c++ gcc-gfortran gzip file fontconfig less libcurl-devel make openssl openssl-devel readline-devel tar glibc-langpack-en \\" + System.lineSeparator() + + "vi which xz-devel zlib-devel findutils glibc-static libstdc++ libstdc++-devel libstdc++-static zlib-static \\" + System.lineSeparator() + + "&& microdnf clean all" + System.lineSeparator() + + "RUN fc-cache -f -v"; private final String containerToolJsonKey = "containerTool"; private final String containerToolVersionJsonKey = "containerToolVersion"; private final String containerImageJsonKey = "containerImage"; @@ -292,7 +297,6 @@ private void initializeContainerImage() { // create Dockerfile if not available for writing or loading bundle try { - // TODO load graalvm docker base if (dockerfile == null) { dockerfile = Files.createTempFile("Dockerfile", null); Files.write(dockerfile, DEFAULT_DOCKERFILE.getBytes()); From 6eb3f0798d97f6be9d316f86d6d80364701c91d1 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 2 May 2023 11:39:52 +0200 Subject: [PATCH 12/61] Fix missing bundlePath error on bundle-create --- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 053b926b3523..c562f33f310f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -289,7 +289,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } private void initializeContainerImage() { - String bundleFileName = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); + String bundleFileName = bundlePath == null ? "" : bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); if(bundleDockerfile != null && dockerfile == null) { dockerfile = bundleDockerfile; From fa087525f45db2ec6a18af55ead92e19411177f0 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 3 May 2023 12:00:40 +0200 Subject: [PATCH 13/61] Pass environment variables to container --- .../com/oracle/svm/driver/BundleSupport.java | 47 +++++++++++++++---- .../com/oracle/svm/driver/NativeImage.java | 7 ++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index c562f33f310f..53172f46d751 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -138,6 +138,7 @@ final class BundleSupport { private final String containerToolJsonKey = "containerTool"; private final String containerToolVersionJsonKey = "containerToolVersion"; private final String containerImageJsonKey = "containerImage"; + final Map containerEnvironment = new HashMap<>(); enum BundleOptionVariants { @@ -351,18 +352,43 @@ private void initializeContainerImage() { int exitStatusCode = createContainer(); switch (ExitStatus.of(exitStatusCode)) { - case OK: - break; - case BUILDER_ERROR: + case OK -> fetchContainerEnvironment(); + case BUILDER_ERROR -> /* Exit, builder has handled error reporting. */ - throw NativeImage.showError(null, null, exitStatusCode); - case OUT_OF_MEMORY: + throw NativeImage.showError(null, null, exitStatusCode); + case OUT_OF_MEMORY -> { nativeImage.showOutOfMemoryWarning(); throw NativeImage.showError(null, null, exitStatusCode); - default: + } + default -> { String message = String.format("Container build request for '%s' failed with exit status %d", nativeImage.imageName, exitStatusCode); throw NativeImage.showError(message, null, exitStatusCode); + } + } + } + + private void fetchContainerEnvironment() { + ProcessBuilder pb = new ProcessBuilder(containerTool, "run", "--rm", containerImage, "bash", "-c", "/usr/bin/env"); + Process p = null; + try { + p = pb.start(); + p.waitFor(); + (new BufferedReader(new InputStreamReader(p.getInputStream()))) + .lines() + .toList() + .forEach(env -> { + String[] envParts = SubstrateUtil.split(env,"=",2); + if(envParts.length == 2) { + containerEnvironment.put(envParts[0], envParts[1]); + } + }); + } catch (IOException | InterruptedException e) { + throw NativeImage.showError(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } } } @@ -431,13 +457,18 @@ private String getFirstProcessResultLine(ProcessBuilder pb) { List createContainerCommand(Path argFile, Path builderArgFile) { Path containerRoot = Path.of("/"); - return List.of(containerTool, "run", "--network=none", "--rm", + List containerCommand = new ArrayList<>(List.of(containerTool, "run", "--network=none", "--rm")); + nativeImage.imageBuilderEnvironment + .forEach((key, value) -> containerCommand.addAll(List.of("-e", key + "=" + value))); + containerCommand.addAll(List.of( "--mount", "type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + containerGraalVMHome + ",readonly", "--mount", "type=bind,source=" + inputDir + ",target=" + containerRoot.resolve(rootDir.relativize(inputDir)) + ",readonly", "--mount", "type=bind,source=" + outputDir + ",target=" + containerRoot.resolve(rootDir.relativize(outputDir)), "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", "--mount", "type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly", - containerImage); + containerImage) + ); + return containerCommand; } private BundleSupport(NativeImage nativeImage) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index f601c022c83d..a0f6513c1d1a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1576,7 +1576,12 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa command.add(NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + builderArgFile); ProcessBuilder pb = new ProcessBuilder(); pb.command(command); - Map environment = pb.environment(); + Map environment; + if(useBundle() && bundleSupport.useContainer) { + environment = bundleSupport.containerEnvironment; + } else { + environment = pb.environment(); + } String deprecatedSanitationKey = "NATIVE_IMAGE_DEPRECATED_BUILDER_SANITATION"; String deprecatedSanitationValue = System.getenv().getOrDefault(deprecatedSanitationKey, "false"); if (Boolean.parseBoolean(deprecatedSanitationValue)) { From 69bc2543c466a2fb42dd2955543507971e9eba62 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 4 May 2023 09:26:32 +0200 Subject: [PATCH 14/61] Add support for static builds in containers --- .../com/oracle/svm/driver/BundleSupport.java | 66 +++++++++++++------ .../com/oracle/svm/driver/NativeImage.java | 2 +- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 53172f46d751..312fb1124f89 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -117,7 +117,7 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = ".nib"; - final Path containerGraalVMHome = Path.of("/graalvm"); + static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); boolean useContainer; private String containerTool; private String bundleContainerTool; @@ -129,15 +129,39 @@ final class BundleSupport { private Path bundleDockerfile; private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private static final String DEFAULT_DOCKERFILE = "ARG BASE_IMAGE=container-registry.oracle.com/os/oraclelinux:8-slim" + System.lineSeparator() + - "FROM ${BASE_IMAGE}" + System.lineSeparator() + + "FROM ${BASE_IMAGE} as base" + System.lineSeparator() + "RUN microdnf update -y oraclelinux-release-el8 \\" + System.lineSeparator() + - "&& microdnf --enablerepo ol8_codeready_builder install bzip2-devel ed gcc gcc-c++ gcc-gfortran gzip file fontconfig less libcurl-devel make openssl openssl-devel readline-devel tar glibc-langpack-en \\" + System.lineSeparator() + - "vi which xz-devel zlib-devel findutils glibc-static libstdc++ libstdc++-devel libstdc++-static zlib-static \\" + System.lineSeparator() + - "&& microdnf clean all" + System.lineSeparator() + - "RUN fc-cache -f -v"; - private final String containerToolJsonKey = "containerTool"; - private final String containerToolVersionJsonKey = "containerToolVersion"; - private final String containerImageJsonKey = "containerImage"; + " && microdnf --enablerepo ol8_codeready_builder install bzip2-devel ed gcc gcc-c++ gcc-gfortran gzip file fontconfig less libcurl-devel make openssl openssl-devel readline-devel tar glibc-langpack-en \\" + System.lineSeparator() + + " vi which xz-devel zlib-devel findutils glibc-static libstdc++ libstdc++-devel libstdc++-static zlib-static \\" + System.lineSeparator() + + " && microdnf clean all" + System.lineSeparator() + + "RUN fc-cache -f -v" + System.lineSeparator() + + "ENV LANG=en_US.UTF-8 \\" + System.lineSeparator() + + " JAVA_HOME=" + CONTAINER_GRAAL_VM_HOME + System.lineSeparator() + + "WORKDIR /"; + private static final String DEFAULT_DOCKERFILE_MUSLIB = "FROM base as muslib" + System.lineSeparator() + + "ARG TEMP_REGION=\"\"" + System.lineSeparator() + + "ARG MUSL_LOCATION=http://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-native.tgz" + System.lineSeparator() + + "ARG ZLIB_LOCATION=https://zlib.net/fossils/zlib-1.2.11.tar.gz" + System.lineSeparator() + + "ENV TOOLCHAIN_DIR=/usr/local/musl \\" + System.lineSeparator() + + " CC=$TOOLCHAIN_DIR/bin/gcc" + System.lineSeparator() + + "RUN echo \"$TEMP_REGION\" > /etc/dnf/vars/ociregion \\" + System.lineSeparator() + + " && rm -rf /etc/yum.repos.d/ol8_graalvm_community.repo \\" + System.lineSeparator() + + " && mkdir -p $TOOLCHAIN_DIR \\" + System.lineSeparator() + + " && microdnf install -y wget tar gzip make \\" + System.lineSeparator() + + " && wget $MUSL_LOCATION && tar -xvf x86_64-linux-musl-native.tgz -C $TOOLCHAIN_DIR --strip-components=1 \\" + System.lineSeparator() + + " && wget $ZLIB_LOCATION && tar -xvf zlib-1.2.11.tar.gz \\" + System.lineSeparator() + + " && cd zlib-1.2.11 \\" + System.lineSeparator() + + " && ./configure --prefix=$TOOLCHAIN_DIR --static \\" + System.lineSeparator() + + " && make && make install" + System.lineSeparator() + + "FROM base as final" + System.lineSeparator() + + "COPY --from=muslib /usr/local/musl /usr/local/musl" + System.lineSeparator() + + "RUN echo \"\" > /etc/dnf/vars/ociregion" + System.lineSeparator() + + "ENV TOOLCHAIN_DIR=/usr/local/musl \\" + System.lineSeparator() + + " CC=$TOOLCHAIN_DIR/bin/gcc" + System.lineSeparator() + + "ENV PATH=$TOOLCHAIN_DIR/bin:$PATH"; + private static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; + private static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; + private static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; final Map containerEnvironment = new HashMap<>(); @@ -300,9 +324,13 @@ private void initializeContainerImage() { try { if (dockerfile == null) { dockerfile = Files.createTempFile("Dockerfile", null); - Files.write(dockerfile, DEFAULT_DOCKERFILE.getBytes()); + String dockerfileText = DEFAULT_DOCKERFILE; + if(nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { + dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; + } + Files.write(dockerfile, dockerfileText.getBytes()); dockerfile.toFile().deleteOnExit(); - containerImage = SubstrateUtil.digest(DEFAULT_DOCKERFILE); + containerImage = SubstrateUtil.digest(dockerfileText); } else { containerImage = SubstrateUtil.digest(Files.readString(dockerfile)); } @@ -461,7 +489,7 @@ List createContainerCommand(Path argFile, Path builderArgFile) { nativeImage.imageBuilderEnvironment .forEach((key, value) -> containerCommand.addAll(List.of("-e", key + "=" + value))); containerCommand.addAll(List.of( - "--mount", "type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + containerGraalVMHome + ",readonly", + "--mount", "type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + CONTAINER_GRAAL_VM_HOME + ",readonly", "--mount", "type=bind,source=" + inputDir + ",target=" + containerRoot.resolve(rootDir.relativize(inputDir)) + ",readonly", "--mount", "type=bind,source=" + outputDir + ",target=" + containerRoot.resolve(rootDir.relativize(outputDir)), "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", @@ -588,9 +616,9 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { if(Files.exists(containerFile)) { try (Reader reader = Files.newBufferedReader(containerFile)) { EconomicMap json = JSONParser.parseDict(reader); - if(json.get(containerImageJsonKey) != null) bundleContainerImage = json.get(containerImageJsonKey).toString(); - if(json.get(containerToolJsonKey) != null) bundleContainerTool = json.get(containerToolJsonKey).toString(); - if(json.get(containerToolVersionJsonKey) != null) bundleContainerToolVersion = json.get(containerToolVersionJsonKey).toString(); + if(json.get(CONTAINER_IMAGE_JSON_KEY) != null) bundleContainerImage = json.get(CONTAINER_IMAGE_JSON_KEY).toString(); + if(json.get(CONTAINER_TOOL_JSON_KEY) != null) bundleContainerTool = json.get(CONTAINER_TOOL_JSON_KEY).toString(); + if(json.get(CONTAINER_TOOL_VERSION_JSON_KEY) != null) bundleContainerToolVersion = json.get(CONTAINER_TOOL_VERSION_JSON_KEY).toString(); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } @@ -659,7 +687,7 @@ Path restoreCanonicalization(Path before) { void replacePathsForContainerBuild(List arguments) { arguments.replaceAll(arg -> arg - .replace(nativeImage.config.getJavaHome().toString(), containerGraalVMHome.toString()) + .replace(nativeImage.config.getJavaHome().toString(), CONTAINER_GRAAL_VM_HOME.toString()) .replace(rootDir.toString(), "") ); } @@ -939,9 +967,9 @@ private Path writeBundle() { if(useContainer) { Map containerInfo = new HashMap<>(); - if(containerImage != null) containerInfo.put(containerImageJsonKey, containerImage); - if(containerTool != null) containerInfo.put(containerToolJsonKey, containerTool); - if(containerToolVersion != null) containerInfo.put(containerToolVersionJsonKey, containerToolVersion); + if(containerImage != null) containerInfo.put(CONTAINER_IMAGE_JSON_KEY, containerImage); + if(containerTool != null) containerInfo.put(CONTAINER_TOOL_JSON_KEY, containerTool); + if(containerToolVersion != null) containerInfo.put(CONTAINER_TOOL_VERSION_JSON_KEY, containerToolVersion); if(!containerInfo.isEmpty()) { Path containerFile = stageDir.resolve("container.json"); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index a0f6513c1d1a..958b3a801a63 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1561,7 +1561,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa bundleSupport.replacePathsForContainerBuild(arguments); bundleSupport.replacePathsForContainerBuild(finalImageBuilderArgs); Path binJava = Paths.get("bin", "java"); - javaExecutable = bundleSupport.containerGraalVMHome.resolve(binJava).toString(); + javaExecutable = BundleSupport.CONTAINER_GRAAL_VM_HOME.resolve(binJava).toString(); } Path argFile = createVMInvocationArgumentFile(arguments); From e9b03943f7ebd59923b4f67f426c07edb143effa Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 4 May 2023 10:19:14 +0200 Subject: [PATCH 15/61] Quote mount calls and environment variables for containers to escape spaces --- .../src/com/oracle/svm/driver/BundleSupport.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 312fb1124f89..c9c5ec1d5f5a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -487,13 +487,13 @@ List createContainerCommand(Path argFile, Path builderArgFile) { Path containerRoot = Path.of("/"); List containerCommand = new ArrayList<>(List.of(containerTool, "run", "--network=none", "--rm")); nativeImage.imageBuilderEnvironment - .forEach((key, value) -> containerCommand.addAll(List.of("-e", key + "=" + value))); + .forEach((key, value) -> containerCommand.addAll(List.of("-e", key + "=" + SubstrateUtil.quoteShellArg(value)))); containerCommand.addAll(List.of( - "--mount", "type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + CONTAINER_GRAAL_VM_HOME + ",readonly", - "--mount", "type=bind,source=" + inputDir + ",target=" + containerRoot.resolve(rootDir.relativize(inputDir)) + ",readonly", - "--mount", "type=bind,source=" + outputDir + ",target=" + containerRoot.resolve(rootDir.relativize(outputDir)), - "--mount", "type=bind,source=" + argFile + ",target=" + argFile + ",readonly", - "--mount", "type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly", + "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + CONTAINER_GRAAL_VM_HOME.toString() + ",readonly"), + "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + inputDir + ",target=" + containerRoot.resolve(rootDir.relativize(inputDir)) + ",readonly"), + "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + outputDir + ",target=" + containerRoot.resolve(rootDir.relativize(outputDir))), + "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + argFile + ",target=" + argFile + ",readonly"), + "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly"), containerImage) ); return containerCommand; From d0b4b0afb02e2cb976e7e41c2a0b06911f98c6e1 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Sun, 21 May 2023 00:31:09 +0200 Subject: [PATCH 16/61] Store default Dockerfiles as resource files --- .../resources/container-default/Dockerfile | 14 ++++++++ .../Dockerfile_muslib_extension | 29 ++++++++++++++++ .../com/oracle/svm/driver/BundleSupport.java | 33 ++----------------- 3 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile create mode 100644 substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension diff --git a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile new file mode 100644 index 000000000000..9cd85d1a955d --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile @@ -0,0 +1,14 @@ +ARG BASE_IMAGE=container-registry.oracle.com/os/oraclelinux:8-slim + +FROM ${BASE_IMAGE} as base + +RUN microdnf update -y oraclelinux-release-el8 \ + && microdnf --enablerepo ol8_codeready_builder install bzip2-devel ed gcc gcc-c++ gcc-gfortran gzip file fontconfig less libcurl-devel make openssl openssl-devel readline-devel tar glibc-langpack-en \ + vi which xz-devel zlib-devel findutils glibc-static libstdc++ libstdc++-devel libstdc++-static zlib-static \ + && microdnf clean all +RUN fc-cache -f -v + +ENV LANG=en_US.UTF-8 \ + JAVA_HOME=/graalvm + +WORKDIR / \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension new file mode 100644 index 000000000000..a52d7d1ca374 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension @@ -0,0 +1,29 @@ +FROM base as muslib + +ARG TEMP_REGION="" +ARG MUSL_LOCATION=http://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-native.tgz +ARG ZLIB_LOCATION=https://zlib.net/fossils/zlib-1.2.11.tar.gz + +ENV TOOLCHAIN_DIR=/usr/local/musl \ + CC=$TOOLCHAIN_DIR/bin/gcc + +RUN echo "$TEMP_REGION" > /etc/dnf/vars/ociregion \ + && rm -rf /etc/yum.repos.d/ol8_graalvm_community.repo \ + && mkdir -p $TOOLCHAIN_DIR \ + && microdnf install -y wget tar gzip make \ + && wget $MUSL_LOCATION && tar -xvf x86_64-linux-musl-native.tgz -C $TOOLCHAIN_DIR --strip-components=1 \ + && wget $ZLIB_LOCATION && tar -xvf zlib-1.2.11.tar.gz \ + && cd zlib-1.2.11 \ + && ./configure --prefix=$TOOLCHAIN_DIR --static \ + && make && make install \ + + +FROM base as final + +COPY --from=muslib /usr/local/musl /usr/local/musl + +RUN echo "" > /etc/dnf/vars/ociregion + +ENV TOOLCHAIN_DIR=/usr/local/musl \ + CC=$TOOLCHAIN_DIR/bin/gcc \ + PATH=$TOOLCHAIN_DIR/bin:$PATH \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index c9c5ec1d5f5a..6d39c17cfd5a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -128,37 +128,8 @@ final class BundleSupport { private Path dockerfile; private Path bundleDockerfile; private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); - private static final String DEFAULT_DOCKERFILE = "ARG BASE_IMAGE=container-registry.oracle.com/os/oraclelinux:8-slim" + System.lineSeparator() + - "FROM ${BASE_IMAGE} as base" + System.lineSeparator() + - "RUN microdnf update -y oraclelinux-release-el8 \\" + System.lineSeparator() + - " && microdnf --enablerepo ol8_codeready_builder install bzip2-devel ed gcc gcc-c++ gcc-gfortran gzip file fontconfig less libcurl-devel make openssl openssl-devel readline-devel tar glibc-langpack-en \\" + System.lineSeparator() + - " vi which xz-devel zlib-devel findutils glibc-static libstdc++ libstdc++-devel libstdc++-static zlib-static \\" + System.lineSeparator() + - " && microdnf clean all" + System.lineSeparator() + - "RUN fc-cache -f -v" + System.lineSeparator() + - "ENV LANG=en_US.UTF-8 \\" + System.lineSeparator() + - " JAVA_HOME=" + CONTAINER_GRAAL_VM_HOME + System.lineSeparator() + - "WORKDIR /"; - private static final String DEFAULT_DOCKERFILE_MUSLIB = "FROM base as muslib" + System.lineSeparator() + - "ARG TEMP_REGION=\"\"" + System.lineSeparator() + - "ARG MUSL_LOCATION=http://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-native.tgz" + System.lineSeparator() + - "ARG ZLIB_LOCATION=https://zlib.net/fossils/zlib-1.2.11.tar.gz" + System.lineSeparator() + - "ENV TOOLCHAIN_DIR=/usr/local/musl \\" + System.lineSeparator() + - " CC=$TOOLCHAIN_DIR/bin/gcc" + System.lineSeparator() + - "RUN echo \"$TEMP_REGION\" > /etc/dnf/vars/ociregion \\" + System.lineSeparator() + - " && rm -rf /etc/yum.repos.d/ol8_graalvm_community.repo \\" + System.lineSeparator() + - " && mkdir -p $TOOLCHAIN_DIR \\" + System.lineSeparator() + - " && microdnf install -y wget tar gzip make \\" + System.lineSeparator() + - " && wget $MUSL_LOCATION && tar -xvf x86_64-linux-musl-native.tgz -C $TOOLCHAIN_DIR --strip-components=1 \\" + System.lineSeparator() + - " && wget $ZLIB_LOCATION && tar -xvf zlib-1.2.11.tar.gz \\" + System.lineSeparator() + - " && cd zlib-1.2.11 \\" + System.lineSeparator() + - " && ./configure --prefix=$TOOLCHAIN_DIR --static \\" + System.lineSeparator() + - " && make && make install" + System.lineSeparator() + - "FROM base as final" + System.lineSeparator() + - "COPY --from=muslib /usr/local/musl /usr/local/musl" + System.lineSeparator() + - "RUN echo \"\" > /etc/dnf/vars/ociregion" + System.lineSeparator() + - "ENV TOOLCHAIN_DIR=/usr/local/musl \\" + System.lineSeparator() + - " CC=$TOOLCHAIN_DIR/bin/gcc" + System.lineSeparator() + - "ENV PATH=$TOOLCHAIN_DIR/bin:$PATH"; + private static final String DEFAULT_DOCKERFILE = NativeImage.getResource("/container-default/Dockerfile"); + private static final String DEFAULT_DOCKERFILE_MUSLIB = NativeImage.getResource("/container-default/Dockerfile_muslib_extension"); private static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; private static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; private static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; From 924743166455b307ec1e624e9073836ed6e5908d Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Sun, 21 May 2023 00:35:54 +0200 Subject: [PATCH 17/61] Extract extended option parsing to separate function --- .../com/oracle/svm/driver/BundleSupport.java | 88 ++++++++++--------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 6d39c17cfd5a..b1a77de2157d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -219,49 +219,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma Arrays.stream(options) .skip(1) - .forEach(option -> { - String optionValue = null; - String[] optionParts = SubstrateUtil.split(option, "=", 2); - if (optionParts.length == 2) { - option = optionParts[0]; - optionValue = optionParts[1]; - } - switch (ExtendedBundleOptions.get(option)) { - case dry_run -> nativeImage.setDryRun(true); - case container -> { - if (bundleSupport.useContainer) { - throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); - } - bundleSupport.useContainer = true; - if (optionValue != null) { - if (!SUPPORTED_CONTAINER_TOOLS.contains(optionValue)) { - throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, SUPPORTED_CONTAINER_TOOLS)); - } - bundleSupport.containerTool = optionValue; - } - } - case dockerfile -> { - if (!bundleSupport.useContainer) { - throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", option, ExtendedBundleOptions.container)); - } - if (bundleSupport.dockerfile != null) { - throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); - } - if (optionValue != null) { - bundleSupport.dockerfile = Path.of(optionValue); - if (!Files.isReadable(bundleSupport.dockerfile)) { - throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", bundleSupport.dockerfile.toAbsolutePath())); - } - } - } - default -> { - String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) - .map(Enum::toString) - .collect(Collectors.joining(", ")); - throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", option, suggestedOptions)); - } - } - }); + .forEach(bundleSupport::parseExtendedOption); if(bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { @@ -284,6 +242,50 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } + private void parseExtendedOption(String option) { + String optionValue = null; + String[] optionParts = SubstrateUtil.split(option, "=", 2); + if (optionParts.length == 2) { + option = optionParts[0]; + optionValue = optionParts[1]; + } + switch (ExtendedBundleOptions.get(option)) { + case dry_run -> nativeImage.setDryRun(true); + case container -> { + if (useContainer) { + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); + } + useContainer = true; + if (optionValue != null) { + if (!SUPPORTED_CONTAINER_TOOLS.contains(optionValue)) { + throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, SUPPORTED_CONTAINER_TOOLS)); + } + containerTool = optionValue; + } + } + case dockerfile -> { + if (!useContainer) { + throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", option, ExtendedBundleOptions.container)); + } + if (dockerfile != null) { + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); + } + if (optionValue != null) { + dockerfile = Path.of(optionValue); + if (!Files.isReadable(dockerfile)) { + throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", dockerfile.toAbsolutePath())); + } + } + } + default -> { + String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) + .map(Enum::toString) + .collect(Collectors.joining(", ")); + throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", option, suggestedOptions)); + } + } + } + private void initializeContainerImage() { String bundleFileName = bundlePath == null ? "" : bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); From de4f98bdff0df8a36f8acc4332e54d1caf8e29e6 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Sun, 21 May 2023 00:38:51 +0200 Subject: [PATCH 18/61] Use java.nio.file.Files#writeString --- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index b1a77de2157d..29e119af0753 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -301,7 +301,7 @@ private void initializeContainerImage() { if(nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; } - Files.write(dockerfile, dockerfileText.getBytes()); + Files.writeString(dockerfile, dockerfileText); dockerfile.toFile().deleteOnExit(); containerImage = SubstrateUtil.digest(dockerfileText); } else { From 31f8608d87c09ac1405cea8b2f457f368d0e932f Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Sun, 21 May 2023 00:56:43 +0200 Subject: [PATCH 19/61] Use try-with-resource blocks for reading process results --- .../src/com/oracle/svm/driver/BundleSupport.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 29e119af0753..cc525df09cfb 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -375,8 +375,8 @@ private void fetchContainerEnvironment() { try { p = pb.start(); p.waitFor(); - (new BufferedReader(new InputStreamReader(p.getInputStream()))) - .lines() + try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + processResult.lines() .toList() .forEach(env -> { String[] envParts = SubstrateUtil.split(env,"=",2); @@ -384,6 +384,7 @@ private void fetchContainerEnvironment() { containerEnvironment.put(envParts[0], envParts[1]); } }); + } } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); } finally { @@ -408,11 +409,10 @@ private int createContainer() { try { p = pb.start(); int status = p.waitFor(); - if(status == 0 && imageId != null) { - Stream result = (new BufferedReader(new InputStreamReader(p.getInputStream()))).lines(); - if(!imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { + if(status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { + try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { nativeImage.showMessage(String.format("%sUpdated container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); - result.forEach(System.out::println); + processResult.lines().forEach(System.out::println); } } return status; @@ -446,7 +446,9 @@ private String getFirstProcessResultLine(ProcessBuilder pb) { try { p = pb.start(); p.waitFor(); - return (new BufferedReader(new InputStreamReader(p.getInputStream()))).readLine(); + try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + return processResult.readLine(); + } } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); } finally { From 8e2b7a27fdaefdba20607695f4426b2c9efd898e Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 23 May 2023 15:16:15 +0200 Subject: [PATCH 20/61] Fix style issues --- .../com/oracle/svm/driver/BundleSupport.java | 162 +++++++++++------- .../com/oracle/svm/driver/NativeImage.java | 9 +- 2 files changed, 107 insertions(+), 64 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index cc525df09cfb..77c5635e4be8 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -61,6 +61,7 @@ import java.util.jar.JarFile; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.oracle.svm.core.util.ExitStatus; @@ -221,12 +222,12 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma .skip(1) .forEach(bundleSupport::parseExtendedOption); - if(bundleSupport.useContainer) { + if (bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); bundleSupport.useContainer = false; } else { - if(nativeImage.isDryRun()) { + if (nativeImage.isDryRun()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); } else { bundleSupport.initializeContainerImage(); @@ -243,17 +244,23 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } private void parseExtendedOption(String option) { - String optionValue = null; + String optionKey; + String optionValue; + String[] optionParts = SubstrateUtil.split(option, "=", 2); if (optionParts.length == 2) { - option = optionParts[0]; + optionKey = optionParts[0]; optionValue = optionParts[1]; + } else { + optionKey = option; + optionValue = null; } - switch (ExtendedBundleOptions.get(option)) { + + switch (ExtendedBundleOptions.get(optionKey)) { case dry_run -> nativeImage.setDryRun(true); case container -> { if (useContainer) { - throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); } useContainer = true; if (optionValue != null) { @@ -265,10 +272,10 @@ private void parseExtendedOption(String option) { } case dockerfile -> { if (!useContainer) { - throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", option, ExtendedBundleOptions.container)); + throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, ExtendedBundleOptions.container)); } if (dockerfile != null) { - throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option)); + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); } if (optionValue != null) { dockerfile = Path.of(optionValue); @@ -281,7 +288,7 @@ private void parseExtendedOption(String option) { String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) .map(Enum::toString) .collect(Collectors.joining(", ")); - throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", option, suggestedOptions)); + throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", optionKey, suggestedOptions)); } } } @@ -289,7 +296,7 @@ private void parseExtendedOption(String option) { private void initializeContainerImage() { String bundleFileName = bundlePath == null ? "" : bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); - if(bundleDockerfile != null && dockerfile == null) { + if (bundleDockerfile != null && dockerfile == null) { dockerfile = bundleDockerfile; } @@ -298,7 +305,7 @@ private void initializeContainerImage() { if (dockerfile == null) { dockerfile = Files.createTempFile("Dockerfile", null); String dockerfileText = DEFAULT_DOCKERFILE; - if(nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { + if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; } Files.writeString(dockerfile, dockerfileText); @@ -311,23 +318,23 @@ private void initializeContainerImage() { throw NativeImage.showError(e.getMessage()); } - if(bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { + if (bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { NativeImage.showWarning(String.format("The given bundle file %s was created with a different dockerfile.", bundleFileName)); } - if(bundleContainerTool != null && containerTool == null) { + if (bundleContainerTool != null && containerTool == null) { containerTool = bundleContainerTool; } - if(containerTool != null) { - if(!isToolAvailable(containerTool)) { + if (containerTool != null) { + if (!isToolAvailable(containerTool)) { throw NativeImage.showError("Configured container tool not available."); - } else if(containerTool.equals("docker") && !isRootlessDocker()) { + } else if (containerTool.equals("docker") && !isRootlessDocker()) { throw NativeImage.showError("Only rootless docker is supported for containerized builds."); } containerToolVersion = getContainerToolVersion(containerTool); - if(bundleContainerTool != null) { + if (bundleContainerTool != null) { if (!containerTool.equals(bundleContainerTool)) { NativeImage.showWarning(String.format("The given bundle file %s was created with container tool '%s' (using '%s').", bundleFileName, bundleContainerTool, containerTool)); } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { @@ -335,9 +342,9 @@ private void initializeContainerImage() { } } } else { - for(String tool : SUPPORTED_CONTAINER_TOOLS) { - if(isToolAvailable(tool)) { - if(tool.equals("docker") && !isRootlessDocker()) { + for (String tool : SUPPORTED_CONTAINER_TOOLS) { + if (isToolAvailable(tool)) { + if (tool.equals("docker") && !isRootlessDocker()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); continue; } @@ -377,13 +384,13 @@ private void fetchContainerEnvironment() { p.waitFor(); try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { processResult.lines() - .toList() - .forEach(env -> { - String[] envParts = SubstrateUtil.split(env,"=",2); - if(envParts.length == 2) { - containerEnvironment.put(envParts[0], envParts[1]); - } - }); + .toList() + .forEach(env -> { + String[] envParts = SubstrateUtil.split(env, "=", 2); + if (envParts.length == 2) { + containerEnvironment.put(envParts[0], envParts[1]); + } + }); } } catch (IOException | InterruptedException e) { throw NativeImage.showError(e.getMessage()); @@ -399,7 +406,7 @@ private int createContainer() { ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); String imageId = getFirstProcessResultLine(pbCheckForImage); - if(imageId == null) { + if (imageId == null) { pb.inheritIO(); } else { nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); @@ -409,7 +416,7 @@ private int createContainer() { try { p = pb.start(); int status = p.waitFor(); - if(status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { + if (status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { nativeImage.showMessage(String.format("%sUpdated container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); processResult.lines().forEach(System.out::println); @@ -425,23 +432,23 @@ private int createContainer() { } } - private boolean isToolAvailable(String tool) { + private static boolean isToolAvailable(String tool) { return Arrays.stream(SubstrateUtil.split(System.getenv("PATH"), ":")) .map(str -> Path.of(str).resolve(tool)) .anyMatch(Files::isExecutable); } - private String getContainerToolVersion(String tool) { + private static String getContainerToolVersion(String tool) { ProcessBuilder pb = new ProcessBuilder(tool, "--version"); return getFirstProcessResultLine(pb); } - private boolean isRootlessDocker() { + private static boolean isRootlessDocker() { ProcessBuilder pb = new ProcessBuilder("docker", "context", "show"); return getFirstProcessResultLine(pb).equals("rootless"); } - private String getFirstProcessResultLine(ProcessBuilder pb) { + private static String getFirstProcessResultLine(ProcessBuilder pb) { Process p = null; try { p = pb.start(); @@ -460,20 +467,41 @@ private String getFirstProcessResultLine(ProcessBuilder pb) { List createContainerCommand(Path argFile, Path builderArgFile) { Path containerRoot = Path.of("/"); - List containerCommand = new ArrayList<>(List.of(containerTool, "run", "--network=none", "--rm")); + List containerCommand = new ArrayList<>(); + + // run docker tool without network access and remove container after image build is finished + containerCommand.add(containerTool); + containerCommand.add("run"); + containerCommand.add("--network=none"); + containerCommand.add("--rm"); + + // inject environment variables into container nativeImage.imageBuilderEnvironment - .forEach((key, value) -> containerCommand.addAll(List.of("-e", key + "=" + SubstrateUtil.quoteShellArg(value)))); - containerCommand.addAll(List.of( - "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + nativeImage.config.getJavaHome() + ",target=" + CONTAINER_GRAAL_VM_HOME.toString() + ",readonly"), - "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + inputDir + ",target=" + containerRoot.resolve(rootDir.relativize(inputDir)) + ",readonly"), - "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + outputDir + ",target=" + containerRoot.resolve(rootDir.relativize(outputDir))), - "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + argFile + ",target=" + argFile + ",readonly"), - "--mount", SubstrateUtil.quoteShellArg("type=bind,source=" + builderArgFile + ",target=" + builderArgFile + ",readonly"), - containerImage) - ); + .forEach((key, value) -> { + containerCommand.add("-e"); + containerCommand.add(key + "=" + SubstrateUtil.quoteShellArg(value)); + }); + + // mount java home, input and output directories and argument files for native image build + containerCommand.addAll(getMountCommand(nativeImage.config.getJavaHome(), CONTAINER_GRAAL_VM_HOME, true)); + containerCommand.addAll(getMountCommand(inputDir, containerRoot.resolve(rootDir.relativize(inputDir)), true)); + containerCommand.addAll(getMountCommand(outputDir, containerRoot.resolve(rootDir.relativize(outputDir)), false)); + containerCommand.addAll(getMountCommand(argFile, argFile, true)); + containerCommand.addAll(getMountCommand(builderArgFile, builderArgFile, true)); + + // specify container name + containerCommand.add(containerImage); + return containerCommand; } + private static List getMountCommand(Path source, Path target, boolean readonly) { + return List.of( + "--mount", + SubstrateUtil.quoteShellArg("type=bind,source=" + source + ",target=" + target + (readonly ? ",readonly" : "")) + ); + } + private BundleSupport(NativeImage nativeImage) { Objects.requireNonNull(nativeImage); this.nativeImage = nativeImage; @@ -588,26 +616,32 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } Path containerFile = stageDir.resolve("container.json"); - if(Files.exists(containerFile)) { + if (Files.exists(containerFile)) { try (Reader reader = Files.newBufferedReader(containerFile)) { EconomicMap json = JSONParser.parseDict(reader); - if(json.get(CONTAINER_IMAGE_JSON_KEY) != null) bundleContainerImage = json.get(CONTAINER_IMAGE_JSON_KEY).toString(); - if(json.get(CONTAINER_TOOL_JSON_KEY) != null) bundleContainerTool = json.get(CONTAINER_TOOL_JSON_KEY).toString(); - if(json.get(CONTAINER_TOOL_VERSION_JSON_KEY) != null) bundleContainerToolVersion = json.get(CONTAINER_TOOL_VERSION_JSON_KEY).toString(); + if (json.get(CONTAINER_IMAGE_JSON_KEY) != null) { + bundleContainerImage = json.get(CONTAINER_IMAGE_JSON_KEY).toString(); + } + if (json.get(CONTAINER_TOOL_JSON_KEY) != null) { + bundleContainerTool = json.get(CONTAINER_TOOL_JSON_KEY).toString(); + } + if (json.get(CONTAINER_TOOL_VERSION_JSON_KEY) != null) { + bundleContainerToolVersion = json.get(CONTAINER_TOOL_VERSION_JSON_KEY).toString(); + } } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } - if(bundleContainerTool != null) { + if (bundleContainerTool != null) { String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" (%s)", bundleContainerToolVersion); nativeImage.showMessage(String.format("%sBundled native-image was created in a container with %s%s.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString)); - if(useContainer) { + if (useContainer) { nativeImage.showMessage(String.format("%sUsing %s for native-image container build. Specify other container tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, ExtendedBundleOptions.container)); } } } bundleDockerfile = stageDir.resolve("Dockerfile"); - if(!Files.isReadable(bundleDockerfile)) { + if (!Files.isReadable(bundleDockerfile)) { bundleDockerfile = null; } @@ -940,13 +974,19 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + environmentFile, e); } - if(useContainer) { + if (useContainer) { Map containerInfo = new HashMap<>(); - if(containerImage != null) containerInfo.put(CONTAINER_IMAGE_JSON_KEY, containerImage); - if(containerTool != null) containerInfo.put(CONTAINER_TOOL_JSON_KEY, containerTool); - if(containerToolVersion != null) containerInfo.put(CONTAINER_TOOL_VERSION_JSON_KEY, containerToolVersion); + if (containerImage != null) { + containerInfo.put(CONTAINER_IMAGE_JSON_KEY, containerImage); + } + if (containerTool != null) { + containerInfo.put(CONTAINER_TOOL_JSON_KEY, containerTool); + } + if (containerToolVersion != null) { + containerInfo.put(CONTAINER_TOOL_VERSION_JSON_KEY, containerToolVersion); + } - if(!containerInfo.isEmpty()) { + if (!containerInfo.isEmpty()) { Path containerFile = stageDir.resolve("container.json"); try (JsonWriter writer = new JsonWriter(containerFile)) { writer.print(containerInfo); @@ -955,8 +995,8 @@ private Path writeBundle() { } } - if(dockerfile != null) { - Path bundleDockerfile = stageDir.resolve("Dockerfile"); + if (dockerfile != null) { + bundleDockerfile = stageDir.resolve("Dockerfile"); try { Files.copy(dockerfile, bundleDockerfile); } catch (IOException e) { @@ -1158,7 +1198,7 @@ private void loadAndVerify() { fileVersionKey = PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR; int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); String message = String.format("The given bundle file %s was created with newer bundle-file-format version %d.%d" + - " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, BUNDLE_FILE_FORMAT_VERSION_MAJOR, BUNDLE_FILE_FORMAT_VERSION_MINOR); + " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, BUNDLE_FILE_FORMAT_VERSION_MAJOR, BUNDLE_FILE_FORMAT_VERSION_MINOR); if (major > BUNDLE_FILE_FORMAT_VERSION_MAJOR) { throw NativeImage.showError(message); } else if (major == BUNDLE_FILE_FORMAT_VERSION_MAJOR) { @@ -1189,9 +1229,9 @@ private void loadAndVerify() { nativeImage.showMessage("%sLoaded Bundle from %s", BUNDLE_INFO_MESSAGE_PREFIX, bundleFileName); nativeImage.showMessage("%sBundle created at '%s'", BUNDLE_INFO_MESSAGE_PREFIX, localDateStr); nativeImage.showMessage("%sUsing version: '%s'%s (vendor '%s'%s) on platform: '%s'%s", BUNDLE_INFO_MESSAGE_PREFIX, - bundleVersion, currentVersion, - bundleVendor, currentVendor, - bundlePlatform, currentPlatform); + bundleVersion, currentVersion, + bundleVendor, currentVendor, + bundlePlatform, currentPlatform); } private boolean forceBuilderOnClasspath() { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 958b3a801a63..241e9990127c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1557,7 +1557,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa /* Construct ProcessBuilder command from final arguments */ List command = new ArrayList<>(); - if(useBundle() && bundleSupport.useContainer) { + if (useBundle() && bundleSupport.useContainer) { bundleSupport.replacePathsForContainerBuild(arguments); bundleSupport.replacePathsForContainerBuild(finalImageBuilderArgs); Path binJava = Paths.get("bin", "java"); @@ -1567,7 +1567,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa Path argFile = createVMInvocationArgumentFile(arguments); Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); - if(!isDryRun() && useBundle() && bundleSupport.useContainer) { + if (useBundle() && bundleSupport.useContainer) { command.addAll(bundleSupport.createContainerCommand(argFile, builderArgFile)); } @@ -1577,7 +1577,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa ProcessBuilder pb = new ProcessBuilder(); pb.command(command); Map environment; - if(useBundle() && bundleSupport.useContainer) { + if (useBundle() && bundleSupport.useContainer) { environment = bundleSupport.containerEnvironment; } else { environment = pb.environment(); @@ -1607,6 +1607,9 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa List completeCommandList = new ArrayList<>(); completeCommandList.addAll(environment.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).sorted().toList()); + if (useBundle() && bundleSupport.useContainer) { + completeCommandList.addAll(bundleSupport.createContainerCommand(argFile, builderArgFile)); + } completeCommandList.add(javaExecutable); completeCommandList.addAll(arguments); completeCommandList.addAll(finalImageBuilderArgs); From 30bfb5cffeae3cb79f6eb43a929bd9a0c1b72354 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 17 May 2023 10:32:56 +0200 Subject: [PATCH 21/61] Executable bundle first draft --- substratevm/mx.substratevm/suite.py | 18 + .../svm/driver/launcher/BundleLauncher.java | 363 ++++++++++++++++++ .../com/oracle/svm/driver/BundleSupport.java | 50 ++- 3 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 28418b919c62..a6a4ef0b9b2c 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -842,6 +842,7 @@ ], "dependencies": [ "com.oracle.svm.hosted", + "com.oracle.svm.driver.launcher", ], "requires" : [ "jdk.management", @@ -857,6 +858,22 @@ "jacoco" : "exclude", }, + "com.oracle.svm.driver.launcher": { + "subDir": "src", + "sourceDirs": [ + "src" + ], + "checkstyle": "com.oracle.svm.hosted", + "workingSets": "SVM", + "annotationProcessors": [ + "compiler:GRAAL_PROCESSOR", + "SVM_PROCESSOR", + ], + "javaCompliance" : "17+", + "spotbugs": "false", + "jacoco" : "exclude", + }, + "com.oracle.svm.junit": { "subDir": "src", "sourceDirs": [ @@ -1690,6 +1707,7 @@ "mainClass": "com.oracle.svm.driver.NativeImage", "dependencies": [ "com.oracle.svm.driver", + "com.oracle.svm.driver.launcher", "svm-compiler-flags-builder", ], "distDependencies": [ diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java new file mode 100644 index 000000000000..a11ba870d535 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -0,0 +1,363 @@ +package com.oracle.svm.driver.launcher; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.stream.Stream; + + +public class BundleLauncher { + private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; + + private static Path rootDir; + private static Path inputDir; + private static Path stageDir; + private static Path auxiliaryDir; + private static Path classPathDir; + private static Path modulePathDir; + + + public static void main(String[] args) { + List command = new ArrayList<>(); + Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); + command.add(javaExecutable.toString()); + + String bundleFilePath = BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + unpackBundle(Path.of(bundleFilePath)); + + Path environmentFile = stageDir.resolve("environment.json"); + if (Files.isReadable(environmentFile)) { + try (Reader reader = Files.newBufferedReader(environmentFile)) { + //new EnvironmentParser(nativeImage.imageBuilderEnvironment).parseAndRegister(reader); + } catch (IOException e) { + throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); + } + } + + + List classpath = new ArrayList<>(); + if (Files.isDirectory(classPathDir)) { + try (Stream walk = Files.walk(classPathDir, 1)) { + walk.filter(path -> path.toString().endsWith(".jar") || (Files.isDirectory(path) && !path.equals(classPathDir))) + .map(Path::toString) + .forEach(classpath::add); + } catch (IOException e) { + throw new RuntimeException("Failed to iterate through directory " + classPathDir, e); + } + + classpath.add(classPathDir.toString()); + command.add("-cp"); + command.add(String.join(File.pathSeparator, classpath)); + } + + + List modulePath = new ArrayList<>(); + if (Files.isDirectory(modulePathDir)) { + try (Stream walk = Files.walk(modulePathDir, 1)) { + walk.filter(Files::isDirectory) + .filter(path -> !path.equals(modulePathDir)) + .map(Path::toString) + .forEach(modulePath::add); + } catch (IOException e) { + throw new RuntimeException("Failed to iterate through directory " + modulePathDir, e); + } + + if(!modulePath.isEmpty()) { + command.add("-p"); + command.add(String.join(File.pathSeparator, modulePath)); + } + } + + Path buildArgsFile = stageDir.resolve("run.json"); + try (Reader reader = Files.newBufferedReader(buildArgsFile)) { + command.addAll(parseArray(readFully(reader))); + } catch (IOException e) { + throw new RuntimeException("Failed to read bundle-file " + buildArgsFile, e); + } + + if (System.getenv("BUNDLE_LAUNCHER_VERBOSE") != null) { + System.out.println("Exec: " + String.join(" ", command)); + } + + ProcessBuilder pb = new ProcessBuilder(command); + Process p = null; + try { + p = pb.inheritIO().start(); + p.waitFor(); + } catch (IOException | InterruptedException e) { + showError("Failed to run bundled application"); + } finally { + if(p != null) { + p.destroy(); + } + } + } + + + private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); + + private static Path getJavaExecutable() { + Path binJava = Paths.get("bin", System.getProperty("os.name").contains("Windows") ? "java.exe" : "java"); + + Path javaCandidate = buildTimeJavaHome.resolve(binJava); + if (Files.isExecutable(javaCandidate)) { + return javaCandidate; + } + + javaCandidate = Paths.get(".").resolve(binJava); + if (Files.isExecutable(javaCandidate)) { + return javaCandidate; + } + + String javaHome = System.getenv("JAVA_HOME"); + if (javaHome == null) { + showError("No " + binJava + " and no environment variable JAVA_HOME"); + } + try { + javaCandidate = Paths.get(javaHome).resolve(binJava); + if (Files.isExecutable(javaCandidate)) { + return javaCandidate; + } + } catch (InvalidPathException e) { + /* fallthrough */ + } + showError("No " + binJava + " and invalid JAVA_HOME=" + javaHome); + return null; + } + + private static void showError(String s) { + System.err.println("Error: " + s); + System.exit(1); + } + + private static String readFully(final Reader reader) throws IOException { + final char[] arr = new char[1024]; + final StringBuilder sb = new StringBuilder(); + + try { + int numChars; + while ((numChars = reader.read(arr, 0, arr.length)) > 0) { + sb.append(arr, 0, numChars); + } + } finally { + reader.close(); + } + + return sb.toString(); + } + + + private static final int STATE_EMPTY = 0; + private static final int STATE_ELEMENT_PARSED = 1; + private static final int STATE_COMMA_PARSED = 2; + + private static int pos; + + private static List parseArray(String jsonArray) { + List result = new ArrayList<>(); + int state = STATE_EMPTY; + pos = 0; + + skipWhiteSpace(jsonArray); + if(jsonArray.charAt(pos) != '[') { + throw new RuntimeException("Expected [ but found " + jsonArray.charAt(pos)); + } + pos++; + + while (pos < jsonArray.length()) { + pos = skipWhiteSpace(jsonArray); + + switch (jsonArray.charAt(pos)) { + case ',' -> { + if (state != STATE_ELEMENT_PARSED) { + throw new RuntimeException("Trailing comma is not allowed in JSON"); + } + state = STATE_COMMA_PARSED; + pos++; + } + case ']' -> { + if (state == STATE_COMMA_PARSED) { + throw new RuntimeException("Trailing comma is not allowed in JSON"); + } + return result; + } + default -> { + if (state == STATE_ELEMENT_PARSED) { + throw new RuntimeException("Expected , or ] but found " + jsonArray.charAt(pos)); + } + result.add(parseString(jsonArray)); + state = STATE_ELEMENT_PARSED; + } + } + } + + throw new RuntimeException("Expected , or ] but found eof"); + } + + private static String parseString(String json) { + // String buffer is only instantiated if string contains escape sequences. + int start = ++pos; + StringBuilder sb = null; + + while (pos < json.length()) { + final int c = json.charAt(pos); + pos++; + if (c <= 0x1f) { + // Characters < 0x1f are not allowed in JSON strings. + throw new RuntimeException("String contains control character"); + + } else if (c == '\\') { + if (sb == null) { + sb = new StringBuilder(pos - start + 16); + } + sb.append(json, start, pos - 1); + sb.append(parseEscapeSequence(json)); + start = pos; + + } else if (c == '"') { + if (sb != null) { + sb.append(json, start, pos - 1); + return sb.toString(); + } + return json.substring(start, pos - 1); + } + } + + throw new RuntimeException("Missing close quote"); + } + + private static char parseEscapeSequence(String json) { + final int c = json.charAt(pos); + pos++; + return switch (c) { + case '"' -> '"'; + case '\\' -> '\\'; + case '/' -> '/'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'u' -> parseUnicodeEscape(json); + default -> throw new RuntimeException("Invalid escape character"); + }; + } + + private static char parseUnicodeEscape(String json) { + return (char) (parseHexDigit(json) << 12 | parseHexDigit(json) << 8 | parseHexDigit(json) << 4 | parseHexDigit(json)); + } + + private static int parseHexDigit(String json) { + final int c = json.charAt(pos); + pos++; + if (c >= '0' && c <= '9') { + return c - '0'; + } else if (c >= 'A' && c <= 'F') { + return c + 10 - 'A'; + } else if (c >= 'a' && c <= 'f') { + return c + 10 - 'a'; + } + throw new RuntimeException("Invalid hex digit"); + } + + private static int skipWhiteSpace(String str) { + while (pos < str.length()) { + switch (str.charAt(pos)) { + case '\t', '\r', '\n', ' ' -> pos++; + default -> { + return pos; + } + } + } + + return pos; + } + + private static final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); + private static Path createBundleRootDir() throws IOException { + Path bundleRoot = Files.createTempDirectory(BUNDLE_TEMP_DIR_PREFIX); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + deleteBundleRoot.set(true); + deleteAllFiles(bundleRoot); + })); + return bundleRoot; + } + + + private static final String deletedFileSuffix = ".deleted"; + private static boolean isDeletedPath(Path toDelete) { + return toDelete.getFileName().toString().endsWith(deletedFileSuffix); + } + + private static void deleteAllFiles(Path toDelete) { + try { + Path deletedPath = toDelete; + if (!isDeletedPath(deletedPath)) { + deletedPath = toDelete.resolveSibling(toDelete.getFileName() + deletedFileSuffix); + Files.move(toDelete, deletedPath); + } + Files.walk(deletedPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } catch (IOException e) { + //if (isVerbose()) { + System.out.println("Could not recursively delete path: " + toDelete); + e.printStackTrace(); + //} + } + } + + + private static void unpackBundle(Path bundleFilePath) { + try { + rootDir = createBundleRootDir(); + inputDir = rootDir.resolve("input"); + + try (JarFile archive = new JarFile(bundleFilePath.toFile())) { + Enumeration jarEntries = archive.entries(); + while (jarEntries.hasMoreElements() && !deleteBundleRoot.get()) { + JarEntry jarEntry = jarEntries.nextElement(); + Path bundleEntry = rootDir.resolve(jarEntry.getName()); + if (bundleEntry.startsWith(inputDir)) { + try { + Path bundleFileParent = bundleEntry.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); + } + Files.copy(archive.getInputStream(jarEntry), bundleEntry); + } catch (IOException e) { + throw new RuntimeException("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); + } + } + } + } + } catch (IOException e) { + throw new RuntimeException("Unable to expand bundle directory layout from bundle file " + bundleFilePath, e); + } + + if (deleteBundleRoot.get()) { + /* Abort image build request without error message and exit with 0 */ + throw new RuntimeException(""); + } + + try { + inputDir = rootDir.resolve("input"); + stageDir = Files.createDirectories(inputDir.resolve("stage")); + auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); + Path classesDir = inputDir.resolve("classes"); + classPathDir = Files.createDirectories(classesDir.resolve("cp")); + modulePathDir = Files.createDirectories(classesDir.resolve("p")); + } catch (IOException e) { + throw new RuntimeException("Unable to create bundle directory layout", e); + } + } +} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 77c5635e4be8..962413899195 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -65,7 +65,9 @@ import java.util.stream.Stream; import com.oracle.svm.core.util.ExitStatus; +import com.oracle.svm.driver.launcher.BundleLauncher; import org.graalvm.collections.EconomicMap; +import org.graalvm.collections.Pair; import org.graalvm.util.json.JSONParser; import org.graalvm.util.json.JSONParserException; @@ -952,6 +954,31 @@ private Path writeBundle() { nativeImage.deleteAllFiles(metaInfDir); } + Path launcherFilePath = rootDir.resolve(BundleLauncher.class.getName().replace(".","/") + ".class"); + + try { + Files.createDirectories(launcherFilePath.getParent()); + Files.copy(BundleLauncher.class.getResourceAsStream(BundleLauncher.class.getSimpleName() + ".class"), launcherFilePath); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + launcherFilePath, e); + } + + /* + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(launcherFilePath))) { + //String jarEntryName = BundleLauncher.class.getName().replace(".","/") + ".class"; + String jarEntryName = BundleLauncher.class.getSimpleName() + ".class"; + JarEntry entry = new JarEntry(jarEntryName); + jarOutStream.putNextEntry(entry); + Path tempLauncherFile = Files.createTempFile(BundleLauncher.class.getSimpleName(), null); + Files.copy(BundleLauncher.class.getResourceAsStream(BundleLauncher.class.getSimpleName() + ".class"), tempLauncherFile, StandardCopyOption.REPLACE_EXISTING); + Files.copy(tempLauncherFile, jarOutStream); + Files.delete(tempLauncherFile); + jarOutStream.closeEntry(); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + launcherFilePath, e); + } + */ + Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (JsonWriter writer = new JsonWriter(pathCanonicalizationsFile)) { /* Printing as list with defined sort-order ensures useful diffs are possible */ @@ -1006,10 +1033,10 @@ private Path writeBundle() { } Path buildArgsFile = stageDir.resolve("build.json"); + ArrayList bundleArgs = new ArrayList<>(updatedNativeImageArgs != null ? updatedNativeImageArgs : nativeImageArgs); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { List equalsNonBundleOptions = List.of(CmdLineOptionHandler.VERBOSE_OPTION, CmdLineOptionHandler.DRY_RUN_OPTION); List startsWithNonBundleOptions = List.of(BUNDLE_OPTION, DefaultOptionHandler.ADD_ENV_VAR_OPTION, nativeImage.oHPath); - ArrayList bundleArgs = new ArrayList<>(updatedNativeImageArgs != null ? updatedNativeImageArgs : nativeImageArgs); ListIterator bundleArgsIterator = bundleArgs.listIterator(); while (bundleArgsIterator.hasNext()) { String arg = bundleArgsIterator.next(); @@ -1029,6 +1056,24 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + buildArgsFile, e); } + Path runArgsFile = stageDir.resolve("run.json"); + try (JsonWriter writer = new JsonWriter(runArgsFile)) { + List isPathArg = List.of("-cp", "-p", "-classpath", "--module-path"); + ListIterator bundleArgsIterator = bundleArgs.listIterator(); + while (bundleArgsIterator.hasNext()) { + String arg = bundleArgsIterator.next(); + if (isPathArg.contains(arg)) { + bundleArgsIterator.remove(); + bundleArgsIterator.next(); + bundleArgsIterator.remove(); + } + } + /* Printing as list with defined sort-order ensures useful diffs are possible */ + JsonPrinter.printCollection(writer, bundleArgs, null, BundleSupport::printBuildArg); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + runArgsFile, e); + } + bundleProperties.write(); Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); @@ -1054,11 +1099,12 @@ private Path writeBundle() { return bundleFilePath; } - private static Manifest createManifest() { + private Manifest createManifest() { Manifest mf = new Manifest(); Attributes attributes = mf.getMainAttributes(); attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); /* If we add run-bundle-as-java-application a launcher mainclass would be added here */ + attributes.put(Attributes.Name.MAIN_CLASS, BundleLauncher.class.getName()); return mf; } From 1690ddc0ca43b746e6ca4269fb006c39ea438976 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 23 May 2023 16:44:16 +0200 Subject: [PATCH 22/61] Add copyright header --- .../svm/driver/launcher/BundleLauncher.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index a11ba870d535..474c88919503 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ package com.oracle.svm.driver.launcher; import java.io.File; From b9678cae850e1d52743a4ab5c77d080f63700915 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 24 May 2023 10:52:42 +0200 Subject: [PATCH 23/61] Fix environment variable sanitation on containerized builds --- .../com/oracle/svm/driver/BundleSupport.java | 29 +------------------ .../com/oracle/svm/driver/NativeImage.java | 7 +---- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 962413899195..0e90246868dc 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -67,7 +67,6 @@ import com.oracle.svm.core.util.ExitStatus; import com.oracle.svm.driver.launcher.BundleLauncher; import org.graalvm.collections.EconomicMap; -import org.graalvm.collections.Pair; import org.graalvm.util.json.JSONParser; import org.graalvm.util.json.JSONParserException; @@ -136,7 +135,6 @@ final class BundleSupport { private static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; private static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; private static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; - final Map containerEnvironment = new HashMap<>(); enum BundleOptionVariants { @@ -362,7 +360,7 @@ private void initializeContainerImage() { int exitStatusCode = createContainer(); switch (ExitStatus.of(exitStatusCode)) { - case OK -> fetchContainerEnvironment(); + case OK -> {} case BUILDER_ERROR -> /* Exit, builder has handled error reporting. */ throw NativeImage.showError(null, null, exitStatusCode); @@ -378,31 +376,6 @@ private void initializeContainerImage() { } } - private void fetchContainerEnvironment() { - ProcessBuilder pb = new ProcessBuilder(containerTool, "run", "--rm", containerImage, "bash", "-c", "/usr/bin/env"); - Process p = null; - try { - p = pb.start(); - p.waitFor(); - try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - processResult.lines() - .toList() - .forEach(env -> { - String[] envParts = SubstrateUtil.split(env, "=", 2); - if (envParts.length == 2) { - containerEnvironment.put(envParts[0], envParts[1]); - } - }); - } - } catch (IOException | InterruptedException e) { - throw NativeImage.showError(e.getMessage()); - } finally { - if (p != null) { - p.destroy(); - } - } - } - private int createContainer() { ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 241e9990127c..fe7b5eaff63d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1576,12 +1576,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa command.add(NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + builderArgFile); ProcessBuilder pb = new ProcessBuilder(); pb.command(command); - Map environment; - if (useBundle() && bundleSupport.useContainer) { - environment = bundleSupport.containerEnvironment; - } else { - environment = pb.environment(); - } + Map environment = pb.environment(); String deprecatedSanitationKey = "NATIVE_IMAGE_DEPRECATED_BUILDER_SANITATION"; String deprecatedSanitationValue = System.getenv().getOrDefault(deprecatedSanitationKey, "false"); if (Boolean.parseBoolean(deprecatedSanitationValue)) { From 95929bec2e770fd1b24cee94595435cc66009bfb Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 24 May 2023 15:28:59 +0200 Subject: [PATCH 24/61] Copy ConfigurationParser and JSONParser and adjust for bundle launcher --- .../svm/driver/launcher/BundleLauncher.java | 235 +++------ .../launcher/configuration/ArgsParser.java | 44 ++ .../BundleConfigurationParser.java | 59 +++ .../configuration/EnvironmentParser.java | 57 +++ .../launcher/json/BundleJSONParser.java | 458 ++++++++++++++++++ .../json/BundleJSONParserException.java | 32 ++ .../com/oracle/svm/driver/BundleSupport.java | 43 +- 7 files changed, 750 insertions(+), 178 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParserException.java diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 474c88919503..1b0e0b932fc4 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.driver.launcher; +import com.oracle.svm.driver.launcher.configuration.ArgsParser; +import com.oracle.svm.driver.launcher.configuration.EnvironmentParser; + import java.io.File; import java.io.IOException; import java.io.Reader; @@ -34,8 +37,14 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Stream; @@ -60,16 +69,6 @@ public static void main(String[] args) { String bundleFilePath = BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath(); unpackBundle(Path.of(bundleFilePath)); - Path environmentFile = stageDir.resolve("environment.json"); - if (Files.isReadable(environmentFile)) { - try (Reader reader = Files.newBufferedReader(environmentFile)) { - //new EnvironmentParser(nativeImage.imageBuilderEnvironment).parseAndRegister(reader); - } catch (IOException e) { - throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); - } - } - - List classpath = new ArrayList<>(); if (Files.isDirectory(classPathDir)) { try (Stream walk = Files.walk(classPathDir, 1)) { @@ -105,16 +104,39 @@ public static void main(String[] args) { Path buildArgsFile = stageDir.resolve("run.json"); try (Reader reader = Files.newBufferedReader(buildArgsFile)) { - command.addAll(parseArray(readFully(reader))); + List argsFromFile = new ArrayList<>(); + new ArgsParser(argsFromFile).parseAndRegister(reader); + command.addAll(argsFromFile); } catch (IOException e) { throw new RuntimeException("Failed to read bundle-file " + buildArgsFile, e); } + ProcessBuilder pb = new ProcessBuilder(command); + + Path environmentFile = stageDir.resolve("environment.json"); + if (Files.isReadable(environmentFile)) { + try (Reader reader = Files.newBufferedReader(environmentFile)) { + Map launcherEnvironment = new HashMap<>(); + new EnvironmentParser(launcherEnvironment).parseAndRegister(reader); + sanitizeJVMEnvironment(pb.environment(), launcherEnvironment); + } catch (IOException e) { + throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); + } + } + if (System.getenv("BUNDLE_LAUNCHER_VERBOSE") != null) { - System.out.println("Exec: " + String.join(" ", command)); + List environmentList = pb.environment() + .entrySet() + .stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .sorted() + .toList(); + System.out.println("Executing ["); + System.out.println(String.join(" \\\n", environmentList)); + System.out.println(String.join(" \\\n", command)); + System.out.println("]"); } - ProcessBuilder pb = new ProcessBuilder(command); Process p = null; try { p = pb.inheritIO().start(); @@ -128,6 +150,50 @@ public static void main(String[] args) { } } + private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { + Set requiredKeys = new HashSet<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); + requiredKeys.add("SRCHOME"); /* Remove once GR-44676 is fixed */ + Function keyMapper; + if (System.getProperty("os.name").contains("Windows")) { + requiredKeys.addAll(List.of("TEMP", "INCLUDE", "LIB")); + keyMapper = String::toUpperCase; + } else { + keyMapper = Function.identity(); + } + Map restrictedEnvironment = new HashMap<>(); + environment.forEach((key, val) -> { + if (requiredKeys.contains(keyMapper.apply(key))) { + restrictedEnvironment.put(key, val); + } + }); + for (Iterator> iterator = imageBuilderEnvironment.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + if (entry.getValue() != null) { + restrictedEnvironment.put(entry.getKey(), entry.getValue()); + } else { + environment.forEach((key, val) -> { + if (keyMapper.apply(key).equals(keyMapper.apply(entry.getKey()))) { + /* + * Record key as it was given by -E (by using `entry.getKey()` + * instead of `key`) to allow creating bundles on Windows that will also + * work on Linux. `System.getEnv(val)` is case-insensitive on Windows but + * not on Linux. + */ + restrictedEnvironment.put(entry.getKey(), val); + /* Capture found value for storing vars in bundle */ + entry.setValue(val); + } + }); + if (entry.getValue() == null) { + /* Remove undefined environment for storing vars in bundle */ + iterator.remove(); + } + } + } + environment.clear(); + environment.putAll(restrictedEnvironment); + } + private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); @@ -165,149 +231,6 @@ private static void showError(String s) { System.exit(1); } - private static String readFully(final Reader reader) throws IOException { - final char[] arr = new char[1024]; - final StringBuilder sb = new StringBuilder(); - - try { - int numChars; - while ((numChars = reader.read(arr, 0, arr.length)) > 0) { - sb.append(arr, 0, numChars); - } - } finally { - reader.close(); - } - - return sb.toString(); - } - - - private static final int STATE_EMPTY = 0; - private static final int STATE_ELEMENT_PARSED = 1; - private static final int STATE_COMMA_PARSED = 2; - - private static int pos; - - private static List parseArray(String jsonArray) { - List result = new ArrayList<>(); - int state = STATE_EMPTY; - pos = 0; - - skipWhiteSpace(jsonArray); - if(jsonArray.charAt(pos) != '[') { - throw new RuntimeException("Expected [ but found " + jsonArray.charAt(pos)); - } - pos++; - - while (pos < jsonArray.length()) { - pos = skipWhiteSpace(jsonArray); - - switch (jsonArray.charAt(pos)) { - case ',' -> { - if (state != STATE_ELEMENT_PARSED) { - throw new RuntimeException("Trailing comma is not allowed in JSON"); - } - state = STATE_COMMA_PARSED; - pos++; - } - case ']' -> { - if (state == STATE_COMMA_PARSED) { - throw new RuntimeException("Trailing comma is not allowed in JSON"); - } - return result; - } - default -> { - if (state == STATE_ELEMENT_PARSED) { - throw new RuntimeException("Expected , or ] but found " + jsonArray.charAt(pos)); - } - result.add(parseString(jsonArray)); - state = STATE_ELEMENT_PARSED; - } - } - } - - throw new RuntimeException("Expected , or ] but found eof"); - } - - private static String parseString(String json) { - // String buffer is only instantiated if string contains escape sequences. - int start = ++pos; - StringBuilder sb = null; - - while (pos < json.length()) { - final int c = json.charAt(pos); - pos++; - if (c <= 0x1f) { - // Characters < 0x1f are not allowed in JSON strings. - throw new RuntimeException("String contains control character"); - - } else if (c == '\\') { - if (sb == null) { - sb = new StringBuilder(pos - start + 16); - } - sb.append(json, start, pos - 1); - sb.append(parseEscapeSequence(json)); - start = pos; - - } else if (c == '"') { - if (sb != null) { - sb.append(json, start, pos - 1); - return sb.toString(); - } - return json.substring(start, pos - 1); - } - } - - throw new RuntimeException("Missing close quote"); - } - - private static char parseEscapeSequence(String json) { - final int c = json.charAt(pos); - pos++; - return switch (c) { - case '"' -> '"'; - case '\\' -> '\\'; - case '/' -> '/'; - case 'b' -> '\b'; - case 'f' -> '\f'; - case 'n' -> '\n'; - case 'r' -> '\r'; - case 't' -> '\t'; - case 'u' -> parseUnicodeEscape(json); - default -> throw new RuntimeException("Invalid escape character"); - }; - } - - private static char parseUnicodeEscape(String json) { - return (char) (parseHexDigit(json) << 12 | parseHexDigit(json) << 8 | parseHexDigit(json) << 4 | parseHexDigit(json)); - } - - private static int parseHexDigit(String json) { - final int c = json.charAt(pos); - pos++; - if (c >= '0' && c <= '9') { - return c - '0'; - } else if (c >= 'A' && c <= 'F') { - return c + 10 - 'A'; - } else if (c >= 'a' && c <= 'f') { - return c + 10 - 'a'; - } - throw new RuntimeException("Invalid hex digit"); - } - - private static int skipWhiteSpace(String str) { - while (pos < str.length()) { - switch (str.charAt(pos)) { - case '\t', '\r', '\n', ' ' -> pos++; - default -> { - return pos; - } - } - } - - return pos; - } - private static final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); private static Path createBundleRootDir() throws IOException { Path bundleRoot = Files.createTempDirectory(BUNDLE_TEMP_DIR_PREFIX); diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java new file mode 100644 index 000000000000..4d27b90c0ab1 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.configuration; + +import java.net.URI; +import java.util.List; + +public class ArgsParser extends BundleConfigurationParser { + + private final List args; + + public ArgsParser(List args) { + this.args = args; + } + + @Override + public void parseAndRegister(Object json, URI origin) { + for (var arg : asList(json, "Expected a list of arguments")) { + args.add(arg.toString()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java new file mode 100644 index 000000000000..15a4594ba5c2 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.configuration; + +import com.oracle.svm.driver.launcher.json.BundleJSONParser; +import com.oracle.svm.driver.launcher.json.BundleJSONParserException; + +import java.io.IOException; +import java.io.Reader; +import java.net.URI; +import java.util.List; +import java.util.Map; + +public abstract class BundleConfigurationParser { + + public void parseAndRegister(Reader reader) throws IOException { + parseAndRegister(new BundleJSONParser(reader).parse(), null); + } + + public abstract void parseAndRegister(Object json, URI origin) throws IOException; + + @SuppressWarnings("unchecked") + public static List asList(Object data, String errorMessage) { + if (data instanceof List) { + return (List) data; + } + throw new BundleJSONParserException(errorMessage); + } + + @SuppressWarnings("unchecked") + public static Map asMap(Object data, String errorMessage) { + if (data instanceof Map) { + return (Map) data; + } + throw new BundleJSONParserException(errorMessage); + } +} diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java new file mode 100644 index 000000000000..949012704bde --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.configuration; + +import com.oracle.svm.driver.launcher.json.BundleJSONParserException; + +import java.net.URI; +import java.util.Map; + +public class EnvironmentParser extends BundleConfigurationParser { + private static final String environmentKeyField = "key"; + private static final String environmentValueField = "val"; + private final Map environment; + + public EnvironmentParser(Map environment) { + environment.clear(); + this.environment = environment; + } + + @Override + public void parseAndRegister(Object json, URI origin) { + for (var rawEntry : asList(json, "Expected a list of environment variable objects")) { + var entry = asMap(rawEntry, "Expected a environment variable object"); + Object envVarKeyString = entry.get(environmentKeyField); + if (envVarKeyString == null) { + throw new BundleJSONParserException("Expected " + environmentKeyField + "-field in environment variable object"); + } + Object envVarValueString = entry.get(environmentValueField); + if (envVarValueString == null) { + throw new BundleJSONParserException("Expected " + environmentValueField + "-field in environment variable object"); + } + environment.put(envVarKeyString.toString(), envVarValueString.toString()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java new file mode 100644 index 000000000000..ca7534149448 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.json; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class BundleJSONParser { + + private final String source; + private final int length; + private int pos = 0; + + private static final int EOF = -1; + + private static final String TRUE = "true"; + private static final String FALSE = "false"; + private static final String NULL = "null"; + + private static final int STATE_EMPTY = 0; + private static final int STATE_ELEMENT_PARSED = 1; + private static final int STATE_COMMA_PARSED = 2; + + public BundleJSONParser(String source) { + this.source = source; + this.length = source.length(); + } + + public BundleJSONParser(Reader source) throws IOException { + this(readFully(source)); + } + + /** + * Public parse method. Parse a string into a JSON object. + * + * @return the parsed JSON Object + */ + public Object parse() { + final Object value = parseLiteral(); + skipWhiteSpace(); + if (pos < length) { + throw expectedError(pos, "eof", toString(peek())); + } + return value; + } + + private Object parseLiteral() { + skipWhiteSpace(); + + final int c = peek(); + if (c == EOF) { + throw expectedError(pos, "json literal", "eof"); + } + return switch (c) { + case '{' -> parseObject(); + case '[' -> parseArray(); + case '"' -> parseString(); + case 'f' -> parseKeyword(FALSE, Boolean.FALSE); + case 't' -> parseKeyword(TRUE, Boolean.TRUE); + case 'n' -> parseKeyword(NULL, null); + default -> { + if (isDigit(c) || c == '-') { + yield parseNumber(); + } else if (c == '.') { + throw numberError(pos); + } else { + throw expectedError(pos, "json literal", toString(c)); + } + } + }; + } + + private Object parseObject() { + Map result = new HashMap<>(); + int state = STATE_EMPTY; + + assert peek() == '{'; + pos++; + + while (pos < length) { + skipWhiteSpace(); + final int c = peek(); + + switch (c) { + case '"' -> { + if (state == STATE_ELEMENT_PARSED) { + throw expectedError(pos, ", or }", toString(c)); + } + final String id = parseString(); + expectColon(); + final Object value = parseLiteral(); + result.put(id, value); + state = STATE_ELEMENT_PARSED; + } + case ',' -> { + if (state != STATE_ELEMENT_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); + } + state = STATE_COMMA_PARSED; + pos++; + } + case '}' -> { + if (state == STATE_COMMA_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); + } + pos++; + return result; + } + default -> throw expectedError(pos, ", or }", toString(c)); + } + } + throw expectedError(pos, ", or }", "eof"); + } + + private void expectColon() { + skipWhiteSpace(); + final int n = next(); + if (n != ':') { + throw expectedError(pos - 1, ":", toString(n)); + } + } + + private Object parseArray() { + List result = new ArrayList<>(); + int state = STATE_EMPTY; + + assert peek() == '['; + pos++; + + while (pos < length) { + skipWhiteSpace(); + final int c = peek(); + + switch (c) { + case ',' -> { + if (state != STATE_ELEMENT_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); + } + state = STATE_COMMA_PARSED; + pos++; + } + case ']' -> { + if (state == STATE_COMMA_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); + } + pos++; + return result; + } + default -> { + if (state == STATE_ELEMENT_PARSED) { + throw expectedError(pos, ", or ]", toString(c)); + } + result.add(parseLiteral()); + state = STATE_ELEMENT_PARSED; + } + } + } + + throw expectedError(pos, ", or ]", "eof"); + } + + private String parseString() { + // String buffer is only instantiated if string contains escape sequences. + int start = ++pos; + StringBuilder sb = null; + + while (pos < length) { + final int c = next(); + if (c <= 0x1f) { + // Characters < 0x1f are not allowed in JSON strings. + throw syntaxError(pos, "String contains control character"); + + } else if (c == '\\') { + if (sb == null) { + sb = new StringBuilder(pos - start + 16); + } + sb.append(source, start, pos - 1); + sb.append(parseEscapeSequence()); + start = pos; + + } else if (c == '"') { + if (sb != null) { + sb.append(source, start, pos - 1); + return sb.toString(); + } + return source.substring(start, pos - 1); + } + } + + throw error("Missing close quote", pos); + } + + private char parseEscapeSequence() { + final int c = next(); + return switch (c) { + case '"' -> '"'; + case '\\' -> '\\'; + case '/' -> '/'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'u' -> parseUnicodeEscape(); + default -> throw error("Invalid escape character", pos - 1); + }; + } + + private char parseUnicodeEscape() { + return (char) (parseHexDigit() << 12 | parseHexDigit() << 8 | parseHexDigit() << 4 | parseHexDigit()); + } + + private int parseHexDigit() { + final int c = next(); + if (c >= '0' && c <= '9') { + return c - '0'; + } else if (c >= 'A' && c <= 'F') { + return c + 10 - 'A'; + } else if (c >= 'a' && c <= 'f') { + return c + 10 - 'a'; + } + throw error("Invalid hex digit", pos - 1); + } + + private static boolean isDigit(final int c) { + return c >= '0' && c <= '9'; + } + + private void skipDigits() { + while (pos < length) { + final int c = peek(); + if (!isDigit(c)) { + break; + } + pos++; + } + } + + private Number parseNumber() { + boolean isFloating = false; + final int start = pos; + int c = next(); + + if (c == '-') { + c = next(); + } + if (!isDigit(c)) { + throw numberError(start); + } + // no more digits allowed after 0 + if (c != '0') { + skipDigits(); + } + + // fraction + if (peek() == '.') { + isFloating = true; + pos++; + if (!isDigit(next())) { + throw numberError(pos - 1); + } + skipDigits(); + } + + // exponent + c = peek(); + if (c == 'e' || c == 'E') { + pos++; + c = next(); + if (c == '-' || c == '+') { + c = next(); + } + if (!isDigit(c)) { + throw numberError(pos - 1); + } + skipDigits(); + } + + String literalValue = source.substring(start, pos); + if (isFloating) { + return Double.parseDouble(literalValue); + } else { + final long l = Long.parseLong(literalValue); + if ((int) l == l) { + return (int) l; + } else { + return l; + } + } + } + + private Object parseKeyword(final String keyword, final Object value) { + if (!source.regionMatches(pos, keyword, 0, keyword.length())) { + throw expectedError(pos, "json literal", "ident"); + } + pos += keyword.length(); + return value; + } + + private int peek() { + if (pos >= length) { + return -1; + } + return source.charAt(pos); + } + + private int next() { + final int next = peek(); + pos++; + return next; + } + + private void skipWhiteSpace() { + while (pos < length) { + switch (peek()) { + case '\t', '\r', '\n', ' ' -> pos++; + default -> { + return; + } + } + } + } + + private static String toString(final int c) { + return c == EOF ? "eof" : String.valueOf((char) c); + } + + private BundleJSONParserException error(final String message, final int start) { + final int lineNum = getLine(start); + final int columnNum = getColumn(start); + final String formatted = format(message, lineNum, columnNum); + return new BundleJSONParserException(formatted); + } + + /** + * Return line number of character position. + * + *

+ * This method can be expensive for large sources as it iterates through all characters up to + * {@code position}. + *

+ * + * @param position Position of character in source content. + * @return Line number. + */ + private int getLine(final int position) { + final CharSequence d = source; + // Line count starts at 1. + int line = 1; + + for (int i = 0; i < position; i++) { + final char ch = d.charAt(i); + // Works for both \n and \r\n. + if (ch == '\n') { + line++; + } + } + + return line; + } + + /** + * Return column number of character position. + * + * @param position Position of character in source content. + * @return Column number. + */ + private int getColumn(final int position) { + return position - findBOLN(position); + } + + /** + * Find the beginning of the line containing position. + * + * @param position Index to offending token. + * @return Index of first character of line. + */ + private int findBOLN(final int position) { + final CharSequence d = source; + for (int i = position - 1; i > 0; i--) { + final char ch = d.charAt(i); + + if (ch == '\n' || ch == '\r') { + return i + 1; + } + } + + return 0; + } + + /** + * Format an error message to include source and line information. + * + * @param message Error message string. + * @param line Source line number. + * @param column Source column number. + * @return formatted string + */ + private static String format(final String message, final int line, final int column) { + return "line " + line + " column " + column + " " + message; + } + + private BundleJSONParserException numberError(final int start) { + return error("Invalid JSON number format", start); + } + + private BundleJSONParserException expectedError(final int start, final String expected, final String found) { + return error("Expected " + expected + " but found " + found, start); + } + + private BundleJSONParserException syntaxError(final int start, final String reason) { + return error("Invalid JSON: " + reason, start); + } + + /** + * Utility function to read all contents of a {@link Reader}, because the JSON parser does not + * support streaming yet. + */ + private static String readFully(final Reader reader) throws IOException { + final char[] arr = new char[1024]; + final StringBuilder sb = new StringBuilder(); + + try (reader) { + int numChars; + while ((numChars = reader.read(arr, 0, arr.length)) > 0) { + sb.append(arr, 0, numChars); + } + } + + return sb.toString(); + } +} diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParserException.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParserException.java new file mode 100644 index 000000000000..a1d00816b176 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParserException.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.json; + +@SuppressWarnings("serial") +public final class BundleJSONParserException extends RuntimeException { + public BundleJSONParserException(final String msg) { + super(msg); + } +} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 0e90246868dc..861454db2fb4 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -927,30 +927,29 @@ private Path writeBundle() { nativeImage.deleteAllFiles(metaInfDir); } - Path launcherFilePath = rootDir.resolve(BundleLauncher.class.getName().replace(".","/") + ".class"); - - try { - Files.createDirectories(launcherFilePath.getParent()); - Files.copy(BundleLauncher.class.getResourceAsStream(BundleLauncher.class.getSimpleName() + ".class"), launcherFilePath); - } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + launcherFilePath, e); - } - - /* - try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(launcherFilePath))) { - //String jarEntryName = BundleLauncher.class.getName().replace(".","/") + ".class"; - String jarEntryName = BundleLauncher.class.getSimpleName() + ".class"; - JarEntry entry = new JarEntry(jarEntryName); - jarOutStream.putNextEntry(entry); - Path tempLauncherFile = Files.createTempFile(BundleLauncher.class.getSimpleName(), null); - Files.copy(BundleLauncher.class.getResourceAsStream(BundleLauncher.class.getSimpleName() + ".class"), tempLauncherFile, StandardCopyOption.REPLACE_EXISTING); - Files.copy(tempLauncherFile, jarOutStream); - Files.delete(tempLauncherFile); - jarOutStream.closeEntry(); + Path launcherPackagePath = rootDir.resolve(BundleLauncher.class.getPackageName().replace(".", "/")); + String driverJarPath = BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + + try (JarFile archive = new JarFile(driverJarPath)) { + Enumeration jarEntries = archive.entries(); + while (jarEntries.hasMoreElements() && !deleteBundleRoot.get()) { + JarEntry jarEntry = jarEntries.nextElement(); + Path bundleEntry = rootDir.resolve(jarEntry.getName()); + if (bundleEntry.startsWith(launcherPackagePath)) { + try { + Path bundleFileParent = bundleEntry.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); + } + Files.copy(archive.getInputStream(jarEntry), bundleEntry); + } catch (IOException e) { + throw new RuntimeException("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); + } + } + } } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + launcherFilePath, e); + throw NativeImage.showError("Failed to write bundle-files for bundle launcher " + launcherPackagePath, e); } - */ Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (JsonWriter writer = new JsonWriter(pathCanonicalizationsFile)) { From 4b3a02f67dc2137f5d6fa80205cc3db3ee5829c3 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 25 May 2023 13:50:58 +0200 Subject: [PATCH 25/61] Parse bundle launcher commandline arguments --- .../svm/driver/launcher/BundleLauncher.java | 61 +++++++++++++------ .../com/oracle/svm/driver/BundleSupport.java | 2 +- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 1b0e0b932fc4..235c8d317f61 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -34,8 +34,11 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; +import java.util.Deque; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -56,7 +59,6 @@ public class BundleLauncher { private static Path rootDir; private static Path inputDir; private static Path stageDir; - private static Path auxiliaryDir; private static Path classPathDir; private static Path modulePathDir; @@ -66,6 +68,9 @@ public static void main(String[] args) { Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); command.add(javaExecutable.toString()); + List programArgs = parseBundleLauncherOptions(args); + command.addAll(programArgs); + String bundleFilePath = BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath(); unpackBundle(Path.of(bundleFilePath)); @@ -96,7 +101,7 @@ public static void main(String[] args) { throw new RuntimeException("Failed to iterate through directory " + modulePathDir, e); } - if(!modulePath.isEmpty()) { + if (!modulePath.isEmpty()) { command.add("-p"); command.add(String.join(File.pathSeparator, modulePath)); } @@ -142,14 +147,41 @@ public static void main(String[] args) { p = pb.inheritIO().start(); p.waitFor(); } catch (IOException | InterruptedException e) { - showError("Failed to run bundled application"); + throw new RuntimeException("Failed to run bundled application"); } finally { - if(p != null) { + if (p != null) { p.destroy(); } } } + private static List parseBundleLauncherOptions(String[] args) { + Deque argQueue = new ArrayDeque<>(Arrays.asList(args)); + List programArgs = new ArrayList<>(); + + while (!argQueue.isEmpty()) { + String arg = argQueue.removeFirst(); + + if (arg.startsWith("--with-native-image-agent")) { + StringBuilder agentArgument = new StringBuilder("-agentlib:native-image-agent="); + String[] parts = arg.split("=", 2); + if (parts.length == 1) { + agentArgument.append("config-output-dir=native-image-config"); + } else { + agentArgument.append(parts[1]); + } + programArgs.add(agentArgument.toString()); + } else if (arg.equals("--")) { + programArgs.addAll(argQueue); + argQueue.clear(); + } else { + programArgs.add(arg); + } + } + + return programArgs; + } + private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { Set requiredKeys = new HashSet<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); requiredKeys.add("SRCHOME"); /* Remove once GR-44676 is fixed */ @@ -212,7 +244,7 @@ private static Path getJavaExecutable() { String javaHome = System.getenv("JAVA_HOME"); if (javaHome == null) { - showError("No " + binJava + " and no environment variable JAVA_HOME"); + throw new RuntimeException("No " + binJava + " and no environment variable JAVA_HOME"); } try { javaCandidate = Paths.get(javaHome).resolve(binJava); @@ -222,13 +254,7 @@ private static Path getJavaExecutable() { } catch (InvalidPathException e) { /* fallthrough */ } - showError("No " + binJava + " and invalid JAVA_HOME=" + javaHome); - return null; - } - - private static void showError(String s) { - System.err.println("Error: " + s); - System.exit(1); + throw new RuntimeException("No " + binJava + " and invalid JAVA_HOME=" + javaHome); } private static final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); @@ -254,12 +280,12 @@ private static void deleteAllFiles(Path toDelete) { deletedPath = toDelete.resolveSibling(toDelete.getFileName() + deletedFileSuffix); Files.move(toDelete, deletedPath); } - Files.walk(deletedPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + try (Stream walk = Files.walk(deletedPath)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } } catch (IOException e) { - //if (isVerbose()) { - System.out.println("Could not recursively delete path: " + toDelete); - e.printStackTrace(); - //} + System.out.println("Could not recursively delete path: " + toDelete); + e.printStackTrace(); } } @@ -299,7 +325,6 @@ private static void unpackBundle(Path bundleFilePath) { try { inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); - auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); Path classesDir = inputDir.resolve("classes"); classPathDir = Files.createDirectories(classesDir.resolve("cp")); modulePathDir = Files.createDirectories(classesDir.resolve("p")); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 861454db2fb4..df71a11bb479 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -360,7 +360,7 @@ private void initializeContainerImage() { int exitStatusCode = createContainer(); switch (ExitStatus.of(exitStatusCode)) { - case OK -> {} + case OK -> { } case BUILDER_ERROR -> /* Exit, builder has handled error reporting. */ throw NativeImage.showError(null, null, exitStatusCode); From 906ce7583e21209115eb075faf156ceaba1fe785 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 30 May 2023 17:23:12 +0200 Subject: [PATCH 26/61] Fix crash on containerized dry-runs --- .../src/com/oracle/svm/driver/NativeImage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index fe7b5eaff63d..37a6a1bbe256 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1602,7 +1602,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa List completeCommandList = new ArrayList<>(); completeCommandList.addAll(environment.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).sorted().toList()); - if (useBundle() && bundleSupport.useContainer) { + if (!isDryRun() && useBundle() && bundleSupport.useContainer) { completeCommandList.addAll(bundleSupport.createContainerCommand(argFile, builderArgFile)); } completeCommandList.add(javaExecutable); From e81c0e28edc0f97f8bb9ef81d02ad05ad7f2c099 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 30 May 2023 17:27:01 +0200 Subject: [PATCH 27/61] Stricter checks for run json and add main class if no executable was given --- .../src/com/oracle/svm/driver/launcher/BundleLauncher.java | 3 +-- .../src/com/oracle/svm/driver/BundleSupport.java | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 235c8d317f61..5723bc3c6cbb 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -77,14 +77,13 @@ public static void main(String[] args) { List classpath = new ArrayList<>(); if (Files.isDirectory(classPathDir)) { try (Stream walk = Files.walk(classPathDir, 1)) { - walk.filter(path -> path.toString().endsWith(".jar") || (Files.isDirectory(path) && !path.equals(classPathDir))) + walk.filter(path -> path.toString().endsWith(".jar") || Files.isDirectory(path)) .map(Path::toString) .forEach(classpath::add); } catch (IOException e) { throw new RuntimeException("Failed to iterate through directory " + classPathDir, e); } - classpath.add(classPathDir.toString()); command.add("-cp"); command.add(String.join(File.pathSeparator, classpath)); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index df71a11bb479..1f8b5abad829 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -1038,8 +1038,15 @@ private Path writeBundle() { bundleArgsIterator.remove(); bundleArgsIterator.next(); bundleArgsIterator.remove(); + } else if (arg.equals("-jar")) { + bundleArgsIterator.next(); + } else if (arg.startsWith("-") || arg.equals(nativeImage.imageName)) { + bundleArgsIterator.remove(); } } + if (bundleArgs.isEmpty()) { + bundleArgs.add(nativeImage.mainClass); + } /* Printing as list with defined sort-order ensures useful diffs are possible */ JsonPrinter.printCollection(writer, bundleArgs, null, BundleSupport::printBuildArg); } catch (IOException e) { From 99dff39929f2412181e9b197fb5cb43ea3e28851 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 1 Jun 2023 12:54:24 +0200 Subject: [PATCH 28/61] Fix bundle launcher environment loading, make bundle launcher with native-image-agent work --- .../svm/driver/launcher/BundleLauncher.java | 83 +++++-------------- 1 file changed, 21 insertions(+), 62 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 5723bc3c6cbb..a808ba4e7545 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -41,13 +41,9 @@ import java.util.Deque; import java.util.Enumeration; import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Stream; @@ -55,25 +51,29 @@ public class BundleLauncher { private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; + static final String BUNDLE_FILE_EXTENSION = ".nib"; + private static final String AUXILIARY_OUTPUT_DIR_NAME = "other"; + private static final String OUTPUT_DIR_NAME = "output"; - private static Path rootDir; - private static Path inputDir; private static Path stageDir; private static Path classPathDir; private static Path modulePathDir; + private static Path bundleFilePath; + public static void main(String[] args) { List command = new ArrayList<>(); + + bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); + unpackBundle(bundleFilePath); + Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); command.add(javaExecutable.toString()); List programArgs = parseBundleLauncherOptions(args); command.addAll(programArgs); - String bundleFilePath = BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - unpackBundle(Path.of(bundleFilePath)); - List classpath = new ArrayList<>(); if (Files.isDirectory(classPathDir)) { try (Stream walk = Files.walk(classPathDir, 1)) { @@ -122,7 +122,7 @@ public static void main(String[] args) { try (Reader reader = Files.newBufferedReader(environmentFile)) { Map launcherEnvironment = new HashMap<>(); new EnvironmentParser(launcherEnvironment).parseAndRegister(reader); - sanitizeJVMEnvironment(pb.environment(), launcherEnvironment); + pb.environment().putAll(launcherEnvironment); } catch (IOException e) { throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); } @@ -162,14 +162,16 @@ private static List parseBundleLauncherOptions(String[] args) { String arg = argQueue.removeFirst(); if (arg.startsWith("--with-native-image-agent")) { - StringBuilder agentArgument = new StringBuilder("-agentlib:native-image-agent="); - String[] parts = arg.split("=", 2); - if (parts.length == 1) { - agentArgument.append("config-output-dir=native-image-config"); - } else { - agentArgument.append(parts[1]); + String bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); + Path bundlePath = bundleFilePath.getParent(); + Path externalAuxiliaryOutputDir = bundlePath.resolve(bundleName + "." + OUTPUT_DIR_NAME).resolve(AUXILIARY_OUTPUT_DIR_NAME); + try { + Files.createDirectories(externalAuxiliaryOutputDir); + System.out.println("Native image agent output written to " + externalAuxiliaryOutputDir); + programArgs.add("-agentlib:native-image-agent=config-output-dir=" + externalAuxiliaryOutputDir); + } catch (IOException e) { + System.out.println("Failed to create native image agent output dir"); } - programArgs.add(agentArgument.toString()); } else if (arg.equals("--")) { programArgs.addAll(argQueue); argQueue.clear(); @@ -181,51 +183,6 @@ private static List parseBundleLauncherOptions(String[] args) { return programArgs; } - private static void sanitizeJVMEnvironment(Map environment, Map imageBuilderEnvironment) { - Set requiredKeys = new HashSet<>(List.of("PATH", "PWD", "HOME", "LANG", "LC_ALL")); - requiredKeys.add("SRCHOME"); /* Remove once GR-44676 is fixed */ - Function keyMapper; - if (System.getProperty("os.name").contains("Windows")) { - requiredKeys.addAll(List.of("TEMP", "INCLUDE", "LIB")); - keyMapper = String::toUpperCase; - } else { - keyMapper = Function.identity(); - } - Map restrictedEnvironment = new HashMap<>(); - environment.forEach((key, val) -> { - if (requiredKeys.contains(keyMapper.apply(key))) { - restrictedEnvironment.put(key, val); - } - }); - for (Iterator> iterator = imageBuilderEnvironment.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - if (entry.getValue() != null) { - restrictedEnvironment.put(entry.getKey(), entry.getValue()); - } else { - environment.forEach((key, val) -> { - if (keyMapper.apply(key).equals(keyMapper.apply(entry.getKey()))) { - /* - * Record key as it was given by -E (by using `entry.getKey()` - * instead of `key`) to allow creating bundles on Windows that will also - * work on Linux. `System.getEnv(val)` is case-insensitive on Windows but - * not on Linux. - */ - restrictedEnvironment.put(entry.getKey(), val); - /* Capture found value for storing vars in bundle */ - entry.setValue(val); - } - }); - if (entry.getValue() == null) { - /* Remove undefined environment for storing vars in bundle */ - iterator.remove(); - } - } - } - environment.clear(); - environment.putAll(restrictedEnvironment); - } - - private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); private static Path getJavaExecutable() { @@ -290,6 +247,8 @@ private static void deleteAllFiles(Path toDelete) { private static void unpackBundle(Path bundleFilePath) { + Path rootDir; + Path inputDir; try { rootDir = createBundleRootDir(); inputDir = rootDir.resolve("input"); From a9efd762f752fb82ae9059d37382e1f70441caa9 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 1 Jun 2023 12:55:16 +0200 Subject: [PATCH 29/61] Switch to allow list for run arguments --- .../com/oracle/svm/driver/BundleSupport.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 1f8b5abad829..9887cc2adb89 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -1030,25 +1030,21 @@ private Path writeBundle() { Path runArgsFile = stageDir.resolve("run.json"); try (JsonWriter writer = new JsonWriter(runArgsFile)) { - List isPathArg = List.of("-cp", "-p", "-classpath", "--module-path"); + List equalsRunArg = List.of("-jar", "-m", "--module"); + List runArgs = new ArrayList<>(); ListIterator bundleArgsIterator = bundleArgs.listIterator(); while (bundleArgsIterator.hasNext()) { String arg = bundleArgsIterator.next(); - if (isPathArg.contains(arg)) { - bundleArgsIterator.remove(); - bundleArgsIterator.next(); - bundleArgsIterator.remove(); - } else if (arg.equals("-jar")) { - bundleArgsIterator.next(); - } else if (arg.startsWith("-") || arg.equals(nativeImage.imageName)) { - bundleArgsIterator.remove(); + if (equalsRunArg.contains(arg)) { + runArgs.add(arg); + runArgs.add(bundleArgsIterator.next()); } } - if (bundleArgs.isEmpty()) { - bundleArgs.add(nativeImage.mainClass); + if (runArgs.isEmpty()) { + runArgs.add(nativeImage.mainClass); } /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, bundleArgs, null, BundleSupport::printBuildArg); + JsonPrinter.printCollection(writer, runArgs, null, BundleSupport::printBuildArg); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + runArgsFile, e); } From f8315090c2dba077fd3371f2c5ff175604df1ffa Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 7 Jun 2023 09:35:22 +0200 Subject: [PATCH 30/61] Copy launcher class files from resources to bundle --- substratevm/mx.substratevm/mx_substratevm.py | 4 +- .../com/oracle/svm/driver/BundleSupport.java | 43 ++++++++++--------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 09807772bfa0..1a9bd6563e92 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -1056,7 +1056,9 @@ def _native_image_launcher_extra_jvm_args(): destination="bin/", jar_distributions=["substratevm:SVM_DRIVER"], main_class=_native_image_launcher_main_class(), - build_args=driver_build_args, + build_args=driver_build_args + [ + '-H:IncludeResources=com/oracle/svm/driver/launcher/.*', + ], extra_jvm_args=_native_image_launcher_extra_jvm_args(), home_finder=False, ), diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 9887cc2adb89..4295844707fd 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -27,14 +27,17 @@ import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.net.URI; import java.nio.file.CopyOption; +import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -927,28 +930,26 @@ private Path writeBundle() { nativeImage.deleteAllFiles(metaInfDir); } - Path launcherPackagePath = rootDir.resolve(BundleLauncher.class.getPackageName().replace(".", "/")); - String driverJarPath = BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - try (JarFile archive = new JarFile(driverJarPath)) { - Enumeration jarEntries = archive.entries(); - while (jarEntries.hasMoreElements() && !deleteBundleRoot.get()) { - JarEntry jarEntry = jarEntries.nextElement(); - Path bundleEntry = rootDir.resolve(jarEntry.getName()); - if (bundleEntry.startsWith(launcherPackagePath)) { - try { - Path bundleFileParent = bundleEntry.getParent(); - if (bundleFileParent != null) { - Files.createDirectories(bundleFileParent); + Path bundleLauncherFile = Paths.get("/").resolve(BundleLauncher.class.getName().replace(".", "/") + ".class"); + try (FileSystem fs = FileSystems.newFileSystem(BundleSupport.class.getResource(bundleLauncherFile.toString()).toURI(), new HashMap<>()); + Stream walk = Files.walk(fs.getPath(bundleLauncherFile.getParent().toString())) + ) { + walk.filter(Predicate.not(Files::isDirectory)) + .map(Path::toString) + .forEach(sourcePath -> { + Path target = rootDir.resolve(Paths.get("/").relativize(Paths.get(sourcePath))); + try (InputStream source = BundleSupport.class.getResourceAsStream(sourcePath)) { + Path bundleFileParent = target.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); + } + Files.copy(source, target); + } catch (Exception e) { + throw NativeImage.showError("Failed to write bundle-file " + target, e); } - Files.copy(archive.getInputStream(jarEntry), bundleEntry); - } catch (IOException e) { - throw new RuntimeException("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); - } - } - } - } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-files for bundle launcher " + launcherPackagePath, e); + }); + } catch (Exception e) { + throw NativeImage.showError("Failed to read bundle launcher resources '" + bundleLauncherFile.getParent() + "'", e); } Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); From 4da3052b787dfb8f7114cb7c5b22b77a76d3d25d Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 7 Jun 2023 12:01:23 +0200 Subject: [PATCH 31/61] Extract BundleSupport functionality to BundleLauncher --- .../svm/driver/launcher/BundleLauncher.java | 78 +++++---- ...{ArgsParser.java => BundleArgsParser.java} | 4 +- .../BundleContainerSettingsParser.java | 42 +++++ ...rser.java => BundleEnvironmentParser.java} | 4 +- .../configuration/BundlePathMapParser.java | 59 +++++++ .../com/oracle/svm/driver/BundleSupport.java | 153 +++++------------- 6 files changed, 185 insertions(+), 155 deletions(-) rename substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/{ArgsParser.java => BundleArgsParser.java} (93%) create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleContainerSettingsParser.java rename substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/{EnvironmentParser.java => BundleEnvironmentParser.java} (94%) create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index a808ba4e7545..4fd70bffc217 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -24,14 +24,13 @@ */ package com.oracle.svm.driver.launcher; -import com.oracle.svm.driver.launcher.configuration.ArgsParser; -import com.oracle.svm.driver.launcher.configuration.EnvironmentParser; +import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; +import com.oracle.svm.driver.launcher.configuration.BundleEnvironmentParser; import java.io.File; import java.io.IOException; import java.io.Reader; import java.nio.file.Files; -import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayDeque; @@ -50,10 +49,19 @@ public class BundleLauncher { - private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; - static final String BUNDLE_FILE_EXTENSION = ".nib"; - private static final String AUXILIARY_OUTPUT_DIR_NAME = "other"; - private static final String OUTPUT_DIR_NAME = "output"; + + public static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; + public static final String BUNDLE_FILE_EXTENSION = ".nib"; + + public static final String INPUT_DIR_NAME = "input"; + public static final String STAGE_DIR_NAME = "stage"; + public static final String AUXILIARY_DIR_NAME = "auxiliary"; + public static final String CLASSES_DIR_NAME = "classes"; + public static final String CLASSPATH_DIR_NAME = "cp"; + public static final String MODULE_PATH_DIR_NAME = "p"; + public static final String OUTPUT_DIR_NAME = "output"; + public static final String IMAGE_PATH_OUTPUT_DIR_NAME = "default"; + public static final String AUXILIARY_OUTPUT_DIR_NAME = "other"; private static Path stageDir; private static Path classPathDir; @@ -106,13 +114,13 @@ public static void main(String[] args) { } } - Path buildArgsFile = stageDir.resolve("run.json"); - try (Reader reader = Files.newBufferedReader(buildArgsFile)) { + Path argsFile = stageDir.resolve("run.json"); + try (Reader reader = Files.newBufferedReader(argsFile)) { List argsFromFile = new ArrayList<>(); - new ArgsParser(argsFromFile).parseAndRegister(reader); + new BundleArgsParser(argsFromFile).parseAndRegister(reader); command.addAll(argsFromFile); } catch (IOException e) { - throw new RuntimeException("Failed to read bundle-file " + buildArgsFile, e); + throw new RuntimeException("Failed to read bundle-file " + argsFile, e); } ProcessBuilder pb = new ProcessBuilder(command); @@ -121,7 +129,7 @@ public static void main(String[] args) { if (Files.isReadable(environmentFile)) { try (Reader reader = Files.newBufferedReader(environmentFile)) { Map launcherEnvironment = new HashMap<>(); - new EnvironmentParser(launcherEnvironment).parseAndRegister(reader); + new BundleEnvironmentParser(launcherEnvironment).parseAndRegister(reader); pb.environment().putAll(launcherEnvironment); } catch (IOException e) { throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); @@ -187,30 +195,22 @@ private static List parseBundleLauncherOptions(String[] args) { private static Path getJavaExecutable() { Path binJava = Paths.get("bin", System.getProperty("os.name").contains("Windows") ? "java.exe" : "java"); - - Path javaCandidate = buildTimeJavaHome.resolve(binJava); - if (Files.isExecutable(javaCandidate)) { - return javaCandidate; - } - - javaCandidate = Paths.get(".").resolve(binJava); - if (Files.isExecutable(javaCandidate)) { - return javaCandidate; + if (Files.isExecutable(buildTimeJavaHome.resolve(binJava))) { + return buildTimeJavaHome.resolve(binJava); } String javaHome = System.getenv("JAVA_HOME"); if (javaHome == null) { - throw new RuntimeException("No " + binJava + " and no environment variable JAVA_HOME"); + throw new RuntimeException("Environment variable JAVA_HOME is not set"); } - try { - javaCandidate = Paths.get(javaHome).resolve(binJava); - if (Files.isExecutable(javaCandidate)) { - return javaCandidate; - } - } catch (InvalidPathException e) { - /* fallthrough */ + Path javaHomeDir = Paths.get(javaHome); + if (!Files.isDirectory(javaHomeDir)) { + throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory"); + } + if (!Files.isExecutable(javaHomeDir.resolve(binJava))) { + throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory with a " + binJava + " executable"); } - throw new RuntimeException("No " + binJava + " and invalid JAVA_HOME=" + javaHome); + return javaHomeDir.resolve(binJava); } private static final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); @@ -223,12 +223,10 @@ private static Path createBundleRootDir() throws IOException { return bundleRoot; } - private static final String deletedFileSuffix = ".deleted"; private static boolean isDeletedPath(Path toDelete) { return toDelete.getFileName().toString().endsWith(deletedFileSuffix); } - private static void deleteAllFiles(Path toDelete) { try { Path deletedPath = toDelete; @@ -245,13 +243,12 @@ private static void deleteAllFiles(Path toDelete) { } } - private static void unpackBundle(Path bundleFilePath) { Path rootDir; Path inputDir; try { rootDir = createBundleRootDir(); - inputDir = rootDir.resolve("input"); + inputDir = rootDir.resolve(INPUT_DIR_NAME); try (JarFile archive = new JarFile(bundleFilePath.toFile())) { Enumeration jarEntries = archive.entries(); @@ -276,16 +273,15 @@ private static void unpackBundle(Path bundleFilePath) { } if (deleteBundleRoot.get()) { - /* Abort image build request without error message and exit with 0 */ - throw new RuntimeException(""); + /* Abort bundle run request without error message and exit with 0 */ + throw new Error(null, null); } try { - inputDir = rootDir.resolve("input"); - stageDir = Files.createDirectories(inputDir.resolve("stage")); - Path classesDir = inputDir.resolve("classes"); - classPathDir = Files.createDirectories(classesDir.resolve("cp")); - modulePathDir = Files.createDirectories(classesDir.resolve("p")); + stageDir = Files.createDirectories(inputDir.resolve(STAGE_DIR_NAME)); + Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); + classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); + modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); } catch (IOException e) { throw new RuntimeException("Unable to create bundle directory layout", e); } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleArgsParser.java similarity index 93% rename from substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java rename to substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleArgsParser.java index 4d27b90c0ab1..ef2450d144c9 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/ArgsParser.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleArgsParser.java @@ -27,11 +27,11 @@ import java.net.URI; import java.util.List; -public class ArgsParser extends BundleConfigurationParser { +public class BundleArgsParser extends BundleConfigurationParser { private final List args; - public ArgsParser(List args) { + public BundleArgsParser(List args) { this.args = args; } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleContainerSettingsParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleContainerSettingsParser.java new file mode 100644 index 000000000000..d78921241b05 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleContainerSettingsParser.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.configuration; + +import java.net.URI; +import java.util.Map; + +public class BundleContainerSettingsParser extends BundleConfigurationParser { + private final Map containerSettings; + + public BundleContainerSettingsParser(Map containerSettings) { + this.containerSettings = containerSettings; + } + + @Override + public void parseAndRegister(Object json, URI origin) { + Map jsonMap = asMap(json, "Expected a map of container settings and values"); + jsonMap.forEach((k, v) -> containerSettings.put(k, v.toString())); + } +} diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java similarity index 94% rename from substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java rename to substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java index 949012704bde..6f9b5aab81ac 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/EnvironmentParser.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java @@ -29,12 +29,12 @@ import java.net.URI; import java.util.Map; -public class EnvironmentParser extends BundleConfigurationParser { +public class BundleEnvironmentParser extends BundleConfigurationParser { private static final String environmentKeyField = "key"; private static final String environmentValueField = "val"; private final Map environment; - public EnvironmentParser(Map environment) { + public BundleEnvironmentParser(Map environment) { environment.clear(); this.environment = environment; } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java new file mode 100644 index 000000000000..6eafddde9520 --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher.configuration; + +import com.oracle.svm.driver.launcher.json.BundleJSONParserException; + +import java.net.URI; +import java.nio.file.Path; +import java.util.Map; + +public class BundlePathMapParser extends BundleConfigurationParser { + + private static final String substitutionMapSrcField = "src"; + private static final String substitutionMapDstField = "dst"; + + private final Map pathMap; + + public BundlePathMapParser(Map pathMap) { + this.pathMap = pathMap; + } + + @Override + public void parseAndRegister(Object json, URI origin) { + for (var rawEntry : asList(json, "Expected a list of path substitution objects")) { + var entry = asMap(rawEntry, "Expected a substitution object"); + Object srcPathString = entry.get(substitutionMapSrcField); + if (srcPathString == null) { + throw new BundleJSONParserException("Expected " + substitutionMapSrcField + "-field in substitution object"); + } + Object dstPathString = entry.get(substitutionMapDstField); + if (dstPathString == null) { + throw new BundleJSONParserException("Expected " + substitutionMapDstField + "-field in substitution object"); + } + pathMap.put(Path.of(srcPathString.toString()), Path.of(dstPathString.toString())); + } + } +} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 4295844707fd..543017151849 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -31,7 +31,6 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; -import java.net.URI; import java.nio.file.CopyOption; import java.nio.file.FileSystem; import java.nio.file.FileSystems; @@ -69,13 +68,13 @@ import com.oracle.svm.core.util.ExitStatus; import com.oracle.svm.driver.launcher.BundleLauncher; -import org.graalvm.collections.EconomicMap; -import org.graalvm.util.json.JSONParser; -import org.graalvm.util.json.JSONParserException; +import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; +import com.oracle.svm.driver.launcher.configuration.BundleContainerSettingsParser; +import com.oracle.svm.driver.launcher.configuration.BundleEnvironmentParser; +import com.oracle.svm.driver.launcher.configuration.BundlePathMapParser; import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.configure.ConfigurationParser; import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.util.json.JsonPrinter; import com.oracle.svm.core.util.json.JsonWriter; @@ -83,6 +82,16 @@ import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.StringUtil; +import static com.oracle.svm.driver.launcher.BundleLauncher.AUXILIARY_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.AUXILIARY_OUTPUT_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.CLASSES_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.CLASSPATH_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.IMAGE_PATH_OUTPUT_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.INPUT_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.MODULE_PATH_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.OUTPUT_DIR_NAME; +import static com.oracle.svm.driver.launcher.BundleLauncher.STAGE_DIR_NAME; + final class BundleSupport { final NativeImage nativeImage; @@ -111,7 +120,7 @@ final class BundleSupport { private static final int BUNDLE_FILE_FORMAT_VERSION_MINOR = 9; private static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; - private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; + private static final String BUNDLE_TEMP_DIR_PREFIX = BundleLauncher.BUNDLE_TEMP_DIR_PREFIX; private static final String ORIGINAL_DIR_EXTENSION = ".orig"; private Path bundlePath; @@ -120,7 +129,7 @@ final class BundleSupport { private final BundleProperties bundleProperties; static final String BUNDLE_OPTION = "--bundle"; - static final String BUNDLE_FILE_EXTENSION = ".nib"; + static final String BUNDLE_FILE_EXTENSION = BundleLauncher.BUNDLE_FILE_EXTENSION; static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); boolean useContainer; @@ -491,15 +500,15 @@ private BundleSupport(NativeImage nativeImage) { bundleProperties = new BundleProperties(); bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); - inputDir = rootDir.resolve("input"); - stageDir = Files.createDirectories(inputDir.resolve("stage")); - auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); - Path classesDir = inputDir.resolve("classes"); - classPathDir = Files.createDirectories(classesDir.resolve("cp")); - modulePathDir = Files.createDirectories(classesDir.resolve("p")); - outputDir = rootDir.resolve("output"); - imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); - auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); + inputDir = rootDir.resolve(INPUT_DIR_NAME); + stageDir = Files.createDirectories(inputDir.resolve(STAGE_DIR_NAME)); + auxiliaryDir = Files.createDirectories(inputDir.resolve(AUXILIARY_DIR_NAME)); + Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); + classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); + modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); + outputDir = rootDir.resolve(OUTPUT_DIR_NAME); + imagePathOutputDir = Files.createDirectories(outputDir.resolve(IMAGE_PATH_OUTPUT_DIR_NAME)); + auxiliaryOutputDir = Files.createDirectories(outputDir.resolve(AUXILIARY_OUTPUT_DIR_NAME)); } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } @@ -522,7 +531,7 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { bundleProperties = new BundleProperties(); bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); - outputDir = rootDir.resolve("output"); + outputDir = rootDir.resolve(OUTPUT_DIR_NAME); String originalOutputDirName = outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); @@ -560,34 +569,34 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { nativeImage.config.modulePathBuild = !forceBuilderOnClasspath; try { - inputDir = rootDir.resolve("input"); - stageDir = Files.createDirectories(inputDir.resolve("stage")); - auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); - Path classesDir = inputDir.resolve("classes"); - classPathDir = Files.createDirectories(classesDir.resolve("cp")); - modulePathDir = Files.createDirectories(classesDir.resolve("p")); - imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); - auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); + inputDir = rootDir.resolve(INPUT_DIR_NAME); + stageDir = Files.createDirectories(inputDir.resolve(STAGE_DIR_NAME)); + auxiliaryDir = Files.createDirectories(inputDir.resolve(AUXILIARY_DIR_NAME)); + Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); + classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); + modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); + imagePathOutputDir = Files.createDirectories(outputDir.resolve(IMAGE_PATH_OUTPUT_DIR_NAME)); + auxiliaryOutputDir = Files.createDirectories(outputDir.resolve(AUXILIARY_OUTPUT_DIR_NAME)); } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (Reader reader = Files.newBufferedReader(pathCanonicalizationsFile)) { - new PathMapParser(pathCanonicalizations).parseAndRegister(reader); + new BundlePathMapParser(pathCanonicalizations).parseAndRegister(reader); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathCanonicalizationsFile, e); } Path pathSubstitutionsFile = stageDir.resolve("path_substitutions.json"); try (Reader reader = Files.newBufferedReader(pathSubstitutionsFile)) { - new PathMapParser(pathSubstitutions).parseAndRegister(reader); + new BundlePathMapParser(pathSubstitutions).parseAndRegister(reader); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } Path environmentFile = stageDir.resolve("environment.json"); if (Files.isReadable(environmentFile)) { try (Reader reader = Files.newBufferedReader(environmentFile)) { - new EnvironmentParser(nativeImage.imageBuilderEnvironment).parseAndRegister(reader); + new BundleEnvironmentParser(nativeImage.imageBuilderEnvironment).parseAndRegister(reader); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + environmentFile, e); } @@ -596,16 +605,11 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { Path containerFile = stageDir.resolve("container.json"); if (Files.exists(containerFile)) { try (Reader reader = Files.newBufferedReader(containerFile)) { - EconomicMap json = JSONParser.parseDict(reader); - if (json.get(CONTAINER_IMAGE_JSON_KEY) != null) { - bundleContainerImage = json.get(CONTAINER_IMAGE_JSON_KEY).toString(); - } - if (json.get(CONTAINER_TOOL_JSON_KEY) != null) { - bundleContainerTool = json.get(CONTAINER_TOOL_JSON_KEY).toString(); - } - if (json.get(CONTAINER_TOOL_VERSION_JSON_KEY) != null) { - bundleContainerToolVersion = json.get(CONTAINER_TOOL_VERSION_JSON_KEY).toString(); - } + Map containerSettings = new HashMap<>(); + new BundleContainerSettingsParser(containerSettings).parseAndRegister(reader); + bundleContainerImage = containerSettings.getOrDefault(CONTAINER_IMAGE_JSON_KEY, bundleContainerImage); + bundleContainerTool = containerSettings.getOrDefault(CONTAINER_TOOL_JSON_KEY, bundleContainerTool); + bundleContainerToolVersion = containerSettings.getOrDefault(CONTAINER_TOOL_VERSION_JSON_KEY, bundleContainerToolVersion); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } @@ -627,7 +631,7 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { Path buildArgsFile = stageDir.resolve("build.json"); try (Reader reader = Files.newBufferedReader(buildArgsFile)) { List buildArgsFromFile = new ArrayList<>(); - new BuildArgsParser(buildArgsFromFile).parseAndRegister(reader); + new BundleArgsParser(buildArgsFromFile).parseAndRegister(reader); nativeImageArgs = Collections.unmodifiableList(buildArgsFromFile); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + buildArgsFile, e); @@ -1079,7 +1083,6 @@ private Manifest createManifest() { Manifest mf = new Manifest(); Attributes attributes = mf.getMainAttributes(); attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); - /* If we add run-bundle-as-java-application a launcher mainclass would be added here */ attributes.put(Attributes.Name.MAIN_CLASS, BundleLauncher.class.getName()); return mf; } @@ -1109,76 +1112,6 @@ private static void printEnvironmentVariable(Map.Entry entry, Js w.append('}'); } - private static final class PathMapParser extends ConfigurationParser { - - private final Map pathMap; - - private PathMapParser(Map pathMap) { - super(true); - this.pathMap = pathMap; - } - - @Override - public void parseAndRegister(Object json, URI origin) { - for (var rawEntry : asList(json, "Expected a list of path substitution objects")) { - var entry = asMap(rawEntry, "Expected a substitution object"); - Object srcPathString = entry.get(substitutionMapSrcField); - if (srcPathString == null) { - throw new JSONParserException("Expected " + substitutionMapSrcField + "-field in substitution object"); - } - Object dstPathString = entry.get(substitutionMapDstField); - if (dstPathString == null) { - throw new JSONParserException("Expected " + substitutionMapDstField + "-field in substitution object"); - } - pathMap.put(Path.of(srcPathString.toString()), Path.of(dstPathString.toString())); - } - } - } - - private static final class EnvironmentParser extends ConfigurationParser { - - private final Map environment; - - private EnvironmentParser(Map environment) { - super(true); - environment.clear(); - this.environment = environment; - } - - @Override - public void parseAndRegister(Object json, URI origin) { - for (var rawEntry : asList(json, "Expected a list of environment variable objects")) { - var entry = asMap(rawEntry, "Expected a environment variable object"); - Object envVarKeyString = entry.get(environmentKeyField); - if (envVarKeyString == null) { - throw new JSONParserException("Expected " + environmentKeyField + "-field in environment variable object"); - } - Object envVarValueString = entry.get(environmentValueField); - if (envVarValueString == null) { - throw new JSONParserException("Expected " + environmentValueField + "-field in environment variable object"); - } - environment.put(envVarKeyString.toString(), envVarValueString.toString()); - } - } - } - - private static final class BuildArgsParser extends ConfigurationParser { - - private final List args; - - private BuildArgsParser(List args) { - super(true); - this.args = args; - } - - @Override - public void parseAndRegister(Object json, URI origin) { - for (var arg : asList(json, "Expected a list of arguments")) { - args.add(arg.toString()); - } - } - } - private static final Path bundlePropertiesFileName = Path.of("META-INF/nibundle.properties"); private final class BundleProperties { From 5e26dd6fdf2deba4c3908c46022d050d69218f03 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 12 Jun 2023 16:31:13 +0200 Subject: [PATCH 32/61] Fix executing bundles created with -jar and executing bundles with additional options --- .../svm/driver/launcher/BundleLauncher.java | 22 ++++++++++--------- .../com/oracle/svm/driver/BundleSupport.java | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 4fd70bffc217..c9939405d048 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -79,8 +79,9 @@ public static void main(String[] args) { Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); command.add(javaExecutable.toString()); - List programArgs = parseBundleLauncherOptions(args); - command.addAll(programArgs); + List applicationArgs = new ArrayList<>(); + List launchArgs = parseBundleLauncherArgs(args, applicationArgs); + command.addAll(launchArgs); List classpath = new ArrayList<>(); if (Files.isDirectory(classPathDir)) { @@ -123,6 +124,8 @@ public static void main(String[] args) { throw new RuntimeException("Failed to read bundle-file " + argsFile, e); } + command.addAll(applicationArgs); + ProcessBuilder pb = new ProcessBuilder(command); Path environmentFile = stageDir.resolve("environment.json"); @@ -162,9 +165,9 @@ public static void main(String[] args) { } } - private static List parseBundleLauncherOptions(String[] args) { + private static List parseBundleLauncherArgs(String[] args, List applicationArgs) { Deque argQueue = new ArrayDeque<>(Arrays.asList(args)); - List programArgs = new ArrayList<>(); + List launchArgs = new ArrayList<>(); while (!argQueue.isEmpty()) { String arg = argQueue.removeFirst(); @@ -176,19 +179,19 @@ private static List parseBundleLauncherOptions(String[] args) { try { Files.createDirectories(externalAuxiliaryOutputDir); System.out.println("Native image agent output written to " + externalAuxiliaryOutputDir); - programArgs.add("-agentlib:native-image-agent=config-output-dir=" + externalAuxiliaryOutputDir); + launchArgs.add("-agentlib:native-image-agent=config-output-dir=" + externalAuxiliaryOutputDir); } catch (IOException e) { System.out.println("Failed to create native image agent output dir"); } } else if (arg.equals("--")) { - programArgs.addAll(argQueue); + applicationArgs.addAll(argQueue); argQueue.clear(); } else { - programArgs.add(arg); + applicationArgs.add(arg); } } - return programArgs; + return launchArgs; } private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); @@ -244,10 +247,9 @@ private static void deleteAllFiles(Path toDelete) { } private static void unpackBundle(Path bundleFilePath) { - Path rootDir; Path inputDir; try { - rootDir = createBundleRootDir(); + Path rootDir = createBundleRootDir(); inputDir = rootDir.resolve(INPUT_DIR_NAME); try (JarFile archive = new JarFile(bundleFilePath.toFile())) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 543017151849..3306ee8e094a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -1035,7 +1035,7 @@ private Path writeBundle() { Path runArgsFile = stageDir.resolve("run.json"); try (JsonWriter writer = new JsonWriter(runArgsFile)) { - List equalsRunArg = List.of("-jar", "-m", "--module"); + List equalsRunArg = List.of("-m", "--module"); List runArgs = new ArrayList<>(); ListIterator bundleArgsIterator = bundleArgs.listIterator(); while (bundleArgsIterator.hasNext()) { From 0dd424467af6172f2c81508923b01d431e7fc590 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 19 Jun 2023 11:20:19 +0200 Subject: [PATCH 33/61] Use Hosted options for creating run.json, Overwrite bundle launcher files if exist --- .../com/oracle/svm/driver/BundleSupport.java | 41 +++++++++++-------- .../com/oracle/svm/driver/NativeImage.java | 6 ++- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 3306ee8e094a..eccea8281f5f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -947,7 +947,7 @@ private Path writeBundle() { if (bundleFileParent != null) { Files.createDirectories(bundleFileParent); } - Files.copy(source, target); + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); } catch (Exception e) { throw NativeImage.showError("Failed to write bundle-file " + target, e); } @@ -1033,25 +1033,30 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + buildArgsFile, e); } - Path runArgsFile = stageDir.resolve("run.json"); - try (JsonWriter writer = new JsonWriter(runArgsFile)) { - List equalsRunArg = List.of("-m", "--module"); - List runArgs = new ArrayList<>(); - ListIterator bundleArgsIterator = bundleArgs.listIterator(); - while (bundleArgsIterator.hasNext()) { - String arg = bundleArgsIterator.next(); - if (equalsRunArg.contains(arg)) { - runArgs.add(arg); - runArgs.add(bundleArgsIterator.next()); + // skip run.json for shared library bundles + if (nativeImage.buildExecutable) { + Path runArgsFile = stageDir.resolve("run.json"); + try (JsonWriter writer = new JsonWriter(runArgsFile)) { + List runArgs = new ArrayList<>(); + + boolean hasMainClassModule = nativeImage.mainClassModule != null && !nativeImage.mainClassModule.isEmpty(); + boolean hasMainClass = nativeImage.mainClass != null && !nativeImage.mainClass.isEmpty(); + if (hasMainClassModule) { + runArgs.add("-m"); + StringBuilder mainModule = new StringBuilder(nativeImage.mainClassModule); + if (hasMainClass) { + mainModule.append("/").append(nativeImage.mainClass); + } + runArgs.add(mainModule.toString()); + } else { + runArgs.add(nativeImage.mainClass); } + + /* Printing as list with defined sort-order ensures useful diffs are possible */ + JsonPrinter.printCollection(writer, runArgs, null, BundleSupport::printBuildArg); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + runArgsFile, e); } - if (runArgs.isEmpty()) { - runArgs.add(nativeImage.mainClass); - } - /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, runArgs, null, BundleSupport::printBuildArg); - } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + runArgsFile, e); } bundleProperties.write(); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 37a6a1bbe256..2034dba101e1 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1094,7 +1094,7 @@ private int completeImageBuild() { imageBuilderJavaArgs.addAll(getAgentArguments()); mainClass = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHClass); - boolean buildExecutable = imageBuilderArgs.stream().noneMatch(arg -> arg.startsWith(oHEnableSharedLibraryFlagPrefix)); + buildExecutable = imageBuilderArgs.stream().noneMatch(arg -> arg.startsWith(oHEnableSharedLibraryFlagPrefix)); boolean listModules = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(oH + "+" + "ListModules")); printFlags |= imageBuilderArgs.stream().anyMatch(arg -> arg.matches("-H:MicroArchitecture(@[^=]*)?=list")); @@ -1115,7 +1115,7 @@ private int completeImageBuild() { if (!jarOptionMode) { /* Main-class from customImageBuilderArgs counts as explicitMainClass */ boolean explicitMainClass = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHClass) != null; - String mainClassModule = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHModule); + mainClassModule = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHModule); boolean hasMainClassModule = mainClassModule != null && !mainClassModule.isEmpty(); boolean hasMainClass = mainClass != null && !mainClass.isEmpty(); @@ -1409,7 +1409,9 @@ private void addTargetArguments() { } } + boolean buildExecutable; String mainClass; + String mainClassModule; String imageName; Path imagePath; From 5b2992e7728db20062878574f4276bc96da12246 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 19 Jun 2023 11:21:53 +0200 Subject: [PATCH 34/61] Always create a dockerfile for bundles --- .../com/oracle/svm/driver/BundleSupport.java | 72 +++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index eccea8281f5f..ce7607d03e8c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -140,7 +140,6 @@ final class BundleSupport { private String containerImage; private String bundleContainerImage; private Path dockerfile; - private Path bundleDockerfile; private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private static final String DEFAULT_DOCKERFILE = NativeImage.getResource("/container-default/Dockerfile"); private static final String DEFAULT_DOCKERFILE_MUSLIB = NativeImage.getResource("/container-default/Dockerfile_muslib_extension"); @@ -235,15 +234,15 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma .forEach(bundleSupport::parseExtendedOption); if (bundleSupport.useContainer) { - if (!OS.LINUX.isCurrent()) { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); - bundleSupport.useContainer = false; - } else { + if (OS.LINUX.isCurrent()) { if (nativeImage.isDryRun()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); } else { bundleSupport.initializeContainerImage(); } + } else { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); + bundleSupport.useContainer = false; } } @@ -255,6 +254,21 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } + private void createDockerfile() { + // create Dockerfile if not available + try { + dockerfile = stageDir.resolve("Dockerfile"); + String dockerfileText = DEFAULT_DOCKERFILE; + if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { + dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; + } + Files.writeString(dockerfile, dockerfileText); + dockerfile.toFile().deleteOnExit(); + } catch (IOException e) { + throw NativeImage.showError("Failed to create default Dockerfile " + dockerfile); + } + } + private void parseExtendedOption(String option) { String optionKey; String optionValue; @@ -286,14 +300,13 @@ private void parseExtendedOption(String option) { if (!useContainer) { throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, ExtendedBundleOptions.container)); } - if (dockerfile != null) { - throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); - } if (optionValue != null) { dockerfile = Path.of(optionValue); if (!Files.isReadable(dockerfile)) { throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", dockerfile.toAbsolutePath())); } + } else { + throw NativeImage.showError(String.format("native-image option %s requires a dockerfile argument. E.g. %s=path/to/Dockerfile.", optionKey, optionKey)); } } default -> { @@ -308,26 +321,13 @@ private void parseExtendedOption(String option) { private void initializeContainerImage() { String bundleFileName = bundlePath == null ? "" : bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); - if (bundleDockerfile != null && dockerfile == null) { - dockerfile = bundleDockerfile; + if (dockerfile == null) { + createDockerfile(); } - - // create Dockerfile if not available for writing or loading bundle try { - if (dockerfile == null) { - dockerfile = Files.createTempFile("Dockerfile", null); - String dockerfileText = DEFAULT_DOCKERFILE; - if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { - dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; - } - Files.writeString(dockerfile, dockerfileText); - dockerfile.toFile().deleteOnExit(); - containerImage = SubstrateUtil.digest(dockerfileText); - } else { - containerImage = SubstrateUtil.digest(Files.readString(dockerfile)); - } + containerImage = SubstrateUtil.digest(Files.readString(dockerfile)); } catch (IOException e) { - throw NativeImage.showError(e.getMessage()); + throw NativeImage.showError("Could not read Dockerfile " + dockerfile); } if (bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { @@ -622,11 +622,7 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } } - bundleDockerfile = stageDir.resolve("Dockerfile"); - if (!Files.isReadable(bundleDockerfile)) { - bundleDockerfile = null; - } - + dockerfile = stageDir.resolve("Dockerfile"); Path buildArgsFile = stageDir.resolve("build.json"); try (Reader reader = Files.newBufferedReader(buildArgsFile)) { @@ -998,15 +994,17 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + containerFile, e); } } + } - if (dockerfile != null) { - bundleDockerfile = stageDir.resolve("Dockerfile"); - try { - Files.copy(dockerfile, bundleDockerfile); - } catch (IOException e) { - throw NativeImage.showError("Failed to write bundle-file " + bundleDockerfile, e); - } + Path dockerfilePath = stageDir.resolve("Dockerfile"); + try { + if (dockerfile == null) { + createDockerfile(); + } else if (!dockerfilePath.equals(dockerfile)) { + Files.copy(dockerfile, dockerfilePath); } + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + dockerfilePath, e); } Path buildArgsFile = stageDir.resolve("build.json"); From be57ee239eede3224ef887e1068d1896758be80f Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 19 Jun 2023 11:26:08 +0200 Subject: [PATCH 35/61] Rework bundle launcher Check for run.json Add verbose commandline argument Add update bundle option to native-image-agent execution --- .../svm/driver/launcher/BundleLauncher.java | 175 +++++++++++++----- 1 file changed, 130 insertions(+), 45 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index c9939405d048..ace75953a72e 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -68,14 +68,77 @@ public class BundleLauncher { private static Path modulePathDir; private static Path bundleFilePath; + private static String bundleName; + private static Path agentOutputDir; + private static String newBundleName = null; + private static boolean updateBundle = false; + + public static boolean verbose = false; - public static void main(String[] args) { - List command = new ArrayList<>(); + public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); + bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); + agentOutputDir = bundleFilePath.getParent().resolve(Paths.get(bundleName + "." + OUTPUT_DIR_NAME, "launcher")); unpackBundle(bundleFilePath); + // if we did not create a run.json bundle is not executable, e.g. shared library bundles + if (!Files.exists(stageDir.resolve("run.json"))) { + System.out.println("Bundle " + bundleFilePath + " is not executable!"); + System.exit(1); + } + + List command = createLaunchCommand(args); + ProcessBuilder pb = new ProcessBuilder(command); + + Path environmentFile = stageDir.resolve("environment.json"); + if (Files.isReadable(environmentFile)) { + try (Reader reader = Files.newBufferedReader(environmentFile)) { + Map launcherEnvironment = new HashMap<>(); + new BundleEnvironmentParser(launcherEnvironment).parseAndRegister(reader); + pb.environment().putAll(launcherEnvironment); + } catch (IOException e) { + throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); + } + } + + if (verbose) { + List environmentList = pb.environment() + .entrySet() + .stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .sorted() + .toList(); + System.out.println("Executing ["); + System.out.println(String.join(" \\\n", environmentList)); + System.out.println(String.join(" \\\n", command)); + System.out.println("]"); + } + + Process p = null; + int exitCode; + try { + p = pb.inheritIO().start(); + exitCode = p.waitFor(); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to run bundled application"); + } finally { + if (p != null) { + p.destroy(); + } + } + + if (updateBundle) { + exitCode = updateBundle(); + } + + System.exit(exitCode); + } + + private static List createLaunchCommand(String[] args) { + List command = new ArrayList<>(); + Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); command.add(javaExecutable.toString()); @@ -126,38 +189,29 @@ public static void main(String[] args) { command.addAll(applicationArgs); - ProcessBuilder pb = new ProcessBuilder(command); + return command; + } - Path environmentFile = stageDir.resolve("environment.json"); - if (Files.isReadable(environmentFile)) { - try (Reader reader = Files.newBufferedReader(environmentFile)) { - Map launcherEnvironment = new HashMap<>(); - new BundleEnvironmentParser(launcherEnvironment).parseAndRegister(reader); - pb.environment().putAll(launcherEnvironment); - } catch (IOException e) { - throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); - } - } + private static int updateBundle() { + List command = new ArrayList<>(); - if (System.getenv("BUNDLE_LAUNCHER_VERBOSE") != null) { - List environmentList = pb.environment() - .entrySet() - .stream() - .map(e -> e.getKey() + "=" + e.getValue()) - .sorted() - .toList(); - System.out.println("Executing ["); - System.out.println(String.join(" \\\n", environmentList)); - System.out.println(String.join(" \\\n", command)); - System.out.println("]"); - } + Path nativeImageExecutable = getNativeImageExecutable().toAbsolutePath().normalize(); + command.add(nativeImageExecutable.toString()); + + Path newBundleFilePath = newBundleName == null ? bundleFilePath : bundleFilePath.getParent().resolve(newBundleName + BUNDLE_FILE_EXTENSION); + command.add("--bundle-apply=" + bundleFilePath); + command.add("--bundle-create=" + newBundleFilePath + ",dry-run"); + command.add("-cp"); + command.add(agentOutputDir.toString()); + + ProcessBuilder pb = new ProcessBuilder(command); Process p = null; try { p = pb.inheritIO().start(); - p.waitFor(); + return p.waitFor(); } catch (IOException | InterruptedException e) { - throw new RuntimeException("Failed to run bundled application"); + throw new RuntimeException("Failed to create updated bundle."); } finally { if (p != null) { p.destroy(); @@ -173,16 +227,28 @@ private static List parseBundleLauncherArgs(String[] args, List String arg = argQueue.removeFirst(); if (arg.startsWith("--with-native-image-agent")) { - String bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); - Path bundlePath = bundleFilePath.getParent(); - Path externalAuxiliaryOutputDir = bundlePath.resolve(bundleName + "." + OUTPUT_DIR_NAME).resolve(AUXILIARY_OUTPUT_DIR_NAME); + if (arg.indexOf(',') >= 0) { + String option = arg.substring(arg.indexOf(',') + 1); + if (option.startsWith("update-bundle")) { + updateBundle = true; + if (option.indexOf('=') >= 0) { + newBundleName = option.substring(option.indexOf('=')).replace(BUNDLE_FILE_EXTENSION, ""); + } + } else { + throw new RuntimeException(String.format("Unknown option %s. Valid option is: update-bundle[=].", option)); + } + } + + Path outputDir = agentOutputDir.resolve(Paths.get("META-INF", "native-image", bundleName + "-agent")); try { - Files.createDirectories(externalAuxiliaryOutputDir); - System.out.println("Native image agent output written to " + externalAuxiliaryOutputDir); - launchArgs.add("-agentlib:native-image-agent=config-output-dir=" + externalAuxiliaryOutputDir); + Files.createDirectories(outputDir); + System.out.println("Native image agent output written to " + agentOutputDir); + launchArgs.add("-agentlib:native-image-agent=config-output-dir=" + outputDir); } catch (IOException e) { System.out.println("Failed to create native image agent output dir"); } + } else if (arg.equals("--verbose")) { + verbose = true; } else if (arg.equals("--")) { applicationArgs.addAll(argQueue); argQueue.clear(); @@ -202,6 +268,10 @@ private static Path getJavaExecutable() { return buildTimeJavaHome.resolve(binJava); } + return getJavaHomeExecutable(binJava); + } + + private static Path getJavaHomeExecutable(Path executable) { String javaHome = System.getenv("JAVA_HOME"); if (javaHome == null) { throw new RuntimeException("Environment variable JAVA_HOME is not set"); @@ -210,10 +280,27 @@ private static Path getJavaExecutable() { if (!Files.isDirectory(javaHomeDir)) { throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory"); } - if (!Files.isExecutable(javaHomeDir.resolve(binJava))) { - throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory with a " + binJava + " executable"); + if (!Files.isExecutable(javaHomeDir.resolve(executable))) { + throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory with a " + executable + " executable"); } - return javaHomeDir.resolve(binJava); + return javaHomeDir.resolve(executable); + } + + private static Path getNativeImageExecutable() { + Path binNativeImage = Paths.get("bin", System.getProperty("os.name").contains("Windows") ? "native-image.exe" : "native-image"); + if (Files.isExecutable(buildTimeJavaHome.resolve(binNativeImage))) { + return buildTimeJavaHome.resolve(binNativeImage); + } + + String graalVMHome = System.getenv("GRAALVM_HOME"); + if (graalVMHome != null) { + Path graalVMHomeDir = Paths.get(graalVMHome); + if (Files.isDirectory(graalVMHomeDir) && Files.isExecutable(graalVMHomeDir.resolve(binNativeImage))) { + return graalVMHomeDir.resolve(binNativeImage); + } + } + + return getJavaHomeExecutable(binNativeImage); } private static final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); @@ -257,16 +344,14 @@ private static void unpackBundle(Path bundleFilePath) { while (jarEntries.hasMoreElements() && !deleteBundleRoot.get()) { JarEntry jarEntry = jarEntries.nextElement(); Path bundleEntry = rootDir.resolve(jarEntry.getName()); - if (bundleEntry.startsWith(inputDir)) { - try { - Path bundleFileParent = bundleEntry.getParent(); - if (bundleFileParent != null) { - Files.createDirectories(bundleFileParent); - } - Files.copy(archive.getInputStream(jarEntry), bundleEntry); - } catch (IOException e) { - throw new RuntimeException("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); + try { + Path bundleFileParent = bundleEntry.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); } + Files.copy(archive.getInputStream(jarEntry), bundleEntry); + } catch (IOException e) { + throw new RuntimeException("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); } } } From cf32db36c16289f7f11fd7d44964155bde260563 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 19 Jun 2023 16:35:12 +0200 Subject: [PATCH 36/61] Add container support for executable bundles --- .../svm/driver/launcher/BundleLauncher.java | 75 +++++- .../driver/launcher/BundleLauncherUtil.java | 65 +++++ .../svm/driver/launcher/ContainerSupport.java | 241 ++++++++++++++++++ 3 files changed, 372 insertions(+), 9 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index ace75953a72e..e0cb846af803 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -47,6 +47,8 @@ import java.util.jar.JarFile; import java.util.stream.Stream; +import static com.oracle.svm.driver.launcher.ContainerSupport.CONTAINER_GRAAL_VM_HOME; + public class BundleLauncher { @@ -63,6 +65,9 @@ public class BundleLauncher { public static final String IMAGE_PATH_OUTPUT_DIR_NAME = "default"; public static final String AUXILIARY_OUTPUT_DIR_NAME = "other"; + private static Path rootDir; + private static Path inputDir; + private static Path outputDir; private static Path stageDir; private static Path classPathDir; private static Path modulePathDir; @@ -76,6 +81,8 @@ public class BundleLauncher { public static boolean verbose = false; + public static ContainerSupport containerSupport; + public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); @@ -93,9 +100,9 @@ public static void main(String[] args) { ProcessBuilder pb = new ProcessBuilder(command); Path environmentFile = stageDir.resolve("environment.json"); + Map launcherEnvironment = new HashMap<>(); if (Files.isReadable(environmentFile)) { try (Reader reader = Files.newBufferedReader(environmentFile)) { - Map launcherEnvironment = new HashMap<>(); new BundleEnvironmentParser(launcherEnvironment).parseAndRegister(reader); pb.environment().putAll(launcherEnvironment); } catch (IOException e) { @@ -103,6 +110,22 @@ public static void main(String[] args) { } } + if (useContainer()) { + Path javaHome = getJavaExecutable().getParent().getParent(); + ContainerSupport.replaceContainerPaths(command, javaHome, rootDir); + + // TODO also mount agentDir if necessary + + Map mountMapping = new HashMap<>(); + Path containerRoot = Paths.get("/"); + mountMapping.put(javaHome, new ContainerSupport.TargetPath(containerRoot.resolve(CONTAINER_GRAAL_VM_HOME),true)); + mountMapping.put(inputDir, new ContainerSupport.TargetPath(containerRoot.resolve(INPUT_DIR_NAME),true)); + mountMapping.put(outputDir, new ContainerSupport.TargetPath(containerRoot.resolve(OUTPUT_DIR_NAME),false)); + + containerSupport.initializeContainerImage(); + command.addAll(0, containerSupport.createContainerCommand(launcherEnvironment, mountMapping)); + } + if (verbose) { List environmentList = pb.environment() .entrySet() @@ -136,6 +159,10 @@ public static void main(String[] args) { System.exit(exitCode); } + public static boolean useContainer() { + return containerSupport != null; + } + private static List createLaunchCommand(String[] args) { List command = new ArrayList<>(); @@ -247,13 +274,43 @@ private static List parseBundleLauncherArgs(String[] args, List } catch (IOException e) { System.out.println("Failed to create native image agent output dir"); } - } else if (arg.equals("--verbose")) { - verbose = true; - } else if (arg.equals("--")) { - applicationArgs.addAll(argQueue); - argQueue.clear(); + } else if (arg.startsWith("--container")) { + if (useContainer()) { + throw new RuntimeException("native-image bundle allows option container to be specified only once."); + } + Path dockerfile; + if (arg.indexOf(',') != -1) { + String option = arg.substring(arg.indexOf(',') + 1); + arg = arg.substring(0, arg.indexOf(',')); + + if (option.startsWith("dockerfile")) { + if (option.indexOf('=') != -1) { + dockerfile = Paths.get(option.substring(option.indexOf('=') + 1)); + if (!Files.isReadable(dockerfile)) { + throw new Error(String.format("Dockerfile '%s' is not readable", dockerfile.toAbsolutePath())); + } + } else { + throw new Error("container option dockerfile requires a dockerfile argument. E.g. dockerfile=path/to/Dockerfile."); + } + } else { + throw new Error(String.format("Unknown option %s. Valid option is: dockerfile=path/to/Dockerfile.", option)); + } + } else { + dockerfile = stageDir.resolve("Dockerfile"); + } + containerSupport = new ContainerSupport(dockerfile, stageDir); + if (arg.indexOf('=') != -1) { + containerSupport.containerTool = arg.substring(arg.indexOf('=') + 1); + } } else { - applicationArgs.add(arg); + switch (arg) { + case "--verbose" -> verbose = true; + case "--" -> { + applicationArgs.addAll(argQueue); + argQueue.clear(); + } + default -> applicationArgs.add(arg); + } } } @@ -334,9 +391,8 @@ private static void deleteAllFiles(Path toDelete) { } private static void unpackBundle(Path bundleFilePath) { - Path inputDir; try { - Path rootDir = createBundleRootDir(); + rootDir = createBundleRootDir(); inputDir = rootDir.resolve(INPUT_DIR_NAME); try (JarFile archive = new JarFile(bundleFilePath.toFile())) { @@ -369,6 +425,7 @@ private static void unpackBundle(Path bundleFilePath) { Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); + outputDir = Files.createDirectories(rootDir.resolve(OUTPUT_DIR_NAME)); } catch (IOException e) { throw new RuntimeException("Unable to create bundle directory layout", e); } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java new file mode 100644 index 000000000000..5e413d36e4dd --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class BundleLauncherUtil { + + private static final char[] HEX = "0123456789abcdef".toCharArray(); + public static String digest(String value) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-1"); + md.update(value.getBytes(StandardCharsets.UTF_8)); + return toHex(md.digest()); + } catch (NoSuchAlgorithmException ex) { + throw new Error(ex); + } + } + public static String toHex(byte[] data) { + StringBuilder r = new StringBuilder(data.length * 2); + for (byte b : data) { + r.append(HEX[(b >> 4) & 0xf]); + r.append(HEX[b & 0xf]); + } + return r.toString(); + } + + private static final Pattern SAFE_SHELL_ARG = Pattern.compile("[A-Za-z0-9@%_\\-+=:,./]+"); + public static String quoteShellArg(String arg) { + if (arg.isEmpty()) { + return "''"; + } + Matcher m = SAFE_SHELL_ARG.matcher(arg); + if (m.matches()) { + return arg; + } + return "'" + arg.replace("'", "'\"'\"'") + "'"; + } +} diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java new file mode 100644 index 000000000000..91d86323946e --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver.launcher; + +import com.oracle.svm.driver.launcher.configuration.BundleContainerSettingsParser; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ContainerSupport { + // TODO put anything container related in here, maybe even with function references(for logging), take care + + public String containerTool; + public String bundleContainerTool; + public String containerToolVersion; + public String bundleContainerToolVersion; + public String containerImage; + public String bundleContainerImage; + public Path dockerfile; + + private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); + private static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; + private static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; + private static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; + + private static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; + + public static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); + + public ContainerSupport(Path dockerfile, Path bundleStageDir) { + this.dockerfile = dockerfile; + + if (bundleStageDir != null) { + Path containerFile = bundleStageDir.resolve("container.json"); + if (Files.exists(containerFile)) { + try (Reader reader = Files.newBufferedReader(containerFile)) { + Map containerSettings = new HashMap<>(); + new BundleContainerSettingsParser(containerSettings).parseAndRegister(reader); + bundleContainerImage = containerSettings.getOrDefault(CONTAINER_IMAGE_JSON_KEY, bundleContainerImage); + bundleContainerTool = containerSettings.getOrDefault(CONTAINER_TOOL_JSON_KEY, bundleContainerTool); + bundleContainerToolVersion = containerSettings.getOrDefault(CONTAINER_TOOL_VERSION_JSON_KEY, bundleContainerToolVersion); + } catch (IOException e) { + throw new Error("Failed to read bundle-file " + containerFile, e); + } + if (bundleContainerTool != null) { + String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" (%s)", bundleContainerToolVersion); + System.out.printf("%sBundled native-image was created in a container with %s%s.%n", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString); + } + } + } + } + + public int initializeContainerImage() { + try { + containerImage = BundleLauncherUtil.digest(Files.readString(dockerfile)); + } catch (IOException e) { + throw new Error("Could not read Dockerfile " + dockerfile); + } + + if (bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { + System.out.println("Warning: The bundled image was created with a different dockerfile."); + } + + if (bundleContainerTool != null && containerTool == null) { + containerTool = bundleContainerTool; + } + + if (containerTool != null) { + if (!isToolAvailable(containerTool)) { + throw new Error("Configured container tool not available."); + } else if (containerTool.equals("docker") && !isRootlessDocker()) { + throw new Error("Only rootless docker is supported for containerized builds."); + } + containerToolVersion = getContainerToolVersion(containerTool); + + if (bundleContainerTool != null) { + if (!containerTool.equals(bundleContainerTool)) { + System.out.printf("Warning: The bundled image was created with container tool '%s' (using '%s').%n", bundleContainerTool, containerTool); + } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { + System.out.printf("Warning: The bundled image was created with different %s version '%s' (installed '%s').%n", containerTool, bundleContainerToolVersion, containerToolVersion); + } + } + } else { + for (String tool : SUPPORTED_CONTAINER_TOOLS) { + if (isToolAvailable(tool)) { + if (tool.equals("docker") && !isRootlessDocker()) { + System.out.println(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); + continue; + } + containerTool = tool; + containerToolVersion = getContainerToolVersion(tool); + break; + } + } + if (containerTool == null) { + throw new Error(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS)); + } + } + + return createContainer(); + } + + private int createContainer() { + ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); + ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); + + String imageId = getFirstProcessResultLine(pbCheckForImage); + if (imageId == null) { + pb.inheritIO(); + } else { + System.out.printf("%sReusing container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage); + } + + Process p = null; + try { + p = pb.start(); + int status = p.waitFor(); + if (status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { + try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + System.out.printf("%sUpdated container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage); + processResult.lines().forEach(System.out::println); + } + } + return status; + } catch (IOException | InterruptedException e) { + throw new Error(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } + } + } + + private static boolean isToolAvailable(String tool) { + return Arrays.stream(System.getenv("PATH").split(":")) + .map(str -> Path.of(str).resolve(tool)) + .anyMatch(Files::isExecutable); + } + + private static String getContainerToolVersion(String tool) { + ProcessBuilder pb = new ProcessBuilder(tool, "--version"); + return getFirstProcessResultLine(pb); + } + + private static boolean isRootlessDocker() { + ProcessBuilder pb = new ProcessBuilder("docker", "context", "show"); + return getFirstProcessResultLine(pb).equals("rootless"); + } + + private static String getFirstProcessResultLine(ProcessBuilder pb) { + Process p = null; + try { + p = pb.start(); + p.waitFor(); + try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + return processResult.readLine(); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } + } + } + + public record TargetPath(Path path, boolean readonly) { + } + + public List createContainerCommand(Map containerEnvironment, Map mountMapping) { + List containerCommand = new ArrayList<>(); + + // run docker tool without network access and remove container after image build is finished + containerCommand.add(containerTool); + containerCommand.add("run"); + containerCommand.add("--network=none"); + containerCommand.add("--rm"); + + // inject environment variables into container + containerEnvironment.forEach((key, value) -> { + containerCommand.add("-e"); + containerCommand.add(key + "=" + BundleLauncherUtil.quoteShellArg(value)); + }); + + // mount java home, input and output directories and argument files for native image build + mountMapping.forEach((source, target) -> { + containerCommand.add("--mount"); + List mountArgs = new ArrayList<>(); + mountArgs.add("type=bind"); + mountArgs.add("source=" + source); + mountArgs.add("target=" + target.path); + if (target.readonly) { + mountArgs.add("readonly"); + } + containerCommand.add(BundleLauncherUtil.quoteShellArg(String.join(",", mountArgs))); + }); + + // specify container name + containerCommand.add(containerImage); + + return containerCommand; + } + + public static void replaceContainerPaths(List arguments, Path javaHome, Path bundleRoot) { + arguments.replaceAll(arg -> arg + .replace(javaHome.toString(), CONTAINER_GRAAL_VM_HOME.toString()) + .replace(bundleRoot.toString(), "") + ); + } +} From 175ab887c23b0ef14677964a320218865a102281 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 20 Jun 2023 17:32:57 +0200 Subject: [PATCH 37/61] Use ContainerSupport for building containerized native image bundles --- .../svm/driver/launcher/BundleLauncher.java | 45 ++- .../svm/driver/launcher/ContainerSupport.java | 78 +++-- .../com/oracle/svm/driver/BundleSupport.java | 306 ++++-------------- .../com/oracle/svm/driver/NativeImage.java | 28 +- 4 files changed, 153 insertions(+), 304 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index e0cb846af803..dd82f77014a8 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -47,7 +47,9 @@ import java.util.jar.JarFile; import java.util.stream.Stream; -import static com.oracle.svm.driver.launcher.ContainerSupport.CONTAINER_GRAAL_VM_HOME; +import static com.oracle.svm.driver.launcher.ContainerSupport.replaceContainerPaths; +import static com.oracle.svm.driver.launcher.ContainerSupport.mountMappingFor; +import static com.oracle.svm.driver.launcher.ContainerSupport.TargetPath; public class BundleLauncher { @@ -106,21 +108,18 @@ public static void main(String[] args) { new BundleEnvironmentParser(launcherEnvironment).parseAndRegister(reader); pb.environment().putAll(launcherEnvironment); } catch (IOException e) { - throw new RuntimeException("Failed to read bundle-file " + environmentFile, e); + throw new Error("Failed to read bundle-file " + environmentFile, e); } } if (useContainer()) { Path javaHome = getJavaExecutable().getParent().getParent(); - ContainerSupport.replaceContainerPaths(command, javaHome, rootDir); + replaceContainerPaths(command, javaHome, rootDir); - // TODO also mount agentDir if necessary - - Map mountMapping = new HashMap<>(); - Path containerRoot = Paths.get("/"); - mountMapping.put(javaHome, new ContainerSupport.TargetPath(containerRoot.resolve(CONTAINER_GRAAL_VM_HOME),true)); - mountMapping.put(inputDir, new ContainerSupport.TargetPath(containerRoot.resolve(INPUT_DIR_NAME),true)); - mountMapping.put(outputDir, new ContainerSupport.TargetPath(containerRoot.resolve(OUTPUT_DIR_NAME),false)); + Map mountMapping = mountMappingFor(javaHome, inputDir, outputDir); + if (Files.isDirectory(agentOutputDir)) { + mountMapping.put(agentOutputDir, TargetPath.of(agentOutputDir, false)); + } containerSupport.initializeContainerImage(); command.addAll(0, containerSupport.createContainerCommand(launcherEnvironment, mountMapping)); @@ -145,7 +144,7 @@ public static void main(String[] args) { p = pb.inheritIO().start(); exitCode = p.waitFor(); } catch (IOException | InterruptedException e) { - throw new RuntimeException("Failed to run bundled application"); + throw new Error("Failed to run bundled application"); } finally { if (p != null) { p.destroy(); @@ -180,7 +179,7 @@ private static List createLaunchCommand(String[] args) { .map(Path::toString) .forEach(classpath::add); } catch (IOException e) { - throw new RuntimeException("Failed to iterate through directory " + classPathDir, e); + throw new Error("Failed to iterate through directory " + classPathDir, e); } command.add("-cp"); @@ -196,7 +195,7 @@ private static List createLaunchCommand(String[] args) { .map(Path::toString) .forEach(modulePath::add); } catch (IOException e) { - throw new RuntimeException("Failed to iterate through directory " + modulePathDir, e); + throw new Error("Failed to iterate through directory " + modulePathDir, e); } if (!modulePath.isEmpty()) { @@ -211,7 +210,7 @@ private static List createLaunchCommand(String[] args) { new BundleArgsParser(argsFromFile).parseAndRegister(reader); command.addAll(argsFromFile); } catch (IOException e) { - throw new RuntimeException("Failed to read bundle-file " + argsFile, e); + throw new Error("Failed to read bundle-file " + argsFile, e); } command.addAll(applicationArgs); @@ -238,7 +237,7 @@ private static int updateBundle() { p = pb.inheritIO().start(); return p.waitFor(); } catch (IOException | InterruptedException e) { - throw new RuntimeException("Failed to create updated bundle."); + throw new Error("Failed to create updated bundle."); } finally { if (p != null) { p.destroy(); @@ -262,7 +261,7 @@ private static List parseBundleLauncherArgs(String[] args, List newBundleName = option.substring(option.indexOf('=')).replace(BUNDLE_FILE_EXTENSION, ""); } } else { - throw new RuntimeException(String.format("Unknown option %s. Valid option is: update-bundle[=].", option)); + throw new Error(String.format("Unknown option %s. Valid option is: update-bundle[=].", option)); } } @@ -276,7 +275,7 @@ private static List parseBundleLauncherArgs(String[] args, List } } else if (arg.startsWith("--container")) { if (useContainer()) { - throw new RuntimeException("native-image bundle allows option container to be specified only once."); + throw new Error("native-image launcher allows option container to be specified only once."); } Path dockerfile; if (arg.indexOf(',') != -1) { @@ -331,14 +330,14 @@ private static Path getJavaExecutable() { private static Path getJavaHomeExecutable(Path executable) { String javaHome = System.getenv("JAVA_HOME"); if (javaHome == null) { - throw new RuntimeException("Environment variable JAVA_HOME is not set"); + throw new Error("Environment variable JAVA_HOME is not set"); } Path javaHomeDir = Paths.get(javaHome); if (!Files.isDirectory(javaHomeDir)) { - throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory"); + throw new Error("Environment variable JAVA_HOME does not refer to a directory"); } if (!Files.isExecutable(javaHomeDir.resolve(executable))) { - throw new RuntimeException("Environment variable JAVA_HOME does not refer to a directory with a " + executable + " executable"); + throw new Error("Environment variable JAVA_HOME does not refer to a directory with a " + executable + " executable"); } return javaHomeDir.resolve(executable); } @@ -407,12 +406,12 @@ private static void unpackBundle(Path bundleFilePath) { } Files.copy(archive.getInputStream(jarEntry), bundleEntry); } catch (IOException e) { - throw new RuntimeException("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); + throw new Error("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); } } } } catch (IOException e) { - throw new RuntimeException("Unable to expand bundle directory layout from bundle file " + bundleFilePath, e); + throw new Error("Unable to expand bundle directory layout from bundle file " + bundleFilePath, e); } if (deleteBundleRoot.get()) { @@ -427,7 +426,7 @@ private static void unpackBundle(Path bundleFilePath) { modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); outputDir = Files.createDirectories(rootDir.resolve(OUTPUT_DIR_NAME)); } catch (IOException e) { - throw new RuntimeException("Unable to create bundle directory layout", e); + throw new Error("Unable to create bundle directory layout", e); } } } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java index 91d86323946e..dd7f2f3373be 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java @@ -32,15 +32,16 @@ import java.io.Reader; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Consumer; public class ContainerSupport { - // TODO put anything container related in here, maybe even with function references(for logging), take care - public String containerTool; public String bundleContainerTool; public String containerToolVersion; @@ -49,17 +50,28 @@ public class ContainerSupport { public String bundleContainerImage; public Path dockerfile; - private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); - private static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; - private static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; - private static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; + public static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); + public static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; + public static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; + public static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; private static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; public static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); + private final BiFunction errorFunction; + private final Consumer warningPrinter; + private final Consumer messagePrinter; + public ContainerSupport(Path dockerfile, Path bundleStageDir) { + this(dockerfile, bundleStageDir, Error::new, (msg) -> System.out.println("Warning: " + msg), System.out::println); + } + + public ContainerSupport(Path dockerfile, Path bundleStageDir, BiFunction errorFunction, Consumer warningPrinter, Consumer messagePrinter) { this.dockerfile = dockerfile; + this.errorFunction = errorFunction; + this.warningPrinter = warningPrinter; + this.messagePrinter = messagePrinter; if (bundleStageDir != null) { Path containerFile = bundleStageDir.resolve("container.json"); @@ -71,11 +83,11 @@ public ContainerSupport(Path dockerfile, Path bundleStageDir) { bundleContainerTool = containerSettings.getOrDefault(CONTAINER_TOOL_JSON_KEY, bundleContainerTool); bundleContainerToolVersion = containerSettings.getOrDefault(CONTAINER_TOOL_VERSION_JSON_KEY, bundleContainerToolVersion); } catch (IOException e) { - throw new Error("Failed to read bundle-file " + containerFile, e); + throw errorFunction.apply("Failed to read bundle-file " + containerFile, e); } if (bundleContainerTool != null) { String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" (%s)", bundleContainerToolVersion); - System.out.printf("%sBundled native-image was created in a container with %s%s.%n", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString); + messagePrinter.accept(String.format("%sBundled native-image was created in a container with %s%s.%n", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString)); } } } @@ -85,11 +97,11 @@ public int initializeContainerImage() { try { containerImage = BundleLauncherUtil.digest(Files.readString(dockerfile)); } catch (IOException e) { - throw new Error("Could not read Dockerfile " + dockerfile); + throw errorFunction.apply("Could not read Dockerfile " + dockerfile, e); } if (bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { - System.out.println("Warning: The bundled image was created with a different dockerfile."); + warningPrinter.accept("The bundled image was created with a different dockerfile."); } if (bundleContainerTool != null && containerTool == null) { @@ -98,24 +110,24 @@ public int initializeContainerImage() { if (containerTool != null) { if (!isToolAvailable(containerTool)) { - throw new Error("Configured container tool not available."); + throw errorFunction.apply("Configured container tool not available.", null); } else if (containerTool.equals("docker") && !isRootlessDocker()) { - throw new Error("Only rootless docker is supported for containerized builds."); + throw errorFunction.apply("Only rootless docker is supported for containerized builds.", null); } containerToolVersion = getContainerToolVersion(containerTool); if (bundleContainerTool != null) { if (!containerTool.equals(bundleContainerTool)) { - System.out.printf("Warning: The bundled image was created with container tool '%s' (using '%s').%n", bundleContainerTool, containerTool); + warningPrinter.accept(String.format("The bundled image was created with container tool '%s' (using '%s').%n", bundleContainerTool, containerTool)); } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { - System.out.printf("Warning: The bundled image was created with different %s version '%s' (installed '%s').%n", containerTool, bundleContainerToolVersion, containerToolVersion); + warningPrinter.accept(String.format("The bundled image was created with different %s version '%s' (installed '%s').%n", containerTool, bundleContainerToolVersion, containerToolVersion)); } } } else { for (String tool : SUPPORTED_CONTAINER_TOOLS) { if (isToolAvailable(tool)) { if (tool.equals("docker") && !isRootlessDocker()) { - System.out.println(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); + messagePrinter.accept(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); continue; } containerTool = tool; @@ -124,7 +136,7 @@ public int initializeContainerImage() { } } if (containerTool == null) { - throw new Error(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS)); + throw errorFunction.apply(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS), null); } } @@ -139,7 +151,7 @@ private int createContainer() { if (imageId == null) { pb.inheritIO(); } else { - System.out.printf("%sReusing container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage); + messagePrinter.accept(String.format("%sReusing container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); } Process p = null; @@ -148,13 +160,13 @@ private int createContainer() { int status = p.waitFor(); if (status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - System.out.printf("%sUpdated container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage); - processResult.lines().forEach(System.out::println); + messagePrinter.accept(String.format("%sUpdated container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); + processResult.lines().forEach(messagePrinter); } } return status; } catch (IOException | InterruptedException e) { - throw new Error(e.getMessage()); + throw errorFunction.apply(e.getMessage(), e); } finally { if (p != null) { p.destroy(); @@ -162,23 +174,23 @@ private int createContainer() { } } - private static boolean isToolAvailable(String tool) { + private boolean isToolAvailable(String tool) { return Arrays.stream(System.getenv("PATH").split(":")) .map(str -> Path.of(str).resolve(tool)) .anyMatch(Files::isExecutable); } - private static String getContainerToolVersion(String tool) { + private String getContainerToolVersion(String tool) { ProcessBuilder pb = new ProcessBuilder(tool, "--version"); return getFirstProcessResultLine(pb); } - private static boolean isRootlessDocker() { + private boolean isRootlessDocker() { ProcessBuilder pb = new ProcessBuilder("docker", "context", "show"); return getFirstProcessResultLine(pb).equals("rootless"); } - private static String getFirstProcessResultLine(ProcessBuilder pb) { + private String getFirstProcessResultLine(ProcessBuilder pb) { Process p = null; try { p = pb.start(); @@ -187,7 +199,7 @@ private static String getFirstProcessResultLine(ProcessBuilder pb) { return processResult.readLine(); } } catch (IOException | InterruptedException e) { - throw new RuntimeException(e.getMessage()); + throw errorFunction.apply(e.getMessage(), e); } finally { if (p != null) { p.destroy(); @@ -196,6 +208,22 @@ private static String getFirstProcessResultLine(ProcessBuilder pb) { } public record TargetPath(Path path, boolean readonly) { + public static TargetPath readonly(Path target) { + return of(target, true); + } + + public static TargetPath of(Path target, boolean readonly) { + return new TargetPath(target, readonly); + } + } + + public static Map mountMappingFor(Path javaHome, Path inputDir, Path outputDir) { + Map mountMapping = new HashMap<>(); + Path containerRoot = Paths.get("/"); + mountMapping.put(javaHome, TargetPath.readonly(containerRoot.resolve(CONTAINER_GRAAL_VM_HOME))); + mountMapping.put(inputDir, ContainerSupport.TargetPath.readonly(containerRoot.resolve(BundleLauncher.INPUT_DIR_NAME))); + mountMapping.put(outputDir, ContainerSupport.TargetPath.of(containerRoot.resolve(BundleLauncher.OUTPUT_DIR_NAME), false)); + return mountMapping; } public List createContainerCommand(Map containerEnvironment, Map mountMapping) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index ce7607d03e8c..197e5c6b4cab 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -24,11 +24,9 @@ */ package com.oracle.svm.driver; -import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.nio.file.CopyOption; @@ -68,8 +66,8 @@ import com.oracle.svm.core.util.ExitStatus; import com.oracle.svm.driver.launcher.BundleLauncher; +import com.oracle.svm.driver.launcher.ContainerSupport; import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; -import com.oracle.svm.driver.launcher.configuration.BundleContainerSettingsParser; import com.oracle.svm.driver.launcher.configuration.BundleEnvironmentParser; import com.oracle.svm.driver.launcher.configuration.BundlePathMapParser; @@ -131,21 +129,11 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = BundleLauncher.BUNDLE_FILE_EXTENSION; + public ContainerSupport containerSupport; + static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); - boolean useContainer; - private String containerTool; - private String bundleContainerTool; - private String containerToolVersion; - private String bundleContainerToolVersion; - private String containerImage; - private String bundleContainerImage; - private Path dockerfile; - private static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); private static final String DEFAULT_DOCKERFILE = NativeImage.getResource("/container-default/Dockerfile"); private static final String DEFAULT_DOCKERFILE_MUSLIB = NativeImage.getResource("/container-default/Dockerfile_muslib_extension"); - private static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; - private static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; - private static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; enum BundleOptionVariants { @@ -233,16 +221,34 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma .skip(1) .forEach(bundleSupport::parseExtendedOption); - if (bundleSupport.useContainer) { + if (bundleSupport.useContainer()) { if (OS.LINUX.isCurrent()) { if (nativeImage.isDryRun()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); } else { - bundleSupport.initializeContainerImage(); + if (bundleSupport.containerSupport.dockerfile == null) { + bundleSupport.containerSupport.dockerfile = bundleSupport.createDockerfile(); + } + int exitStatusCode = bundleSupport.containerSupport.initializeContainerImage(); + switch (ExitStatus.of(exitStatusCode)) { + case OK -> { } + case BUILDER_ERROR -> + /* Exit, builder has handled error reporting. */ + throw NativeImage.showError(null, null, exitStatusCode); + case OUT_OF_MEMORY -> { + nativeImage.showOutOfMemoryWarning(); + throw NativeImage.showError(null, null, exitStatusCode); + } + default -> { + String message = String.format("Container build request for '%s' failed with exit status %d", + nativeImage.imageName, exitStatusCode); + throw NativeImage.showError(message, null, exitStatusCode); + } + } } } else { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); - bundleSupport.useContainer = false; + //bundleSupport.useContainer = false; } } @@ -254,19 +260,26 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } - private void createDockerfile() { - // create Dockerfile if not available - try { - dockerfile = stageDir.resolve("Dockerfile"); + public boolean useContainer() { + return containerSupport != null; + } + + private Path createDockerfile() { + // take Dockerfile from bundle or create default if not available + Path dockerfile = stageDir.resolve("Dockerfile"); + if (!Files.exists(dockerfile)) { String dockerfileText = DEFAULT_DOCKERFILE; if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; } - Files.writeString(dockerfile, dockerfileText); - dockerfile.toFile().deleteOnExit(); - } catch (IOException e) { - throw NativeImage.showError("Failed to create default Dockerfile " + dockerfile); + try { + Files.writeString(dockerfile, dockerfileText); + dockerfile.toFile().deleteOnExit(); + } catch (IOException e) { + throw NativeImage.showError("Failed to create default Dockerfile " + dockerfile); + } } + return dockerfile; } private void parseExtendedOption(String option) { @@ -285,25 +298,25 @@ private void parseExtendedOption(String option) { switch (ExtendedBundleOptions.get(optionKey)) { case dry_run -> nativeImage.setDryRun(true); case container -> { - if (useContainer) { + if (useContainer()) { throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); } - useContainer = true; + containerSupport = new ContainerSupport(null, stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); if (optionValue != null) { - if (!SUPPORTED_CONTAINER_TOOLS.contains(optionValue)) { - throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, SUPPORTED_CONTAINER_TOOLS)); + if (!ContainerSupport.SUPPORTED_CONTAINER_TOOLS.contains(optionValue)) { + throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, ContainerSupport.SUPPORTED_CONTAINER_TOOLS)); } - containerTool = optionValue; + containerSupport.containerTool = optionValue; } } case dockerfile -> { - if (!useContainer) { + if (!useContainer()) { throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, ExtendedBundleOptions.container)); } if (optionValue != null) { - dockerfile = Path.of(optionValue); - if (!Files.isReadable(dockerfile)) { - throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", dockerfile.toAbsolutePath())); + containerSupport.dockerfile = Path.of(optionValue); + if (!Files.isReadable(containerSupport.dockerfile)) { + throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", containerSupport.dockerfile.toAbsolutePath())); } } else { throw NativeImage.showError(String.format("native-image option %s requires a dockerfile argument. E.g. %s=path/to/Dockerfile.", optionKey, optionKey)); @@ -318,177 +331,6 @@ private void parseExtendedOption(String option) { } } - private void initializeContainerImage() { - String bundleFileName = bundlePath == null ? "" : bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); - - if (dockerfile == null) { - createDockerfile(); - } - try { - containerImage = SubstrateUtil.digest(Files.readString(dockerfile)); - } catch (IOException e) { - throw NativeImage.showError("Could not read Dockerfile " + dockerfile); - } - - if (bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { - NativeImage.showWarning(String.format("The given bundle file %s was created with a different dockerfile.", bundleFileName)); - } - - if (bundleContainerTool != null && containerTool == null) { - containerTool = bundleContainerTool; - } - - if (containerTool != null) { - if (!isToolAvailable(containerTool)) { - throw NativeImage.showError("Configured container tool not available."); - } else if (containerTool.equals("docker") && !isRootlessDocker()) { - throw NativeImage.showError("Only rootless docker is supported for containerized builds."); - } - containerToolVersion = getContainerToolVersion(containerTool); - - if (bundleContainerTool != null) { - if (!containerTool.equals(bundleContainerTool)) { - NativeImage.showWarning(String.format("The given bundle file %s was created with container tool '%s' (using '%s').", bundleFileName, bundleContainerTool, containerTool)); - } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { - NativeImage.showWarning(String.format("The given bundle file %s was created with different %s version '%s' (installed '%s').", bundleFileName, containerTool, bundleContainerToolVersion, containerToolVersion)); - } - } - } else { - for (String tool : SUPPORTED_CONTAINER_TOOLS) { - if (isToolAvailable(tool)) { - if (tool.equals("docker") && !isRootlessDocker()) { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); - continue; - } - containerTool = tool; - containerToolVersion = getContainerToolVersion(tool); - break; - } - } - if (containerTool == null) { - throw NativeImage.showError(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS)); - } - } - - int exitStatusCode = createContainer(); - switch (ExitStatus.of(exitStatusCode)) { - case OK -> { } - case BUILDER_ERROR -> - /* Exit, builder has handled error reporting. */ - throw NativeImage.showError(null, null, exitStatusCode); - case OUT_OF_MEMORY -> { - nativeImage.showOutOfMemoryWarning(); - throw NativeImage.showError(null, null, exitStatusCode); - } - default -> { - String message = String.format("Container build request for '%s' failed with exit status %d", - nativeImage.imageName, exitStatusCode); - throw NativeImage.showError(message, null, exitStatusCode); - } - } - } - - private int createContainer() { - ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); - ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); - - String imageId = getFirstProcessResultLine(pbCheckForImage); - if (imageId == null) { - pb.inheritIO(); - } else { - nativeImage.showMessage(String.format("%sReusing container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); - } - - Process p = null; - try { - p = pb.start(); - int status = p.waitFor(); - if (status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { - try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - nativeImage.showMessage(String.format("%sUpdated container image %s.", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); - processResult.lines().forEach(System.out::println); - } - } - return status; - } catch (IOException | InterruptedException e) { - throw NativeImage.showError(e.getMessage()); - } finally { - if (p != null) { - p.destroy(); - } - } - } - - private static boolean isToolAvailable(String tool) { - return Arrays.stream(SubstrateUtil.split(System.getenv("PATH"), ":")) - .map(str -> Path.of(str).resolve(tool)) - .anyMatch(Files::isExecutable); - } - - private static String getContainerToolVersion(String tool) { - ProcessBuilder pb = new ProcessBuilder(tool, "--version"); - return getFirstProcessResultLine(pb); - } - - private static boolean isRootlessDocker() { - ProcessBuilder pb = new ProcessBuilder("docker", "context", "show"); - return getFirstProcessResultLine(pb).equals("rootless"); - } - - private static String getFirstProcessResultLine(ProcessBuilder pb) { - Process p = null; - try { - p = pb.start(); - p.waitFor(); - try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - return processResult.readLine(); - } - } catch (IOException | InterruptedException e) { - throw NativeImage.showError(e.getMessage()); - } finally { - if (p != null) { - p.destroy(); - } - } - } - - List createContainerCommand(Path argFile, Path builderArgFile) { - Path containerRoot = Path.of("/"); - List containerCommand = new ArrayList<>(); - - // run docker tool without network access and remove container after image build is finished - containerCommand.add(containerTool); - containerCommand.add("run"); - containerCommand.add("--network=none"); - containerCommand.add("--rm"); - - // inject environment variables into container - nativeImage.imageBuilderEnvironment - .forEach((key, value) -> { - containerCommand.add("-e"); - containerCommand.add(key + "=" + SubstrateUtil.quoteShellArg(value)); - }); - - // mount java home, input and output directories and argument files for native image build - containerCommand.addAll(getMountCommand(nativeImage.config.getJavaHome(), CONTAINER_GRAAL_VM_HOME, true)); - containerCommand.addAll(getMountCommand(inputDir, containerRoot.resolve(rootDir.relativize(inputDir)), true)); - containerCommand.addAll(getMountCommand(outputDir, containerRoot.resolve(rootDir.relativize(outputDir)), false)); - containerCommand.addAll(getMountCommand(argFile, argFile, true)); - containerCommand.addAll(getMountCommand(builderArgFile, builderArgFile, true)); - - // specify container name - containerCommand.add(containerImage); - - return containerCommand; - } - - private static List getMountCommand(Path source, Path target, boolean readonly) { - return List.of( - "--mount", - SubstrateUtil.quoteShellArg("type=bind,source=" + source + ",target=" + target + (readonly ? ",readonly" : "")) - ); - } - private BundleSupport(NativeImage nativeImage) { Objects.requireNonNull(nativeImage); this.nativeImage = nativeImage; @@ -602,28 +444,6 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } } - Path containerFile = stageDir.resolve("container.json"); - if (Files.exists(containerFile)) { - try (Reader reader = Files.newBufferedReader(containerFile)) { - Map containerSettings = new HashMap<>(); - new BundleContainerSettingsParser(containerSettings).parseAndRegister(reader); - bundleContainerImage = containerSettings.getOrDefault(CONTAINER_IMAGE_JSON_KEY, bundleContainerImage); - bundleContainerTool = containerSettings.getOrDefault(CONTAINER_TOOL_JSON_KEY, bundleContainerTool); - bundleContainerToolVersion = containerSettings.getOrDefault(CONTAINER_TOOL_VERSION_JSON_KEY, bundleContainerToolVersion); - } catch (IOException e) { - throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); - } - if (bundleContainerTool != null) { - String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" (%s)", bundleContainerToolVersion); - nativeImage.showMessage(String.format("%sBundled native-image was created in a container with %s%s.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString)); - if (useContainer) { - nativeImage.showMessage(String.format("%sUsing %s for native-image container build. Specify other container tool with option '%s'.", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, ExtendedBundleOptions.container)); - } - } - } - - dockerfile = stageDir.resolve("Dockerfile"); - Path buildArgsFile = stageDir.resolve("build.json"); try (Reader reader = Files.newBufferedReader(buildArgsFile)) { List buildArgsFromFile = new ArrayList<>(); @@ -672,13 +492,6 @@ Path restoreCanonicalization(Path before) { return after; } - void replacePathsForContainerBuild(List arguments) { - arguments.replaceAll(arg -> arg - .replace(nativeImage.config.getJavaHome().toString(), CONTAINER_GRAAL_VM_HOME.toString()) - .replace(rootDir.toString(), "") - ); - } - Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) { Path destinationDir = switch (bundleMemberRole) { @@ -974,16 +787,16 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + environmentFile, e); } - if (useContainer) { + if (useContainer()) { Map containerInfo = new HashMap<>(); - if (containerImage != null) { - containerInfo.put(CONTAINER_IMAGE_JSON_KEY, containerImage); + if (containerSupport.containerImage != null) { + containerInfo.put(ContainerSupport.CONTAINER_IMAGE_JSON_KEY, containerSupport.containerImage); } - if (containerTool != null) { - containerInfo.put(CONTAINER_TOOL_JSON_KEY, containerTool); + if (containerSupport.containerTool != null) { + containerInfo.put(ContainerSupport.CONTAINER_TOOL_JSON_KEY, containerSupport.containerTool); } - if (containerToolVersion != null) { - containerInfo.put(CONTAINER_TOOL_VERSION_JSON_KEY, containerToolVersion); + if (containerSupport.containerToolVersion != null) { + containerInfo.put(ContainerSupport.CONTAINER_TOOL_VERSION_JSON_KEY, containerSupport.containerToolVersion); } if (!containerInfo.isEmpty()) { @@ -998,10 +811,11 @@ private Path writeBundle() { Path dockerfilePath = stageDir.resolve("Dockerfile"); try { - if (dockerfile == null) { + if ((!useContainer() || containerSupport.dockerfile == null) && !Files.exists(dockerfilePath)) { + // if no Dockerfile was created yet create a new default Dockerfile createDockerfile(); - } else if (!dockerfilePath.equals(dockerfile)) { - Files.copy(dockerfile, dockerfilePath); + } else if (useContainer() && containerSupport.dockerfile != null && !dockerfilePath.equals(containerSupport.dockerfile)) { + Files.copy(containerSupport.dockerfile, dockerfilePath); } } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + dockerfilePath, e); @@ -1205,7 +1019,7 @@ private void write() { boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); if (imageBuilt) { - properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(useContainer)); + properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(useContainer())); } properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, NativeImage.platform); properties.put(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, System.getProperty("java.vm.vendor")); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 2034dba101e1..03a250670999 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -102,6 +102,10 @@ import com.oracle.svm.util.ReflectionUtil; import com.oracle.svm.util.StringUtil; +import static com.oracle.svm.driver.launcher.ContainerSupport.replaceContainerPaths; +import static com.oracle.svm.driver.launcher.ContainerSupport.mountMappingFor; +import static com.oracle.svm.driver.launcher.ContainerSupport.TargetPath; + public class NativeImage { private static final String DEFAULT_GENERATOR_CLASS_NAME = NativeImageGeneratorRunner.class.getName(); @@ -1558,10 +1562,11 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa /* Construct ProcessBuilder command from final arguments */ List command = new ArrayList<>(); + List completeCommandList = new ArrayList<>(); - if (useBundle() && bundleSupport.useContainer) { - bundleSupport.replacePathsForContainerBuild(arguments); - bundleSupport.replacePathsForContainerBuild(finalImageBuilderArgs); + if (useBundle() && bundleSupport.useContainer()) { + replaceContainerPaths(arguments, config.getJavaHome(), bundleSupport.rootDir); + replaceContainerPaths(finalImageBuilderArgs, config.getJavaHome(), bundleSupport.rootDir); Path binJava = Paths.get("bin", "java"); javaExecutable = BundleSupport.CONTAINER_GRAAL_VM_HOME.resolve(binJava).toString(); } @@ -1569,13 +1574,20 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa Path argFile = createVMInvocationArgumentFile(arguments); Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); - if (useBundle() && bundleSupport.useContainer) { - command.addAll(bundleSupport.createContainerCommand(argFile, builderArgFile)); + if (useBundle() && bundleSupport.useContainer()) { + Map mountMapping = mountMappingFor(config.getJavaHome(), bundleSupport.inputDir, bundleSupport.outputDir); + mountMapping.put(argFile, TargetPath.readonly(argFile)); + mountMapping.put(builderArgFile, TargetPath.readonly(builderArgFile)); + + List containerCommand = bundleSupport.containerSupport.createContainerCommand(imageBuilderEnvironment, mountMapping); + command.addAll(containerCommand); + completeCommandList.addAll(containerCommand); } command.add(javaExecutable); command.add("@" + argFile); command.add(NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + builderArgFile); + ProcessBuilder pb = new ProcessBuilder(); pb.command(command); Map environment = pb.environment(); @@ -1602,11 +1614,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa LogUtils.warningDeprecatedEnvironmentVariable(ModuleSupport.ENV_VAR_USE_MODULE_SYSTEM); } - List completeCommandList = new ArrayList<>(); - completeCommandList.addAll(environment.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).sorted().toList()); - if (!isDryRun() && useBundle() && bundleSupport.useContainer) { - completeCommandList.addAll(bundleSupport.createContainerCommand(argFile, builderArgFile)); - } + completeCommandList.addAll(0, environment.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).sorted().toList()); completeCommandList.add(javaExecutable); completeCommandList.addAll(arguments); completeCommandList.addAll(finalImageBuilderArgs); From cfc34e1c1e6c189ab1cc88156df3c6c82b1d24da Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 22 Jun 2023 09:04:00 +0200 Subject: [PATCH 38/61] Replace paths for containerized bundle execution inplace --- .../svm/driver/launcher/BundleLauncher.java | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index dd82f77014a8..660ad260fc25 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -47,6 +47,7 @@ import java.util.jar.JarFile; import java.util.stream.Stream; +import static com.oracle.svm.driver.launcher.ContainerSupport.CONTAINER_GRAAL_VM_HOME; import static com.oracle.svm.driver.launcher.ContainerSupport.replaceContainerPaths; import static com.oracle.svm.driver.launcher.ContainerSupport.mountMappingFor; import static com.oracle.svm.driver.launcher.ContainerSupport.TargetPath; @@ -85,6 +86,8 @@ public class BundleLauncher { public static ContainerSupport containerSupport; + public static Map launcherEnvironment = new HashMap<>(); + public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); @@ -98,11 +101,9 @@ public static void main(String[] args) { System.exit(1); } - List command = createLaunchCommand(args); - ProcessBuilder pb = new ProcessBuilder(command); + ProcessBuilder pb = new ProcessBuilder(); Path environmentFile = stageDir.resolve("environment.json"); - Map launcherEnvironment = new HashMap<>(); if (Files.isReadable(environmentFile)) { try (Reader reader = Files.newBufferedReader(environmentFile)) { new BundleEnvironmentParser(launcherEnvironment).parseAndRegister(reader); @@ -111,19 +112,7 @@ public static void main(String[] args) { throw new Error("Failed to read bundle-file " + environmentFile, e); } } - - if (useContainer()) { - Path javaHome = getJavaExecutable().getParent().getParent(); - replaceContainerPaths(command, javaHome, rootDir); - - Map mountMapping = mountMappingFor(javaHome, inputDir, outputDir); - if (Files.isDirectory(agentOutputDir)) { - mountMapping.put(agentOutputDir, TargetPath.of(agentOutputDir, false)); - } - - containerSupport.initializeContainerImage(); - command.addAll(0, containerSupport.createContainerCommand(launcherEnvironment, mountMapping)); - } + pb.command(createLaunchCommand(args)); if (verbose) { List environmentList = pb.environment() @@ -134,7 +123,7 @@ public static void main(String[] args) { .toList(); System.out.println("Executing ["); System.out.println(String.join(" \\\n", environmentList)); - System.out.println(String.join(" \\\n", command)); + System.out.println(String.join(" \\\n", pb.command())); System.out.println("]"); } @@ -166,7 +155,21 @@ private static List createLaunchCommand(String[] args) { List command = new ArrayList<>(); Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); - command.add(javaExecutable.toString()); + + if (useContainer()) { + Path javaHome = javaExecutable.getParent().getParent(); + + Map mountMapping = mountMappingFor(javaHome, inputDir, outputDir); + if (Files.isDirectory(agentOutputDir)) { + mountMapping.put(agentOutputDir, TargetPath.of(agentOutputDir, false)); + } + + containerSupport.initializeContainerImage(); + command.addAll(containerSupport.createContainerCommand(launcherEnvironment, mountMapping)); + command.add(CONTAINER_GRAAL_VM_HOME.resolve(javaHome.relativize(javaExecutable)).toString()); + } else { + command.add(javaExecutable.toString()); + } List applicationArgs = new ArrayList<>(); List launchArgs = parseBundleLauncherArgs(args, applicationArgs); @@ -176,6 +179,7 @@ private static List createLaunchCommand(String[] args) { if (Files.isDirectory(classPathDir)) { try (Stream walk = Files.walk(classPathDir, 1)) { walk.filter(path -> path.toString().endsWith(".jar") || Files.isDirectory(path)) + .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) .map(Path::toString) .forEach(classpath::add); } catch (IOException e) { @@ -190,8 +194,8 @@ private static List createLaunchCommand(String[] args) { List modulePath = new ArrayList<>(); if (Files.isDirectory(modulePathDir)) { try (Stream walk = Files.walk(modulePathDir, 1)) { - walk.filter(Files::isDirectory) - .filter(path -> !path.equals(modulePathDir)) + walk.filter(path -> Files.isDirectory(path) && !path.equals(modulePathDir)) + .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) .map(Path::toString) .forEach(modulePath::add); } catch (IOException e) { From 37ff3f9fef7c4ce8c1732b4fd8d917cc5b8e86d6 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 22 Jun 2023 13:44:55 +0200 Subject: [PATCH 39/61] Fix command creation for container bundle execution --- .../svm/driver/launcher/BundleLauncher.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 660ad260fc25..8964a275387a 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -86,7 +86,9 @@ public class BundleLauncher { public static ContainerSupport containerSupport; - public static Map launcherEnvironment = new HashMap<>(); + private static final List launchArgs = new ArrayList<>(); + private static final List applicationArgs = new ArrayList<>(); + private static final Map launcherEnvironment = new HashMap<>(); public static void main(String[] args) { @@ -101,6 +103,8 @@ public static void main(String[] args) { System.exit(1); } + parseBundleLauncherArgs(args); + ProcessBuilder pb = new ProcessBuilder(); Path environmentFile = stageDir.resolve("environment.json"); @@ -112,7 +116,7 @@ public static void main(String[] args) { throw new Error("Failed to read bundle-file " + environmentFile, e); } } - pb.command(createLaunchCommand(args)); + pb.command(createLaunchCommand()); if (verbose) { List environmentList = pb.environment() @@ -151,7 +155,7 @@ public static boolean useContainer() { return containerSupport != null; } - private static List createLaunchCommand(String[] args) { + private static List createLaunchCommand() { List command = new ArrayList<>(); Path javaExecutable = getJavaExecutable().toAbsolutePath().normalize(); @@ -162,6 +166,7 @@ private static List createLaunchCommand(String[] args) { Map mountMapping = mountMappingFor(javaHome, inputDir, outputDir); if (Files.isDirectory(agentOutputDir)) { mountMapping.put(agentOutputDir, TargetPath.of(agentOutputDir, false)); + launcherEnvironment.put("LD_LIBRARY_PATH", CONTAINER_GRAAL_VM_HOME.resolve("lib").toString()); } containerSupport.initializeContainerImage(); @@ -171,8 +176,6 @@ private static List createLaunchCommand(String[] args) { command.add(javaExecutable.toString()); } - List applicationArgs = new ArrayList<>(); - List launchArgs = parseBundleLauncherArgs(args, applicationArgs); command.addAll(launchArgs); List classpath = new ArrayList<>(); @@ -249,9 +252,8 @@ private static int updateBundle() { } } - private static List parseBundleLauncherArgs(String[] args, List applicationArgs) { + private static void parseBundleLauncherArgs(String[] args) { Deque argQueue = new ArrayDeque<>(Arrays.asList(args)); - List launchArgs = new ArrayList<>(); while (!argQueue.isEmpty()) { String arg = argQueue.removeFirst(); @@ -316,8 +318,6 @@ private static List parseBundleLauncherArgs(String[] args, List } } } - - return launchArgs; } private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); From a9f2eb48215a18c27b537420bd38ca9adbef23bc Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Tue, 4 Jul 2023 15:21:25 +0200 Subject: [PATCH 40/61] Refactoring for containerized builds, Adjust access modifier, Fix skipping containerized builds when not on Linux --- .../svm/driver/launcher/BundleLauncher.java | 81 +++++---- .../driver/launcher/BundleLauncherUtil.java | 6 +- .../svm/driver/launcher/ContainerSupport.java | 106 ++++++------ .../com/oracle/svm/driver/BundleSupport.java | 154 ++++++++---------- .../com/oracle/svm/driver/NativeImage.java | 23 ++- 5 files changed, 172 insertions(+), 198 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 8964a275387a..7fe4629de917 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -47,26 +47,12 @@ import java.util.jar.JarFile; import java.util.stream.Stream; -import static com.oracle.svm.driver.launcher.ContainerSupport.CONTAINER_GRAAL_VM_HOME; -import static com.oracle.svm.driver.launcher.ContainerSupport.replaceContainerPaths; -import static com.oracle.svm.driver.launcher.ContainerSupport.mountMappingFor; -import static com.oracle.svm.driver.launcher.ContainerSupport.TargetPath; - public class BundleLauncher { - public static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; - public static final String BUNDLE_FILE_EXTENSION = ".nib"; - - public static final String INPUT_DIR_NAME = "input"; - public static final String STAGE_DIR_NAME = "stage"; - public static final String AUXILIARY_DIR_NAME = "auxiliary"; - public static final String CLASSES_DIR_NAME = "classes"; - public static final String CLASSPATH_DIR_NAME = "cp"; - public static final String MODULE_PATH_DIR_NAME = "p"; - public static final String OUTPUT_DIR_NAME = "output"; - public static final String IMAGE_PATH_OUTPUT_DIR_NAME = "default"; - public static final String AUXILIARY_OUTPUT_DIR_NAME = "other"; + static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; + private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; + private static final String BUNDLE_FILE_EXTENSION = ".nib"; private static Path rootDir; private static Path inputDir; @@ -82,9 +68,9 @@ public class BundleLauncher { private static String newBundleName = null; private static boolean updateBundle = false; - public static boolean verbose = false; + private static boolean verbose = false; - public static ContainerSupport containerSupport; + private static ContainerSupport containerSupport; private static final List launchArgs = new ArrayList<>(); private static final List applicationArgs = new ArrayList<>(); @@ -94,12 +80,12 @@ public class BundleLauncher { public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); - agentOutputDir = bundleFilePath.getParent().resolve(Paths.get(bundleName + "." + OUTPUT_DIR_NAME, "launcher")); + agentOutputDir = bundleFilePath.getParent().resolve(Paths.get(bundleName + ".output", "launcher")); unpackBundle(bundleFilePath); // if we did not create a run.json bundle is not executable, e.g. shared library bundles if (!Files.exists(stageDir.resolve("run.json"))) { - System.out.println("Bundle " + bundleFilePath + " is not executable!"); + showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Bundle " + bundleFilePath + " is not executable!"); System.exit(1); } @@ -125,10 +111,10 @@ public static void main(String[] args) { .map(e -> e.getKey() + "=" + e.getValue()) .sorted() .toList(); - System.out.println("Executing ["); - System.out.println(String.join(" \\\n", environmentList)); - System.out.println(String.join(" \\\n", pb.command())); - System.out.println("]"); + showMessage("Executing ["); + showMessage(String.join(" \\\n", environmentList)); + showMessage(String.join(" \\\n", pb.command())); + showMessage("]"); } Process p = null; @@ -151,7 +137,7 @@ public static void main(String[] args) { System.exit(exitCode); } - public static boolean useContainer() { + private static boolean useContainer() { return containerSupport != null; } @@ -163,15 +149,15 @@ private static List createLaunchCommand() { if (useContainer()) { Path javaHome = javaExecutable.getParent().getParent(); - Map mountMapping = mountMappingFor(javaHome, inputDir, outputDir); + Map mountMapping = ContainerSupport.mountMappingFor(javaHome, inputDir, outputDir); if (Files.isDirectory(agentOutputDir)) { - mountMapping.put(agentOutputDir, TargetPath.of(agentOutputDir, false)); - launcherEnvironment.put("LD_LIBRARY_PATH", CONTAINER_GRAAL_VM_HOME.resolve("lib").toString()); + mountMapping.put(agentOutputDir, ContainerSupport.TargetPath.of(agentOutputDir, false)); + launcherEnvironment.put("LD_LIBRARY_PATH", ContainerSupport.GRAAL_VM_HOME.resolve("lib").toString()); } - containerSupport.initializeContainerImage(); - command.addAll(containerSupport.createContainerCommand(launcherEnvironment, mountMapping)); - command.add(CONTAINER_GRAAL_VM_HOME.resolve(javaHome.relativize(javaExecutable)).toString()); + containerSupport.initializeImage(); + command.addAll(containerSupport.createCommand(launcherEnvironment, mountMapping)); + command.add(ContainerSupport.GRAAL_VM_HOME.resolve(javaHome.relativize(javaExecutable)).toString()); } else { command.add(javaExecutable.toString()); } @@ -274,14 +260,16 @@ private static void parseBundleLauncherArgs(String[] args) { Path outputDir = agentOutputDir.resolve(Paths.get("META-INF", "native-image", bundleName + "-agent")); try { Files.createDirectories(outputDir); - System.out.println("Native image agent output written to " + agentOutputDir); + showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Native image agent output written to " + agentOutputDir); launchArgs.add("-agentlib:native-image-agent=config-output-dir=" + outputDir); } catch (IOException e) { - System.out.println("Failed to create native image agent output dir"); + throw new Error("Failed to create native image agent output dir"); } } else if (arg.startsWith("--container")) { if (useContainer()) { throw new Error("native-image launcher allows option container to be specified only once."); + } else if (!System.getProperty("os.name").equals("Linux")) { + throw new Error("container option is only supported for Linux"); } Path dockerfile; if (arg.indexOf(',') != -1) { @@ -303,9 +291,9 @@ private static void parseBundleLauncherArgs(String[] args) { } else { dockerfile = stageDir.resolve("Dockerfile"); } - containerSupport = new ContainerSupport(dockerfile, stageDir); + containerSupport = new ContainerSupport(dockerfile, stageDir, Error::new, BundleLauncher::showWarning, BundleLauncher::showMessage); if (arg.indexOf('=') != -1) { - containerSupport.containerTool = arg.substring(arg.indexOf('=') + 1); + containerSupport.tool = arg.substring(arg.indexOf('=') + 1); } } else { switch (arg) { @@ -320,6 +308,13 @@ private static void parseBundleLauncherArgs(String[] args) { } } + private static void showMessage(String msg) { + System.out.println(msg); + } + private static void showWarning(String msg) { + System.out.println("Warning: " + msg); + } + private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); private static Path getJavaExecutable() { @@ -388,7 +383,7 @@ private static void deleteAllFiles(Path toDelete) { walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); } } catch (IOException e) { - System.out.println("Could not recursively delete path: " + toDelete); + showMessage("Could not recursively delete path: " + toDelete); e.printStackTrace(); } } @@ -396,7 +391,7 @@ private static void deleteAllFiles(Path toDelete) { private static void unpackBundle(Path bundleFilePath) { try { rootDir = createBundleRootDir(); - inputDir = rootDir.resolve(INPUT_DIR_NAME); + inputDir = rootDir.resolve("input"); try (JarFile archive = new JarFile(bundleFilePath.toFile())) { Enumeration jarEntries = archive.entries(); @@ -424,11 +419,11 @@ private static void unpackBundle(Path bundleFilePath) { } try { - stageDir = Files.createDirectories(inputDir.resolve(STAGE_DIR_NAME)); - Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); - classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); - modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); - outputDir = Files.createDirectories(rootDir.resolve(OUTPUT_DIR_NAME)); + stageDir = Files.createDirectories(inputDir.resolve("stage")); + Path classesDir = inputDir.resolve("classes"); + classPathDir = Files.createDirectories(classesDir.resolve("cp")); + modulePathDir = Files.createDirectories(classesDir.resolve("p")); + outputDir = Files.createDirectories(rootDir.resolve("output")); } catch (IOException e) { throw new Error("Unable to create bundle directory layout", e); } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java index 5e413d36e4dd..893824620b20 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java @@ -33,7 +33,7 @@ public class BundleLauncherUtil { private static final char[] HEX = "0123456789abcdef".toCharArray(); - public static String digest(String value) { + static String digest(String value) { try { MessageDigest md = MessageDigest.getInstance("SHA-1"); md.update(value.getBytes(StandardCharsets.UTF_8)); @@ -42,7 +42,7 @@ public static String digest(String value) { throw new Error(ex); } } - public static String toHex(byte[] data) { + static String toHex(byte[] data) { StringBuilder r = new StringBuilder(data.length * 2); for (byte b : data) { r.append(HEX[(b >> 4) & 0xf]); @@ -52,7 +52,7 @@ public static String toHex(byte[] data) { } private static final Pattern SAFE_SHELL_ARG = Pattern.compile("[A-Za-z0-9@%_\\-+=:,./]+"); - public static String quoteShellArg(String arg) { + static String quoteShellArg(String arg) { if (arg.isEmpty()) { return "''"; } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java index dd7f2f3373be..87257ab00b5f 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java @@ -42,31 +42,25 @@ import java.util.function.Consumer; public class ContainerSupport { - public String containerTool; - public String bundleContainerTool; - public String containerToolVersion; - public String bundleContainerToolVersion; - public String containerImage; - public String bundleContainerImage; + public String tool; + public String bundleTool; + public String toolVersion; + public String bundleToolVersion; + public String image; + public String bundleImage; public Path dockerfile; - public static final List SUPPORTED_CONTAINER_TOOLS = List.of("podman", "docker"); - public static final String CONTAINER_TOOL_JSON_KEY = "containerTool"; - public static final String CONTAINER_TOOL_VERSION_JSON_KEY = "containerToolVersion"; - public static final String CONTAINER_IMAGE_JSON_KEY = "containerImage"; + public static final List SUPPORTED_TOOLS = List.of("podman", "docker"); + public static final String TOOL_JSON_KEY = "containerTool"; + public static final String TOOL_VERSION_JSON_KEY = "containerToolVersion"; + public static final String IMAGE_JSON_KEY = "containerImage"; - private static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; - - public static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); + public static final Path GRAAL_VM_HOME = Path.of("/graalvm"); private final BiFunction errorFunction; private final Consumer warningPrinter; private final Consumer messagePrinter; - public ContainerSupport(Path dockerfile, Path bundleStageDir) { - this(dockerfile, bundleStageDir, Error::new, (msg) -> System.out.println("Warning: " + msg), System.out::println); - } - public ContainerSupport(Path dockerfile, Path bundleStageDir, BiFunction errorFunction, Consumer warningPrinter, Consumer messagePrinter) { this.dockerfile = dockerfile; this.errorFunction = errorFunction; @@ -79,64 +73,64 @@ public ContainerSupport(Path dockerfile, Path bundleStageDir, BiFunction containerSettings = new HashMap<>(); new BundleContainerSettingsParser(containerSettings).parseAndRegister(reader); - bundleContainerImage = containerSettings.getOrDefault(CONTAINER_IMAGE_JSON_KEY, bundleContainerImage); - bundleContainerTool = containerSettings.getOrDefault(CONTAINER_TOOL_JSON_KEY, bundleContainerTool); - bundleContainerToolVersion = containerSettings.getOrDefault(CONTAINER_TOOL_VERSION_JSON_KEY, bundleContainerToolVersion); + bundleImage = containerSettings.getOrDefault(IMAGE_JSON_KEY, bundleImage); + bundleTool = containerSettings.getOrDefault(TOOL_JSON_KEY, bundleTool); + bundleToolVersion = containerSettings.getOrDefault(TOOL_VERSION_JSON_KEY, bundleToolVersion); } catch (IOException e) { throw errorFunction.apply("Failed to read bundle-file " + containerFile, e); } - if (bundleContainerTool != null) { - String containerToolVersionString = bundleContainerToolVersion == null ? "" : String.format(" (%s)", bundleContainerToolVersion); - messagePrinter.accept(String.format("%sBundled native-image was created in a container with %s%s.%n", BUNDLE_INFO_MESSAGE_PREFIX, bundleContainerTool, containerToolVersionString)); + if (bundleTool != null) { + String containerToolVersionString = bundleToolVersion == null ? "" : String.format(" (%s)", bundleToolVersion); + messagePrinter.accept(String.format("%sBundled native-image was created in a container with %s%s.%n", BundleLauncher.BUNDLE_INFO_MESSAGE_PREFIX, bundleTool, containerToolVersionString)); } } } } - public int initializeContainerImage() { + public int initializeImage() { try { - containerImage = BundleLauncherUtil.digest(Files.readString(dockerfile)); + image = BundleLauncherUtil.digest(Files.readString(dockerfile)); } catch (IOException e) { throw errorFunction.apply("Could not read Dockerfile " + dockerfile, e); } - if (bundleContainerImage != null && !bundleContainerImage.equals(containerImage)) { + if (bundleImage != null && !bundleImage.equals(image)) { warningPrinter.accept("The bundled image was created with a different dockerfile."); } - if (bundleContainerTool != null && containerTool == null) { - containerTool = bundleContainerTool; + if (bundleTool != null && tool == null) { + tool = bundleTool; } - if (containerTool != null) { - if (!isToolAvailable(containerTool)) { + if (tool != null) { + if (!isToolAvailable(tool)) { throw errorFunction.apply("Configured container tool not available.", null); - } else if (containerTool.equals("docker") && !isRootlessDocker()) { + } else if (tool.equals("docker") && !isRootlessDocker()) { throw errorFunction.apply("Only rootless docker is supported for containerized builds.", null); } - containerToolVersion = getContainerToolVersion(containerTool); + toolVersion = getToolVersion(tool); - if (bundleContainerTool != null) { - if (!containerTool.equals(bundleContainerTool)) { - warningPrinter.accept(String.format("The bundled image was created with container tool '%s' (using '%s').%n", bundleContainerTool, containerTool)); - } else if (containerToolVersion != null && bundleContainerToolVersion != null && !containerToolVersion.equals(bundleContainerToolVersion)) { - warningPrinter.accept(String.format("The bundled image was created with different %s version '%s' (installed '%s').%n", containerTool, bundleContainerToolVersion, containerToolVersion)); + if (bundleTool != null) { + if (!tool.equals(bundleTool)) { + warningPrinter.accept(String.format("The bundled image was created with container tool '%s' (using '%s').%n", bundleTool, tool)); + } else if (toolVersion != null && bundleToolVersion != null && !toolVersion.equals(bundleToolVersion)) { + warningPrinter.accept(String.format("The bundled image was created with different %s version '%s' (installed '%s').%n", tool, bundleToolVersion, toolVersion)); } } } else { - for (String tool : SUPPORTED_CONTAINER_TOOLS) { + for (String tool : SUPPORTED_TOOLS) { if (isToolAvailable(tool)) { if (tool.equals("docker") && !isRootlessDocker()) { - messagePrinter.accept(BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); + messagePrinter.accept(BundleLauncher.BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); continue; } - containerTool = tool; - containerToolVersion = getContainerToolVersion(tool); + this.tool = tool; + toolVersion = getToolVersion(tool); break; } } - if (containerTool == null) { - throw errorFunction.apply(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_CONTAINER_TOOLS), null); + if (tool == null) { + throw errorFunction.apply(String.format("Please install one of the following tools before running containerized native image builds: %s", SUPPORTED_TOOLS), null); } } @@ -144,14 +138,14 @@ public int initializeContainerImage() { } private int createContainer() { - ProcessBuilder pbCheckForImage = new ProcessBuilder(containerTool, "images", "-q", containerImage + ":latest"); - ProcessBuilder pb = new ProcessBuilder(containerTool, "build", "-f", dockerfile.toString(), "-t", containerImage, "."); + ProcessBuilder pbCheckForImage = new ProcessBuilder(tool, "images", "-q", image + ":latest"); + ProcessBuilder pb = new ProcessBuilder(tool, "build", "-f", dockerfile.toString(), "-t", image, "."); String imageId = getFirstProcessResultLine(pbCheckForImage); if (imageId == null) { pb.inheritIO(); } else { - messagePrinter.accept(String.format("%sReusing container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); + messagePrinter.accept(String.format("%sReusing container image %s.%n", BundleLauncher.BUNDLE_INFO_MESSAGE_PREFIX, image)); } Process p = null; @@ -160,7 +154,7 @@ private int createContainer() { int status = p.waitFor(); if (status == 0 && imageId != null && !imageId.equals(getFirstProcessResultLine(pbCheckForImage))) { try (var processResult = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - messagePrinter.accept(String.format("%sUpdated container image %s.%n", BUNDLE_INFO_MESSAGE_PREFIX, containerImage)); + messagePrinter.accept(String.format("%sUpdated container image %s.%n", BundleLauncher.BUNDLE_INFO_MESSAGE_PREFIX, image)); processResult.lines().forEach(messagePrinter); } } @@ -180,7 +174,7 @@ private boolean isToolAvailable(String tool) { .anyMatch(Files::isExecutable); } - private String getContainerToolVersion(String tool) { + private String getToolVersion(String tool) { ProcessBuilder pb = new ProcessBuilder(tool, "--version"); return getFirstProcessResultLine(pb); } @@ -220,17 +214,17 @@ public static TargetPath of(Path target, boolean readonly) { public static Map mountMappingFor(Path javaHome, Path inputDir, Path outputDir) { Map mountMapping = new HashMap<>(); Path containerRoot = Paths.get("/"); - mountMapping.put(javaHome, TargetPath.readonly(containerRoot.resolve(CONTAINER_GRAAL_VM_HOME))); - mountMapping.put(inputDir, ContainerSupport.TargetPath.readonly(containerRoot.resolve(BundleLauncher.INPUT_DIR_NAME))); - mountMapping.put(outputDir, ContainerSupport.TargetPath.of(containerRoot.resolve(BundleLauncher.OUTPUT_DIR_NAME), false)); + mountMapping.put(javaHome, TargetPath.readonly(containerRoot.resolve(GRAAL_VM_HOME))); + mountMapping.put(inputDir, ContainerSupport.TargetPath.readonly(containerRoot.resolve("input"))); + mountMapping.put(outputDir, ContainerSupport.TargetPath.of(containerRoot.resolve("output"), false)); return mountMapping; } - public List createContainerCommand(Map containerEnvironment, Map mountMapping) { + public List createCommand(Map containerEnvironment, Map mountMapping) { List containerCommand = new ArrayList<>(); // run docker tool without network access and remove container after image build is finished - containerCommand.add(containerTool); + containerCommand.add(tool); containerCommand.add("run"); containerCommand.add("--network=none"); containerCommand.add("--rm"); @@ -255,14 +249,14 @@ public List createContainerCommand(Map containerEnvironm }); // specify container name - containerCommand.add(containerImage); + containerCommand.add(image); return containerCommand; } - public static void replaceContainerPaths(List arguments, Path javaHome, Path bundleRoot) { + public static void replacePaths(List arguments, Path javaHome, Path bundleRoot) { arguments.replaceAll(arg -> arg - .replace(javaHome.toString(), CONTAINER_GRAAL_VM_HOME.toString()) + .replace(javaHome.toString(), GRAAL_VM_HOME.toString()) .replace(bundleRoot.toString(), "") ); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 197e5c6b4cab..5e5f3204d7f5 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -80,16 +80,6 @@ import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.StringUtil; -import static com.oracle.svm.driver.launcher.BundleLauncher.AUXILIARY_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.AUXILIARY_OUTPUT_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.CLASSES_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.CLASSPATH_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.IMAGE_PATH_OUTPUT_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.INPUT_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.MODULE_PATH_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.OUTPUT_DIR_NAME; -import static com.oracle.svm.driver.launcher.BundleLauncher.STAGE_DIR_NAME; - final class BundleSupport { final NativeImage nativeImage; @@ -118,7 +108,7 @@ final class BundleSupport { private static final int BUNDLE_FILE_FORMAT_VERSION_MINOR = 9; private static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; - private static final String BUNDLE_TEMP_DIR_PREFIX = BundleLauncher.BUNDLE_TEMP_DIR_PREFIX; + private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; private static final String ORIGINAL_DIR_EXTENSION = ".orig"; private Path bundlePath; @@ -127,14 +117,18 @@ final class BundleSupport { private final BundleProperties bundleProperties; static final String BUNDLE_OPTION = "--bundle"; - static final String BUNDLE_FILE_EXTENSION = BundleLauncher.BUNDLE_FILE_EXTENSION; + static final String BUNDLE_FILE_EXTENSION = ".nib"; + + ContainerSupport containerSupport; + boolean useContainer; - public ContainerSupport containerSupport; + private static final String DEFAULT_DOCKERFILE = getDockerfile("Dockerfile"); + private static final String DEFAULT_DOCKERFILE_MUSLIB = getDockerfile("Dockerfile_muslib_extension"); - static final Path CONTAINER_GRAAL_VM_HOME = Path.of("/graalvm"); - private static final String DEFAULT_DOCKERFILE = NativeImage.getResource("/container-default/Dockerfile"); - private static final String DEFAULT_DOCKERFILE_MUSLIB = NativeImage.getResource("/container-default/Dockerfile_muslib_extension"); + private static String getDockerfile(String name) { + return NativeImage.getResource("/container-default/" + name); + } enum BundleOptionVariants { create(), @@ -156,20 +150,18 @@ static ExtendedBundleOptions get(String name) { @Override public String toString() { - return super.toString().replace('_', '-'); + return name().replace('_', '-'); } } static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { try { - String variant = bundleArg.substring(BUNDLE_OPTION.length() + 1); String bundleFilename = null; - String[] options = SubstrateUtil.split(variant, ","); + String[] options = SubstrateUtil.split(bundleArg.substring(BUNDLE_OPTION.length() + 1), ","); - variant = options[0]; - String[] variantParts = SubstrateUtil.split(variant, "=", 2); + String[] variantParts = SubstrateUtil.split(options[0], "=", 2); + String variant = variantParts[0]; if (variantParts.length == 2) { - variant = variantParts[0]; bundleFilename = variantParts[1]; } String applyOptionName = BundleOptionVariants.apply.optionName(); @@ -221,34 +213,33 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma .skip(1) .forEach(bundleSupport::parseExtendedOption); - if (bundleSupport.useContainer()) { - if (OS.LINUX.isCurrent()) { - if (nativeImage.isDryRun()) { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); - } else { - if (bundleSupport.containerSupport.dockerfile == null) { - bundleSupport.containerSupport.dockerfile = bundleSupport.createDockerfile(); + if (bundleSupport.useContainer) { + if (!OS.LINUX.isCurrent()) { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); + bundleSupport.useContainer = false; + } else if (nativeImage.isDryRun()) { + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); + } else { + if (bundleSupport.containerSupport.dockerfile == null) { + bundleSupport.containerSupport.dockerfile = bundleSupport.createDockerfile(); + } + int exitStatusCode = bundleSupport.containerSupport.initializeImage(); + switch (ExitStatus.of(exitStatusCode)) { + case OK -> { } - int exitStatusCode = bundleSupport.containerSupport.initializeContainerImage(); - switch (ExitStatus.of(exitStatusCode)) { - case OK -> { } - case BUILDER_ERROR -> - /* Exit, builder has handled error reporting. */ - throw NativeImage.showError(null, null, exitStatusCode); - case OUT_OF_MEMORY -> { - nativeImage.showOutOfMemoryWarning(); + case BUILDER_ERROR -> + /* Exit, builder has handled error reporting. */ throw NativeImage.showError(null, null, exitStatusCode); - } - default -> { - String message = String.format("Container build request for '%s' failed with exit status %d", - nativeImage.imageName, exitStatusCode); - throw NativeImage.showError(message, null, exitStatusCode); - } + case OUT_OF_MEMORY -> { + nativeImage.showOutOfMemoryWarning(); + throw NativeImage.showError(null, null, exitStatusCode); + } + default -> { + String message = String.format("Container build request for '%s' failed with exit status %d", + nativeImage.imageName, exitStatusCode); + throw NativeImage.showError(message, null, exitStatusCode); } } - } else { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); - //bundleSupport.useContainer = false; } } @@ -260,10 +251,6 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } - public boolean useContainer() { - return containerSupport != null; - } - private Path createDockerfile() { // take Dockerfile from bundle or create default if not available Path dockerfile = stageDir.resolve("Dockerfile"); @@ -298,19 +285,20 @@ private void parseExtendedOption(String option) { switch (ExtendedBundleOptions.get(optionKey)) { case dry_run -> nativeImage.setDryRun(true); case container -> { - if (useContainer()) { + if (containerSupport != null) { throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); } containerSupport = new ContainerSupport(null, stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); + useContainer = true; if (optionValue != null) { - if (!ContainerSupport.SUPPORTED_CONTAINER_TOOLS.contains(optionValue)) { - throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, ContainerSupport.SUPPORTED_CONTAINER_TOOLS)); + if (!ContainerSupport.SUPPORTED_TOOLS.contains(optionValue)) { + throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, ContainerSupport.SUPPORTED_TOOLS)); } - containerSupport.containerTool = optionValue; + containerSupport.tool = optionValue; } } case dockerfile -> { - if (!useContainer()) { + if (containerSupport == null) { throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, ExtendedBundleOptions.container)); } if (optionValue != null) { @@ -342,15 +330,15 @@ private BundleSupport(NativeImage nativeImage) { bundleProperties = new BundleProperties(); bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); - inputDir = rootDir.resolve(INPUT_DIR_NAME); - stageDir = Files.createDirectories(inputDir.resolve(STAGE_DIR_NAME)); - auxiliaryDir = Files.createDirectories(inputDir.resolve(AUXILIARY_DIR_NAME)); - Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); - classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); - modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); - outputDir = rootDir.resolve(OUTPUT_DIR_NAME); - imagePathOutputDir = Files.createDirectories(outputDir.resolve(IMAGE_PATH_OUTPUT_DIR_NAME)); - auxiliaryOutputDir = Files.createDirectories(outputDir.resolve(AUXILIARY_OUTPUT_DIR_NAME)); + inputDir = rootDir.resolve("input"); + stageDir = Files.createDirectories(inputDir.resolve("stage")); + auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); + Path classesDir = inputDir.resolve("classes"); + classPathDir = Files.createDirectories(classesDir.resolve("cp")); + modulePathDir = Files.createDirectories(classesDir.resolve("p")); + outputDir = rootDir.resolve("output"); + imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); + auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } @@ -373,7 +361,7 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { bundleProperties = new BundleProperties(); bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); - outputDir = rootDir.resolve(OUTPUT_DIR_NAME); + outputDir = rootDir.resolve("output"); String originalOutputDirName = outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); @@ -411,14 +399,14 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { nativeImage.config.modulePathBuild = !forceBuilderOnClasspath; try { - inputDir = rootDir.resolve(INPUT_DIR_NAME); - stageDir = Files.createDirectories(inputDir.resolve(STAGE_DIR_NAME)); - auxiliaryDir = Files.createDirectories(inputDir.resolve(AUXILIARY_DIR_NAME)); - Path classesDir = inputDir.resolve(CLASSES_DIR_NAME); - classPathDir = Files.createDirectories(classesDir.resolve(CLASSPATH_DIR_NAME)); - modulePathDir = Files.createDirectories(classesDir.resolve(MODULE_PATH_DIR_NAME)); - imagePathOutputDir = Files.createDirectories(outputDir.resolve(IMAGE_PATH_OUTPUT_DIR_NAME)); - auxiliaryOutputDir = Files.createDirectories(outputDir.resolve(AUXILIARY_OUTPUT_DIR_NAME)); + inputDir = rootDir.resolve("input"); + stageDir = Files.createDirectories(inputDir.resolve("stage")); + auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); + Path classesDir = inputDir.resolve("classes"); + classPathDir = Files.createDirectories(classesDir.resolve("cp")); + modulePathDir = Files.createDirectories(classesDir.resolve("p")); + imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); + auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } @@ -787,16 +775,16 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + environmentFile, e); } - if (useContainer()) { + if (containerSupport != null) { Map containerInfo = new HashMap<>(); - if (containerSupport.containerImage != null) { - containerInfo.put(ContainerSupport.CONTAINER_IMAGE_JSON_KEY, containerSupport.containerImage); + if (containerSupport.image != null) { + containerInfo.put(ContainerSupport.IMAGE_JSON_KEY, containerSupport.image); } - if (containerSupport.containerTool != null) { - containerInfo.put(ContainerSupport.CONTAINER_TOOL_JSON_KEY, containerSupport.containerTool); + if (containerSupport.tool != null) { + containerInfo.put(ContainerSupport.TOOL_JSON_KEY, containerSupport.tool); } - if (containerSupport.containerToolVersion != null) { - containerInfo.put(ContainerSupport.CONTAINER_TOOL_VERSION_JSON_KEY, containerSupport.containerToolVersion); + if (containerSupport.toolVersion != null) { + containerInfo.put(ContainerSupport.TOOL_VERSION_JSON_KEY, containerSupport.toolVersion); } if (!containerInfo.isEmpty()) { @@ -811,10 +799,10 @@ private Path writeBundle() { Path dockerfilePath = stageDir.resolve("Dockerfile"); try { - if ((!useContainer() || containerSupport.dockerfile == null) && !Files.exists(dockerfilePath)) { + if ((containerSupport == null || containerSupport.dockerfile == null) && !Files.exists(dockerfilePath)) { // if no Dockerfile was created yet create a new default Dockerfile createDockerfile(); - } else if (useContainer() && containerSupport.dockerfile != null && !dockerfilePath.equals(containerSupport.dockerfile)) { + } else if (containerSupport != null && containerSupport.dockerfile != null && !dockerfilePath.equals(containerSupport.dockerfile)) { Files.copy(containerSupport.dockerfile, dockerfilePath); } } catch (IOException e) { @@ -1019,7 +1007,7 @@ private void write() { boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); if (imageBuilt) { - properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(useContainer())); + properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(useContainer)); } properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, NativeImage.platform); properties.put(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, System.getProperty("java.vm.vendor")); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 03a250670999..c1ab06a378c5 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -69,6 +69,7 @@ import java.util.stream.Stream; import com.oracle.svm.core.option.OptionOrigin; +import com.oracle.svm.driver.launcher.ContainerSupport; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.nativeimage.Platform; @@ -102,10 +103,6 @@ import com.oracle.svm.util.ReflectionUtil; import com.oracle.svm.util.StringUtil; -import static com.oracle.svm.driver.launcher.ContainerSupport.replaceContainerPaths; -import static com.oracle.svm.driver.launcher.ContainerSupport.mountMappingFor; -import static com.oracle.svm.driver.launcher.ContainerSupport.TargetPath; - public class NativeImage { private static final String DEFAULT_GENERATOR_CLASS_NAME = NativeImageGeneratorRunner.class.getName(); @@ -1564,22 +1561,22 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa List command = new ArrayList<>(); List completeCommandList = new ArrayList<>(); - if (useBundle() && bundleSupport.useContainer()) { - replaceContainerPaths(arguments, config.getJavaHome(), bundleSupport.rootDir); - replaceContainerPaths(finalImageBuilderArgs, config.getJavaHome(), bundleSupport.rootDir); + if (useBundle() && bundleSupport.useContainer) { + ContainerSupport.replacePaths(arguments, config.getJavaHome(), bundleSupport.rootDir); + ContainerSupport.replacePaths(finalImageBuilderArgs, config.getJavaHome(), bundleSupport.rootDir); Path binJava = Paths.get("bin", "java"); - javaExecutable = BundleSupport.CONTAINER_GRAAL_VM_HOME.resolve(binJava).toString(); + javaExecutable = ContainerSupport.GRAAL_VM_HOME.resolve(binJava).toString(); } Path argFile = createVMInvocationArgumentFile(arguments); Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); - if (useBundle() && bundleSupport.useContainer()) { - Map mountMapping = mountMappingFor(config.getJavaHome(), bundleSupport.inputDir, bundleSupport.outputDir); - mountMapping.put(argFile, TargetPath.readonly(argFile)); - mountMapping.put(builderArgFile, TargetPath.readonly(builderArgFile)); + if (useBundle() && bundleSupport.useContainer) { + Map mountMapping = ContainerSupport.mountMappingFor(config.getJavaHome(), bundleSupport.inputDir, bundleSupport.outputDir); + mountMapping.put(argFile, ContainerSupport.TargetPath.readonly(argFile)); + mountMapping.put(builderArgFile, ContainerSupport.TargetPath.readonly(builderArgFile)); - List containerCommand = bundleSupport.containerSupport.createContainerCommand(imageBuilderEnvironment, mountMapping); + List containerCommand = bundleSupport.containerSupport.createCommand(imageBuilderEnvironment, mountMapping); command.addAll(containerCommand); completeCommandList.addAll(containerCommand); } From efbe842c8dec62790c3d39fe6e5586ef78e9d4f5 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 5 Jul 2023 11:20:21 +0200 Subject: [PATCH 41/61] Add GPL License Header for Dockerfiles --- .../resources/container-default/Dockerfile | 23 +++++++++++++++++++ .../Dockerfile_muslib_extension | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile index 9cd85d1a955d..7c15c77f6174 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile +++ b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile @@ -1,3 +1,26 @@ +# Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Oracle designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. + ARG BASE_IMAGE=container-registry.oracle.com/os/oraclelinux:8-slim FROM ${BASE_IMAGE} as base diff --git a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension index a52d7d1ca374..6d49a1a43fdb 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension +++ b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension @@ -1,3 +1,26 @@ +# Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Oracle designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. + FROM base as muslib ARG TEMP_REGION="" From 1e9fadc865dd31e7bd57272be22c162654074539 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 5 Jul 2023 11:21:33 +0200 Subject: [PATCH 42/61] Fix style issues --- .../svm/driver/launcher/BundleLauncher.java | 10 +++++----- .../svm/driver/launcher/ContainerSupport.java | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 7fe4629de917..d226eb827972 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -81,7 +81,7 @@ public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); agentOutputDir = bundleFilePath.getParent().resolve(Paths.get(bundleName + ".output", "launcher")); - unpackBundle(bundleFilePath); + unpackBundle(); // if we did not create a run.json bundle is not executable, e.g. shared library bundles if (!Files.exists(stageDir.resolve("run.json"))) { @@ -257,11 +257,11 @@ private static void parseBundleLauncherArgs(String[] args) { } } - Path outputDir = agentOutputDir.resolve(Paths.get("META-INF", "native-image", bundleName + "-agent")); + Path configOutputDir = agentOutputDir.resolve(Paths.get("META-INF", "native-image", bundleName + "-agent")); try { - Files.createDirectories(outputDir); + Files.createDirectories(configOutputDir); showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Native image agent output written to " + agentOutputDir); - launchArgs.add("-agentlib:native-image-agent=config-output-dir=" + outputDir); + launchArgs.add("-agentlib:native-image-agent=config-output-dir=" + configOutputDir); } catch (IOException e) { throw new Error("Failed to create native image agent output dir"); } @@ -388,7 +388,7 @@ private static void deleteAllFiles(Path toDelete) { } } - private static void unpackBundle(Path bundleFilePath) { + private static void unpackBundle() { try { rootDir = createBundleRootDir(); inputDir = rootDir.resolve("input"); diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java index 87257ab00b5f..d2ab94f17bfb 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java @@ -108,7 +108,7 @@ public int initializeImage() { } else if (tool.equals("docker") && !isRootlessDocker()) { throw errorFunction.apply("Only rootless docker is supported for containerized builds.", null); } - toolVersion = getToolVersion(tool); + toolVersion = getToolVersion(); if (bundleTool != null) { if (!tool.equals(bundleTool)) { @@ -118,14 +118,14 @@ public int initializeImage() { } } } else { - for (String tool : SUPPORTED_TOOLS) { - if (isToolAvailable(tool)) { - if (tool.equals("docker") && !isRootlessDocker()) { + for (String supportedTool : SUPPORTED_TOOLS) { + if (isToolAvailable(supportedTool)) { + if (supportedTool.equals("docker") && !isRootlessDocker()) { messagePrinter.accept(BundleLauncher.BUNDLE_INFO_MESSAGE_PREFIX + "Rootless context missing for docker."); continue; } - this.tool = tool; - toolVersion = getToolVersion(tool); + tool = supportedTool; + toolVersion = getToolVersion(); break; } } @@ -168,13 +168,13 @@ private int createContainer() { } } - private boolean isToolAvailable(String tool) { + private static boolean isToolAvailable(String toolName) { return Arrays.stream(System.getenv("PATH").split(":")) - .map(str -> Path.of(str).resolve(tool)) + .map(str -> Path.of(str).resolve(toolName)) .anyMatch(Files::isExecutable); } - private String getToolVersion(String tool) { + private String getToolVersion() { ProcessBuilder pb = new ProcessBuilder(tool, "--version"); return getFirstProcessResultLine(pb); } From d36d4818279f0b48d39af083ed50e150afdb87ff Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 5 Jul 2023 13:27:23 +0200 Subject: [PATCH 43/61] Fix style issue --- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 5e5f3204d7f5..cefdcdb99834 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -884,7 +884,7 @@ private Path writeBundle() { return bundleFilePath; } - private Manifest createManifest() { + private static Manifest createManifest() { Manifest mf = new Manifest(); Attributes attributes = mf.getMainAttributes(); attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); From f46ad7b359b1c530c05a4cb5fe7fe6aa81b00004 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 5 Jul 2023 14:38:40 +0200 Subject: [PATCH 44/61] Fix formatting issues --- .../svm/driver/launcher/BundleLauncher.java | 33 +- .../driver/launcher/BundleLauncherUtil.java | 3 + .../svm/driver/launcher/ContainerSupport.java | 22 +- .../BundleConfigurationParser.java | 38 +- .../BundleEnvironmentParser.java | 4 +- .../configuration/BundlePathMapParser.java | 4 +- .../launcher/json/BundleJSONParser.java | 710 +++++++++--------- .../com/oracle/svm/driver/BundleSupport.java | 62 +- .../com/oracle/svm/driver/NativeImage.java | 6 +- 9 files changed, 441 insertions(+), 441 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index d226eb827972..72667724c835 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -24,9 +24,6 @@ */ package com.oracle.svm.driver.launcher; -import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; -import com.oracle.svm.driver.launcher.configuration.BundleEnvironmentParser; - import java.io.File; import java.io.IOException; import java.io.Reader; @@ -47,6 +44,8 @@ import java.util.jar.JarFile; import java.util.stream.Stream; +import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; +import com.oracle.svm.driver.launcher.configuration.BundleEnvironmentParser; public class BundleLauncher { @@ -76,7 +75,6 @@ public class BundleLauncher { private static final List applicationArgs = new ArrayList<>(); private static final Map launcherEnvironment = new HashMap<>(); - public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); @@ -106,11 +104,11 @@ public static void main(String[] args) { if (verbose) { List environmentList = pb.environment() - .entrySet() - .stream() - .map(e -> e.getKey() + "=" + e.getValue()) - .sorted() - .toList(); + .entrySet() + .stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .sorted() + .toList(); showMessage("Executing ["); showMessage(String.join(" \\\n", environmentList)); showMessage(String.join(" \\\n", pb.command())); @@ -168,9 +166,9 @@ private static List createLaunchCommand() { if (Files.isDirectory(classPathDir)) { try (Stream walk = Files.walk(classPathDir, 1)) { walk.filter(path -> path.toString().endsWith(".jar") || Files.isDirectory(path)) - .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) - .map(Path::toString) - .forEach(classpath::add); + .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) + .map(Path::toString) + .forEach(classpath::add); } catch (IOException e) { throw new Error("Failed to iterate through directory " + classPathDir, e); } @@ -179,14 +177,13 @@ private static List createLaunchCommand() { command.add(String.join(File.pathSeparator, classpath)); } - List modulePath = new ArrayList<>(); if (Files.isDirectory(modulePathDir)) { try (Stream walk = Files.walk(modulePathDir, 1)) { walk.filter(path -> Files.isDirectory(path) && !path.equals(modulePathDir)) - .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) - .map(Path::toString) - .forEach(modulePath::add); + .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) + .map(Path::toString) + .forEach(modulePath::add); } catch (IOException e) { throw new Error("Failed to iterate through directory " + modulePathDir, e); } @@ -311,6 +308,7 @@ private static void parseBundleLauncherArgs(String[] args) { private static void showMessage(String msg) { System.out.println(msg); } + private static void showWarning(String msg) { System.out.println("Warning: " + msg); } @@ -359,6 +357,7 @@ private static Path getNativeImageExecutable() { } private static final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); + private static Path createBundleRootDir() throws IOException { Path bundleRoot = Files.createTempDirectory(BUNDLE_TEMP_DIR_PREFIX); Runtime.getRuntime().addShutdownHook(new Thread(() -> { @@ -369,9 +368,11 @@ private static Path createBundleRootDir() throws IOException { } private static final String deletedFileSuffix = ".deleted"; + private static boolean isDeletedPath(Path toDelete) { return toDelete.getFileName().toString().endsWith(deletedFileSuffix); } + private static void deleteAllFiles(Path toDelete) { try { Path deletedPath = toDelete; diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java index 893824620b20..3b8f6f9c6318 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncherUtil.java @@ -33,6 +33,7 @@ public class BundleLauncherUtil { private static final char[] HEX = "0123456789abcdef".toCharArray(); + static String digest(String value) { try { MessageDigest md = MessageDigest.getInstance("SHA-1"); @@ -42,6 +43,7 @@ static String digest(String value) { throw new Error(ex); } } + static String toHex(byte[] data) { StringBuilder r = new StringBuilder(data.length * 2); for (byte b : data) { @@ -52,6 +54,7 @@ static String toHex(byte[] data) { } private static final Pattern SAFE_SHELL_ARG = Pattern.compile("[A-Za-z0-9@%_\\-+=:,./]+"); + static String quoteShellArg(String arg) { if (arg.isEmpty()) { return "''"; diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java index d2ab94f17bfb..61430b726228 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.driver.launcher; -import com.oracle.svm.driver.launcher.configuration.BundleContainerSettingsParser; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -41,6 +39,8 @@ import java.util.function.BiFunction; import java.util.function.Consumer; +import com.oracle.svm.driver.launcher.configuration.BundleContainerSettingsParser; + public class ContainerSupport { public String tool; public String bundleTool; @@ -81,7 +81,8 @@ public ContainerSupport(Path dockerfile, Path bundleStageDir, BiFunction Path.of(str).resolve(toolName)) - .anyMatch(Files::isExecutable); + .map(str -> Path.of(str).resolve(toolName)) + .anyMatch(Files::isExecutable); } private String getToolVersion() { @@ -231,9 +232,9 @@ public List createCommand(Map containerEnvironment, Map< // inject environment variables into container containerEnvironment.forEach((key, value) -> { - containerCommand.add("-e"); - containerCommand.add(key + "=" + BundleLauncherUtil.quoteShellArg(value)); - }); + containerCommand.add("-e"); + containerCommand.add(key + "=" + BundleLauncherUtil.quoteShellArg(value)); + }); // mount java home, input and output directories and argument files for native image build mountMapping.forEach((source, target) -> { @@ -256,8 +257,7 @@ public List createCommand(Map containerEnvironment, Map< public static void replacePaths(List arguments, Path javaHome, Path bundleRoot) { arguments.replaceAll(arg -> arg - .replace(javaHome.toString(), GRAAL_VM_HOME.toString()) - .replace(bundleRoot.toString(), "") - ); + .replace(javaHome.toString(), GRAAL_VM_HOME.toString()) + .replace(bundleRoot.toString(), "")); } } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java index 15a4594ba5c2..28d32162229e 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.java @@ -24,36 +24,36 @@ */ package com.oracle.svm.driver.launcher.configuration; -import com.oracle.svm.driver.launcher.json.BundleJSONParser; -import com.oracle.svm.driver.launcher.json.BundleJSONParserException; - import java.io.IOException; import java.io.Reader; import java.net.URI; import java.util.List; import java.util.Map; +import com.oracle.svm.driver.launcher.json.BundleJSONParser; +import com.oracle.svm.driver.launcher.json.BundleJSONParserException; + public abstract class BundleConfigurationParser { - public void parseAndRegister(Reader reader) throws IOException { - parseAndRegister(new BundleJSONParser(reader).parse(), null); - } + public void parseAndRegister(Reader reader) throws IOException { + parseAndRegister(new BundleJSONParser(reader).parse(), null); + } - public abstract void parseAndRegister(Object json, URI origin) throws IOException; + public abstract void parseAndRegister(Object json, URI origin) throws IOException; - @SuppressWarnings("unchecked") - public static List asList(Object data, String errorMessage) { - if (data instanceof List) { - return (List) data; - } - throw new BundleJSONParserException(errorMessage); + @SuppressWarnings("unchecked") + public static List asList(Object data, String errorMessage) { + if (data instanceof List) { + return (List) data; } + throw new BundleJSONParserException(errorMessage); + } - @SuppressWarnings("unchecked") - public static Map asMap(Object data, String errorMessage) { - if (data instanceof Map) { - return (Map) data; - } - throw new BundleJSONParserException(errorMessage); + @SuppressWarnings("unchecked") + public static Map asMap(Object data, String errorMessage) { + if (data instanceof Map) { + return (Map) data; } + throw new BundleJSONParserException(errorMessage); + } } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java index 6f9b5aab81ac..8fc01e653794 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.java @@ -24,11 +24,11 @@ */ package com.oracle.svm.driver.launcher.configuration; -import com.oracle.svm.driver.launcher.json.BundleJSONParserException; - import java.net.URI; import java.util.Map; +import com.oracle.svm.driver.launcher.json.BundleJSONParserException; + public class BundleEnvironmentParser extends BundleConfigurationParser { private static final String environmentKeyField = "key"; private static final String environmentValueField = "val"; diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java index 6eafddde9520..c8c40c8d0013 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.java @@ -24,12 +24,12 @@ */ package com.oracle.svm.driver.launcher.configuration; -import com.oracle.svm.driver.launcher.json.BundleJSONParserException; - import java.net.URI; import java.nio.file.Path; import java.util.Map; +import com.oracle.svm.driver.launcher.json.BundleJSONParserException; + public class BundlePathMapParser extends BundleConfigurationParser { private static final String substitutionMapSrcField = "src"; diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java index ca7534149448..44818e7c0ddf 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/json/BundleJSONParser.java @@ -33,426 +33,426 @@ public class BundleJSONParser { - private final String source; - private final int length; - private int pos = 0; - - private static final int EOF = -1; - - private static final String TRUE = "true"; - private static final String FALSE = "false"; - private static final String NULL = "null"; + private final String source; + private final int length; + private int pos = 0; + + private static final int EOF = -1; + + private static final String TRUE = "true"; + private static final String FALSE = "false"; + private static final String NULL = "null"; + + private static final int STATE_EMPTY = 0; + private static final int STATE_ELEMENT_PARSED = 1; + private static final int STATE_COMMA_PARSED = 2; + + public BundleJSONParser(String source) { + this.source = source; + this.length = source.length(); + } + + public BundleJSONParser(Reader source) throws IOException { + this(readFully(source)); + } + + /** + * Public parse method. Parse a string into a JSON object. + * + * @return the parsed JSON Object + */ + public Object parse() { + final Object value = parseLiteral(); + skipWhiteSpace(); + if (pos < length) { + throw expectedError(pos, "eof", toString(peek())); + } + return value; + } - private static final int STATE_EMPTY = 0; - private static final int STATE_ELEMENT_PARSED = 1; - private static final int STATE_COMMA_PARSED = 2; + private Object parseLiteral() { + skipWhiteSpace(); - public BundleJSONParser(String source) { - this.source = source; - this.length = source.length(); + final int c = peek(); + if (c == EOF) { + throw expectedError(pos, "json literal", "eof"); } + return switch (c) { + case '{' -> parseObject(); + case '[' -> parseArray(); + case '"' -> parseString(); + case 'f' -> parseKeyword(FALSE, Boolean.FALSE); + case 't' -> parseKeyword(TRUE, Boolean.TRUE); + case 'n' -> parseKeyword(NULL, null); + default -> { + if (isDigit(c) || c == '-') { + yield parseNumber(); + } else if (c == '.') { + throw numberError(pos); + } else { + throw expectedError(pos, "json literal", toString(c)); + } + } + }; + } - public BundleJSONParser(Reader source) throws IOException { - this(readFully(source)); - } + private Object parseObject() { + Map result = new HashMap<>(); + int state = STATE_EMPTY; - /** - * Public parse method. Parse a string into a JSON object. - * - * @return the parsed JSON Object - */ - public Object parse() { - final Object value = parseLiteral(); - skipWhiteSpace(); - if (pos < length) { - throw expectedError(pos, "eof", toString(peek())); - } - return value; - } + assert peek() == '{'; + pos++; - private Object parseLiteral() { + while (pos < length) { skipWhiteSpace(); - final int c = peek(); - if (c == EOF) { - throw expectedError(pos, "json literal", "eof"); - } - return switch (c) { - case '{' -> parseObject(); - case '[' -> parseArray(); - case '"' -> parseString(); - case 'f' -> parseKeyword(FALSE, Boolean.FALSE); - case 't' -> parseKeyword(TRUE, Boolean.TRUE); - case 'n' -> parseKeyword(NULL, null); - default -> { - if (isDigit(c) || c == '-') { - yield parseNumber(); - } else if (c == '.') { - throw numberError(pos); - } else { - throw expectedError(pos, "json literal", toString(c)); - } - } - }; - } - - private Object parseObject() { - Map result = new HashMap<>(); - int state = STATE_EMPTY; - - assert peek() == '{'; - pos++; - while (pos < length) { - skipWhiteSpace(); - final int c = peek(); - - switch (c) { - case '"' -> { - if (state == STATE_ELEMENT_PARSED) { - throw expectedError(pos, ", or }", toString(c)); - } - final String id = parseString(); - expectColon(); - final Object value = parseLiteral(); - result.put(id, value); - state = STATE_ELEMENT_PARSED; + switch (c) { + case '"' -> { + if (state == STATE_ELEMENT_PARSED) { + throw expectedError(pos, ", or }", toString(c)); } - case ',' -> { - if (state != STATE_ELEMENT_PARSED) { - throw error("Trailing comma is not allowed in JSON", pos); - } - state = STATE_COMMA_PARSED; - pos++; + final String id = parseString(); + expectColon(); + final Object value = parseLiteral(); + result.put(id, value); + state = STATE_ELEMENT_PARSED; + } + case ',' -> { + if (state != STATE_ELEMENT_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); } - case '}' -> { - if (state == STATE_COMMA_PARSED) { - throw error("Trailing comma is not allowed in JSON", pos); - } - pos++; - return result; + state = STATE_COMMA_PARSED; + pos++; + } + case '}' -> { + if (state == STATE_COMMA_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); } - default -> throw expectedError(pos, ", or }", toString(c)); + pos++; + return result; } + default -> throw expectedError(pos, ", or }", toString(c)); } - throw expectedError(pos, ", or }", "eof"); } - - private void expectColon() { - skipWhiteSpace(); - final int n = next(); - if (n != ':') { - throw expectedError(pos - 1, ":", toString(n)); - } + throw expectedError(pos, ", or }", "eof"); + } + + private void expectColon() { + skipWhiteSpace(); + final int n = next(); + if (n != ':') { + throw expectedError(pos - 1, ":", toString(n)); } + } - private Object parseArray() { - List result = new ArrayList<>(); - int state = STATE_EMPTY; + private Object parseArray() { + List result = new ArrayList<>(); + int state = STATE_EMPTY; - assert peek() == '['; - pos++; + assert peek() == '['; + pos++; + + while (pos < length) { + skipWhiteSpace(); + final int c = peek(); - while (pos < length) { - skipWhiteSpace(); - final int c = peek(); - - switch (c) { - case ',' -> { - if (state != STATE_ELEMENT_PARSED) { - throw error("Trailing comma is not allowed in JSON", pos); - } - state = STATE_COMMA_PARSED; - pos++; + switch (c) { + case ',' -> { + if (state != STATE_ELEMENT_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); } - case ']' -> { - if (state == STATE_COMMA_PARSED) { - throw error("Trailing comma is not allowed in JSON", pos); - } - pos++; - return result; + state = STATE_COMMA_PARSED; + pos++; + } + case ']' -> { + if (state == STATE_COMMA_PARSED) { + throw error("Trailing comma is not allowed in JSON", pos); } - default -> { - if (state == STATE_ELEMENT_PARSED) { - throw expectedError(pos, ", or ]", toString(c)); - } - result.add(parseLiteral()); - state = STATE_ELEMENT_PARSED; + pos++; + return result; + } + default -> { + if (state == STATE_ELEMENT_PARSED) { + throw expectedError(pos, ", or ]", toString(c)); } + result.add(parseLiteral()); + state = STATE_ELEMENT_PARSED; } } - - throw expectedError(pos, ", or ]", "eof"); } - private String parseString() { - // String buffer is only instantiated if string contains escape sequences. - int start = ++pos; - StringBuilder sb = null; + throw expectedError(pos, ", or ]", "eof"); + } - while (pos < length) { - final int c = next(); - if (c <= 0x1f) { - // Characters < 0x1f are not allowed in JSON strings. - throw syntaxError(pos, "String contains control character"); + private String parseString() { + // String buffer is only instantiated if string contains escape sequences. + int start = ++pos; + StringBuilder sb = null; - } else if (c == '\\') { - if (sb == null) { - sb = new StringBuilder(pos - start + 16); - } - sb.append(source, start, pos - 1); - sb.append(parseEscapeSequence()); - start = pos; + while (pos < length) { + final int c = next(); + if (c <= 0x1f) { + // Characters < 0x1f are not allowed in JSON strings. + throw syntaxError(pos, "String contains control character"); - } else if (c == '"') { - if (sb != null) { - sb.append(source, start, pos - 1); - return sb.toString(); - } - return source.substring(start, pos - 1); + } else if (c == '\\') { + if (sb == null) { + sb = new StringBuilder(pos - start + 16); } - } + sb.append(source, start, pos - 1); + sb.append(parseEscapeSequence()); + start = pos; - throw error("Missing close quote", pos); + } else if (c == '"') { + if (sb != null) { + sb.append(source, start, pos - 1); + return sb.toString(); + } + return source.substring(start, pos - 1); + } } - private char parseEscapeSequence() { - final int c = next(); - return switch (c) { - case '"' -> '"'; - case '\\' -> '\\'; - case '/' -> '/'; - case 'b' -> '\b'; - case 'f' -> '\f'; - case 'n' -> '\n'; - case 'r' -> '\r'; - case 't' -> '\t'; - case 'u' -> parseUnicodeEscape(); - default -> throw error("Invalid escape character", pos - 1); - }; + throw error("Missing close quote", pos); + } + + private char parseEscapeSequence() { + final int c = next(); + return switch (c) { + case '"' -> '"'; + case '\\' -> '\\'; + case '/' -> '/'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'u' -> parseUnicodeEscape(); + default -> throw error("Invalid escape character", pos - 1); + }; + } + + private char parseUnicodeEscape() { + return (char) (parseHexDigit() << 12 | parseHexDigit() << 8 | parseHexDigit() << 4 | parseHexDigit()); + } + + private int parseHexDigit() { + final int c = next(); + if (c >= '0' && c <= '9') { + return c - '0'; + } else if (c >= 'A' && c <= 'F') { + return c + 10 - 'A'; + } else if (c >= 'a' && c <= 'f') { + return c + 10 - 'a'; } + throw error("Invalid hex digit", pos - 1); + } - private char parseUnicodeEscape() { - return (char) (parseHexDigit() << 12 | parseHexDigit() << 8 | parseHexDigit() << 4 | parseHexDigit()); - } + private static boolean isDigit(final int c) { + return c >= '0' && c <= '9'; + } - private int parseHexDigit() { - final int c = next(); - if (c >= '0' && c <= '9') { - return c - '0'; - } else if (c >= 'A' && c <= 'F') { - return c + 10 - 'A'; - } else if (c >= 'a' && c <= 'f') { - return c + 10 - 'a'; + private void skipDigits() { + while (pos < length) { + final int c = peek(); + if (!isDigit(c)) { + break; } - throw error("Invalid hex digit", pos - 1); + pos++; } + } + + private Number parseNumber() { + boolean isFloating = false; + final int start = pos; + int c = next(); - private static boolean isDigit(final int c) { - return c >= '0' && c <= '9'; + if (c == '-') { + c = next(); + } + if (!isDigit(c)) { + throw numberError(start); + } + // no more digits allowed after 0 + if (c != '0') { + skipDigits(); } - private void skipDigits() { - while (pos < length) { - final int c = peek(); - if (!isDigit(c)) { - break; - } - pos++; + // fraction + if (peek() == '.') { + isFloating = true; + pos++; + if (!isDigit(next())) { + throw numberError(pos - 1); } + skipDigits(); } - private Number parseNumber() { - boolean isFloating = false; - final int start = pos; - int c = next(); - - if (c == '-') { + // exponent + c = peek(); + if (c == 'e' || c == 'E') { + pos++; + c = next(); + if (c == '-' || c == '+') { c = next(); } if (!isDigit(c)) { - throw numberError(start); - } - // no more digits allowed after 0 - if (c != '0') { - skipDigits(); - } - - // fraction - if (peek() == '.') { - isFloating = true; - pos++; - if (!isDigit(next())) { - throw numberError(pos - 1); - } - skipDigits(); - } - - // exponent - c = peek(); - if (c == 'e' || c == 'E') { - pos++; - c = next(); - if (c == '-' || c == '+') { - c = next(); - } - if (!isDigit(c)) { - throw numberError(pos - 1); - } - skipDigits(); - } - - String literalValue = source.substring(start, pos); - if (isFloating) { - return Double.parseDouble(literalValue); - } else { - final long l = Long.parseLong(literalValue); - if ((int) l == l) { - return (int) l; - } else { - return l; - } + throw numberError(pos - 1); } + skipDigits(); } - private Object parseKeyword(final String keyword, final Object value) { - if (!source.regionMatches(pos, keyword, 0, keyword.length())) { - throw expectedError(pos, "json literal", "ident"); + String literalValue = source.substring(start, pos); + if (isFloating) { + return Double.parseDouble(literalValue); + } else { + final long l = Long.parseLong(literalValue); + if ((int) l == l) { + return (int) l; + } else { + return l; } - pos += keyword.length(); - return value; } + } - private int peek() { - if (pos >= length) { - return -1; - } - return source.charAt(pos); + private Object parseKeyword(final String keyword, final Object value) { + if (!source.regionMatches(pos, keyword, 0, keyword.length())) { + throw expectedError(pos, "json literal", "ident"); } + pos += keyword.length(); + return value; + } - private int next() { - final int next = peek(); - pos++; - return next; + private int peek() { + if (pos >= length) { + return -1; } - - private void skipWhiteSpace() { - while (pos < length) { - switch (peek()) { - case '\t', '\r', '\n', ' ' -> pos++; - default -> { - return; - } + return source.charAt(pos); + } + + private int next() { + final int next = peek(); + pos++; + return next; + } + + private void skipWhiteSpace() { + while (pos < length) { + switch (peek()) { + case '\t', '\r', '\n', ' ' -> pos++; + default -> { + return; } } } - - private static String toString(final int c) { - return c == EOF ? "eof" : String.valueOf((char) c); - } - - private BundleJSONParserException error(final String message, final int start) { - final int lineNum = getLine(start); - final int columnNum = getColumn(start); - final String formatted = format(message, lineNum, columnNum); - return new BundleJSONParserException(formatted); - } - - /** - * Return line number of character position. - * - *

- * This method can be expensive for large sources as it iterates through all characters up to - * {@code position}. - *

- * - * @param position Position of character in source content. - * @return Line number. - */ - private int getLine(final int position) { - final CharSequence d = source; - // Line count starts at 1. - int line = 1; - - for (int i = 0; i < position; i++) { - final char ch = d.charAt(i); - // Works for both \n and \r\n. - if (ch == '\n') { - line++; - } + } + + private static String toString(final int c) { + return c == EOF ? "eof" : String.valueOf((char) c); + } + + private BundleJSONParserException error(final String message, final int start) { + final int lineNum = getLine(start); + final int columnNum = getColumn(start); + final String formatted = format(message, lineNum, columnNum); + return new BundleJSONParserException(formatted); + } + + /** + * Return line number of character position. + * + *

+ * This method can be expensive for large sources as it iterates through all characters up to + * {@code position}. + *

+ * + * @param position Position of character in source content. + * @return Line number. + */ + private int getLine(final int position) { + final CharSequence d = source; + // Line count starts at 1. + int line = 1; + + for (int i = 0; i < position; i++) { + final char ch = d.charAt(i); + // Works for both \n and \r\n. + if (ch == '\n') { + line++; } - - return line; - } - - /** - * Return column number of character position. - * - * @param position Position of character in source content. - * @return Column number. - */ - private int getColumn(final int position) { - return position - findBOLN(position); } - /** - * Find the beginning of the line containing position. - * - * @param position Index to offending token. - * @return Index of first character of line. - */ - private int findBOLN(final int position) { - final CharSequence d = source; - for (int i = position - 1; i > 0; i--) { - final char ch = d.charAt(i); - - if (ch == '\n' || ch == '\r') { - return i + 1; - } + return line; + } + + /** + * Return column number of character position. + * + * @param position Position of character in source content. + * @return Column number. + */ + private int getColumn(final int position) { + return position - findBOLN(position); + } + + /** + * Find the beginning of the line containing position. + * + * @param position Index to offending token. + * @return Index of first character of line. + */ + private int findBOLN(final int position) { + final CharSequence d = source; + for (int i = position - 1; i > 0; i--) { + final char ch = d.charAt(i); + + if (ch == '\n' || ch == '\r') { + return i + 1; } - - return 0; } - /** - * Format an error message to include source and line information. - * - * @param message Error message string. - * @param line Source line number. - * @param column Source column number. - * @return formatted string - */ - private static String format(final String message, final int line, final int column) { - return "line " + line + " column " + column + " " + message; - } - - private BundleJSONParserException numberError(final int start) { - return error("Invalid JSON number format", start); - } - - private BundleJSONParserException expectedError(final int start, final String expected, final String found) { - return error("Expected " + expected + " but found " + found, start); - } - - private BundleJSONParserException syntaxError(final int start, final String reason) { - return error("Invalid JSON: " + reason, start); - } - - /** - * Utility function to read all contents of a {@link Reader}, because the JSON parser does not - * support streaming yet. - */ - private static String readFully(final Reader reader) throws IOException { - final char[] arr = new char[1024]; - final StringBuilder sb = new StringBuilder(); - - try (reader) { - int numChars; - while ((numChars = reader.read(arr, 0, arr.length)) > 0) { - sb.append(arr, 0, numChars); - } + return 0; + } + + /** + * Format an error message to include source and line information. + * + * @param message Error message string. + * @param line Source line number. + * @param column Source column number. + * @return formatted string + */ + private static String format(final String message, final int line, final int column) { + return "line " + line + " column " + column + " " + message; + } + + private BundleJSONParserException numberError(final int start) { + return error("Invalid JSON number format", start); + } + + private BundleJSONParserException expectedError(final int start, final String expected, final String found) { + return error("Expected " + expected + " but found " + found, start); + } + + private BundleJSONParserException syntaxError(final int start, final String reason) { + return error("Invalid JSON: " + reason, start); + } + + /** + * Utility function to read all contents of a {@link Reader}, because the JSON parser does not + * support streaming yet. + */ + private static String readFully(final Reader reader) throws IOException { + final char[] arr = new char[1024]; + final StringBuilder sb = new StringBuilder(); + + try (reader) { + int numChars; + while ((numChars = reader.read(arr, 0, arr.length)) > 0) { + sb.append(arr, 0, numChars); } - - return sb.toString(); } + + return sb.toString(); + } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index cefdcdb99834..fbaffaf75733 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -64,18 +64,17 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.oracle.svm.core.OS; +import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.util.ExitStatus; +import com.oracle.svm.core.util.json.JsonPrinter; +import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.driver.launcher.BundleLauncher; import com.oracle.svm.driver.launcher.ContainerSupport; import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; import com.oracle.svm.driver.launcher.configuration.BundleEnvironmentParser; import com.oracle.svm.driver.launcher.configuration.BundlePathMapParser; - -import com.oracle.svm.core.OS; -import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.option.BundleMember; -import com.oracle.svm.core.util.json.JsonPrinter; -import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.util.ClassUtil; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.StringUtil; @@ -125,7 +124,6 @@ final class BundleSupport { private static final String DEFAULT_DOCKERFILE = getDockerfile("Dockerfile"); private static final String DEFAULT_DOCKERFILE_MUSLIB = getDockerfile("Dockerfile_muslib_extension"); - private static String getDockerfile(String name) { return NativeImage.getResource("/container-default/" + name); } @@ -210,8 +208,8 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } Arrays.stream(options) - .skip(1) - .forEach(bundleSupport::parseExtendedOption); + .skip(1) + .forEach(bundleSupport::parseExtendedOption); if (bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { @@ -229,14 +227,14 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } case BUILDER_ERROR -> /* Exit, builder has handled error reporting. */ - throw NativeImage.showError(null, null, exitStatusCode); + throw NativeImage.showError(null, null, exitStatusCode); case OUT_OF_MEMORY -> { nativeImage.showOutOfMemoryWarning(); throw NativeImage.showError(null, null, exitStatusCode); } default -> { String message = String.format("Container build request for '%s' failed with exit status %d", - nativeImage.imageName, exitStatusCode); + nativeImage.imageName, exitStatusCode); throw NativeImage.showError(message, null, exitStatusCode); } } @@ -312,8 +310,8 @@ private void parseExtendedOption(String option) { } default -> { String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) - .map(Enum::toString) - .collect(Collectors.joining(", ")); + .map(Enum::toString) + .collect(Collectors.joining(", ")); throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", optionKey, suggestedOptions)); } } @@ -480,7 +478,6 @@ Path restoreCanonicalization(Path before) { return after; } - Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) { Path destinationDir = switch (bundleMemberRole) { case Input -> auxiliaryDir; @@ -733,22 +730,21 @@ private Path writeBundle() { Path bundleLauncherFile = Paths.get("/").resolve(BundleLauncher.class.getName().replace(".", "/") + ".class"); try (FileSystem fs = FileSystems.newFileSystem(BundleSupport.class.getResource(bundleLauncherFile.toString()).toURI(), new HashMap<>()); - Stream walk = Files.walk(fs.getPath(bundleLauncherFile.getParent().toString())) - ) { + Stream walk = Files.walk(fs.getPath(bundleLauncherFile.getParent().toString()))) { walk.filter(Predicate.not(Files::isDirectory)) - .map(Path::toString) - .forEach(sourcePath -> { - Path target = rootDir.resolve(Paths.get("/").relativize(Paths.get(sourcePath))); - try (InputStream source = BundleSupport.class.getResourceAsStream(sourcePath)) { - Path bundleFileParent = target.getParent(); - if (bundleFileParent != null) { - Files.createDirectories(bundleFileParent); - } - Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception e) { - throw NativeImage.showError("Failed to write bundle-file " + target, e); - } - }); + .map(Path::toString) + .forEach(sourcePath -> { + Path target = rootDir.resolve(Paths.get("/").relativize(Paths.get(sourcePath))); + try (InputStream source = BundleSupport.class.getResourceAsStream(sourcePath)) { + Path bundleFileParent = target.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); + } + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + throw NativeImage.showError("Failed to write bundle-file " + target, e); + } + }); } catch (Exception e) { throw NativeImage.showError("Failed to read bundle launcher resources '" + bundleLauncherFile.getParent() + "'", e); } @@ -958,7 +954,7 @@ private void loadAndVerify() { fileVersionKey = PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR; int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); String message = String.format("The given bundle file %s was created with newer bundle-file-format version %d.%d" + - " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, BUNDLE_FILE_FORMAT_VERSION_MAJOR, BUNDLE_FILE_FORMAT_VERSION_MINOR); + " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, BUNDLE_FILE_FORMAT_VERSION_MAJOR, BUNDLE_FILE_FORMAT_VERSION_MINOR); if (major > BUNDLE_FILE_FORMAT_VERSION_MAJOR) { throw NativeImage.showError(message); } else if (major == BUNDLE_FILE_FORMAT_VERSION_MAJOR) { @@ -989,9 +985,9 @@ private void loadAndVerify() { nativeImage.showMessage("%sLoaded Bundle from %s", BUNDLE_INFO_MESSAGE_PREFIX, bundleFileName); nativeImage.showMessage("%sBundle created at '%s'", BUNDLE_INFO_MESSAGE_PREFIX, localDateStr); nativeImage.showMessage("%sUsing version: '%s'%s (vendor '%s'%s) on platform: '%s'%s", BUNDLE_INFO_MESSAGE_PREFIX, - bundleVersion, currentVersion, - bundleVendor, currentVendor, - bundlePlatform, currentPlatform); + bundleVersion, currentVersion, + bundleVendor, currentVendor, + bundlePlatform, currentPlatform); } private boolean forceBuilderOnClasspath() { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index c1ab06a378c5..7e637a90d707 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -69,7 +69,6 @@ import java.util.stream.Stream; import com.oracle.svm.core.option.OptionOrigin; -import com.oracle.svm.driver.launcher.ContainerSupport; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.nativeimage.Platform; @@ -92,6 +91,7 @@ import com.oracle.svm.core.util.VMError; import com.oracle.svm.driver.MacroOption.EnabledOption; import com.oracle.svm.driver.MacroOption.Registry; +import com.oracle.svm.driver.launcher.ContainerSupport; import com.oracle.svm.driver.metainf.MetaInfFileType; import com.oracle.svm.driver.metainf.NativeImageMetaInfResourceProcessor; import com.oracle.svm.driver.metainf.NativeImageMetaInfWalker; @@ -2155,7 +2155,7 @@ void setModuleOptionMode(boolean val) { } private void enableModulePathBuild() { - if (config.modulePathBuild == false) { + if (!config.modulePathBuild) { NativeImage.showError("Module options not allowed in this image build. Reason: " + config.imageBuilderModeEnforcer); } config.modulePathBuild = true; @@ -2430,7 +2430,7 @@ private static String safeSubstitution(String source, CharSequence target, CharS return source.replace(target, replacement); } - private static String deletedFileSuffix = ".deleted"; + private static final String deletedFileSuffix = ".deleted"; protected static boolean isDeletedPath(Path toDelete) { return toDelete.getFileName().toString().endsWith(deletedFileSuffix); From 1f81353b849b1c45389bd234e1f922b52d193ae9 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 6 Jul 2023 09:56:06 +0200 Subject: [PATCH 45/61] Update help-extra with extended bundle options --- .../com.oracle.svm.driver/resources/HelpExtra.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index 016955b6da0a..ce64c22d3a91 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -21,14 +21,20 @@ Non-standard options help: --diagnostics-mode Enables logging of image-build information to a diagnostics folder. --dry-run output the command line that would be used for building - --bundle-create[=new-bundle.nib] + --bundle-create[=new-bundle.nib][,dry-run][,container[=][,dockerfile=]] in addition to image building, create a Native Image bundle file (*.nib file) that allows rebuilding of that image again at a later point. If a bundle-file gets passed, the bundle will be created with the given name. Otherwise, the bundle-file name is derived from the image name. - Note both bundle options can be combined with --dry-run to only perform - the bundle operations without any actual image building. - --bundle-apply=some-bundle.nib + Note both bundle options can be extended with ",dry-run" and ",container" + * 'dry-run': only perform the bundle operations without any actual image building. + * 'container': sets up a container image for image building and performs image building + from inside that container. Requires podman or rootless docker to be installed. + If available, 'podman' is preferred and rootless 'docker' is the fallback. Specifying + one or the other as '=' forces the use of a specific tool. + * 'dockerfile=': Use a user provided 'Dockerfile' instead of the default based on + Oracle Linux 8 base images for GraalVM (see https://github.com/graalvm/container) + --bundle-apply=some-bundle.nib[,dry-run][,container[=][,dockerfile=]] an image will be built from the given bundle file with the exact same arguments and files that have been passed to native-image originally to create the bundle. Note that if an extra --bundle-create gets passed From d0f86cb169b345ffd18cd933b733d0bd4701c835 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 6 Jul 2023 10:45:06 +0200 Subject: [PATCH 46/61] Add help text for bundle launcher --- substratevm/mx.substratevm/suite.py | 3 ++- .../driver/launcher/BundleLauncherHelp.txt | 22 +++++++++++++++++++ .../svm/driver/launcher/BundleLauncher.java | 19 ++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index a6a4ef0b9b2c..2c07703ca5ea 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -861,7 +861,8 @@ "com.oracle.svm.driver.launcher": { "subDir": "src", "sourceDirs": [ - "src" + "src", + "resources" ], "checkstyle": "com.oracle.svm.hosted", "workingSets": "SVM", diff --git a/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt b/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt new file mode 100644 index 000000000000..4f6aebd0192d --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt @@ -0,0 +1,22 @@ +This tool can ahead-of-time compile Java code to native executables. + +Usage: java -jar bundle-file [options] [bundle-application-options] + +where options include: + + --with-native-image-agent[,update-bundle[=]] + runs the application with a native-image-agent attached + 'update-bundle' adds the agents output to the bundle-files classpath. + '=' creates a new bundle with the agent output instead. + Note 'update-bundle' requires native-image to be installed + + --container[=][,dockerfile=] + sets up a container image for execution and executes the bundled application + from inside that container. Requires podman or rootless docker to be installed. + If available, 'podman' is preferred and rootless 'docker' is the fallback. Specifying + one or the other as '=' forces the use of a specific tool. + 'dockerfile=': Use a user provided 'Dockerfile' instead of the Dockerfile + bundled with the application + + --verbose enable verbose output + --help print this help message diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 72667724c835..4fcba261bdf3 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -24,9 +24,13 @@ */ package com.oracle.svm.driver.launcher; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -42,6 +46,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; @@ -52,6 +57,7 @@ public class BundleLauncher { static final String BUNDLE_INFO_MESSAGE_PREFIX = "Native Image Bundles: "; private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; private static final String BUNDLE_FILE_EXTENSION = ".nib"; + private static final String HELP_TEXT = getResource("/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt"); private static Path rootDir; private static Path inputDir; @@ -75,6 +81,15 @@ public class BundleLauncher { private static final List applicationArgs = new ArrayList<>(); private static final Map launcherEnvironment = new HashMap<>(); + static String getResource(String resourceName) { + try (InputStream input = BundleLauncher.class.getResourceAsStream(resourceName)) { + BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)); + return reader.lines().collect(Collectors.joining("\n")); + } catch (IOException e) { + throw new Error(e); + } + } + public static void main(String[] args) { bundleFilePath = Paths.get(BundleLauncher.class.getProtectionDomain().getCodeSource().getLocation().getPath()); bundleName = bundleFilePath.getFileName().toString().replace(BUNDLE_FILE_EXTENSION, ""); @@ -294,6 +309,10 @@ private static void parseBundleLauncherArgs(String[] args) { } } else { switch (arg) { + case "--help" -> { + showMessage(HELP_TEXT); + System.exit(0); + } case "--verbose" -> verbose = true; case "--" -> { applicationArgs.addAll(argQueue); From 854a60804a0aded5628c4adc37549d1aa21bfc90 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 19 Jul 2023 08:26:23 +0200 Subject: [PATCH 47/61] Add launcher option tags for APIOption Annotation --- .../oracle/svm/core/NativeImageClassLoaderOptions.java | 4 ++-- .../src/com/oracle/svm/core/RuntimeAssertionsSupport.java | 8 ++++---- .../src/com/oracle/svm/core/option/APIOption.java | 5 +++++ .../src/com/oracle/svm/driver/APIOptionHandler.java | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/NativeImageClassLoaderOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/NativeImageClassLoaderOptions.java index 20a51ab4def7..fc8993b0bffa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/NativeImageClassLoaderOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/NativeImageClassLoaderOptions.java @@ -35,12 +35,12 @@ public class NativeImageClassLoaderOptions { public static final String AddReadsFormat = "=(,)*"; - @APIOption(name = "add-exports", extra = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// + @APIOption(name = "add-exports", extra = true, launcherOption = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// @Option(help = "Value " + AddExportsAndOpensFormat + " updates to export to , regardless of module declaration." + " can be ALL-UNNAMED to export to all unnamed modules.")// public static final HostedOptionKey AddExports = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); - @APIOption(name = "add-opens", extra = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// + @APIOption(name = "add-opens", extra = true, launcherOption = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// @Option(help = "Value " + AddExportsAndOpensFormat + " updates to open to , regardless of module declaration.")// public static final HostedOptionKey AddOpens = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java index 3f916a8c4767..3b2fc6a91e46 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java @@ -91,15 +91,15 @@ public static class Options { private static final char VALUE_SEPARATOR = ':'; - @APIOption(name = {"-ea", "-enableassertions"}, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Enable.class, defaultValue = "", // + @APIOption(name = {"-ea", "-enableassertions"}, launcherOption = true, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Enable.class, defaultValue = "", // customHelp = "also -ea[:[packagename]...|:classname] or -enableassertions[:[packagename]...|:classname]. Enable assertions with specified granularity at run time.")// - @APIOption(name = {"-da", "-disableassertions"}, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Disable.class, defaultValue = "", // + @APIOption(name = {"-da", "-disableassertions"}, launcherOption = true, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Disable.class, defaultValue = "", // customHelp = "also -da[:[packagename]...|:classname] or -disableassertions[:[packagename]...|:classname]. Disable assertions with specified granularity at run time.")// @Option(help = "Enable or disable Java assert statements at run time") // public static final HostedOptionKey RuntimeAssertions = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); - @APIOption(name = {"-esa", "-enablesystemassertions"}, customHelp = "also -enablesystemassertions. Enables assertions in all system classes at run time.") // - @APIOption(name = {"-dsa", "-disablesystemassertions"}, kind = APIOption.APIOptionKind.Negated, // + @APIOption(name = {"-esa", "-enablesystemassertions"}, launcherOption = true, customHelp = "also -enablesystemassertions. Enables assertions in all system classes at run time.") // + @APIOption(name = {"-dsa", "-disablesystemassertions"}, launcherOption = true, kind = APIOption.APIOptionKind.Negated, // customHelp = "also -disablesystemassertions. Disables assertions in all system classes at run time.") // @Option(help = "Enable or disable Java system assertions at run time") // public static final HostedOptionKey RuntimeSystemAssertions = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java index 21200fd882fe..69ddbe8dd53b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java @@ -61,6 +61,11 @@ */ boolean extra() default false; + /** + * This option should be stored in a native image bundle and passed to the jvm when executed with {@code com.oracle.svm.driver.launcher.BundleLauncher}. + */ + boolean launcherOption() default false; + /** * Make a boolean option part of a group of boolean options. **/ diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 1c59e5804fee..cd34cf27f3ac 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -76,7 +76,7 @@ class APIOptionHandler extends NativeImage.OptionHandler { private static final String LEAVE_UNLOCK_SCOPE = SubstrateOptionsParser.commandArgument(SubstrateOptions.UnlockExperimentalVMOptions, "-"); record OptionInfo(String[] variants, char[] valueSeparator, String builderOption, String defaultValue, String helpText, boolean defaultFinal, String deprecationWarning, - List> valueTransformers, APIOptionGroup group, boolean extra) { + List> valueTransformers, APIOptionGroup group, boolean extra, boolean launcherOption) { boolean isDeprecated() { return deprecationWarning.length() > 0; } @@ -259,7 +259,7 @@ private static void extractOption(String optionPrefix, OptionDescriptor optionDe boolean defaultFinal = booleanOption || hasFixedValue; apiOptions.put(apiOptionName, new APIOptionHandler.OptionInfo(apiAnnotation.name(), apiAnnotation.valueSeparator(), builderOption, defaultValue, helpText, - defaultFinal, apiAnnotation.deprecated(), valueTransformers, group, apiAnnotation.extra())); + defaultFinal, apiAnnotation.deprecated(), valueTransformers, group, apiAnnotation.extra(), apiAnnotation.launcherOption())); } if (optionDescriptor.getStability() == OptionStability.STABLE) { From 6a5a4318075a988ee4ef23c78b1ba2377634f36f Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 19 Jul 2023 08:28:21 +0200 Subject: [PATCH 48/61] Use launcherOption tag to collect arguments for executing a bundle with the BundleLauncher, fix BundleLauncher module path creation --- .../src/com/oracle/svm/driver/launcher/BundleLauncher.java | 2 +- .../src/com/oracle/svm/driver/APIOptionHandler.java | 4 ++++ .../src/com/oracle/svm/driver/BundleSupport.java | 3 +-- .../src/com/oracle/svm/driver/NativeImage.java | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 4fcba261bdf3..52579985b862 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -195,7 +195,7 @@ private static List createLaunchCommand() { List modulePath = new ArrayList<>(); if (Files.isDirectory(modulePathDir)) { try (Stream walk = Files.walk(modulePathDir, 1)) { - walk.filter(path -> Files.isDirectory(path) && !path.equals(modulePathDir)) + walk.filter(path -> (Files.isDirectory(path) && !path.equals(modulePathDir)) || path.toString().endsWith(".jar")) .map(path -> useContainer() ? Paths.get("/").resolve(rootDir.relativize(path)) : path) .map(Path::toString) .forEach(modulePath::add); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index cd34cf27f3ac..7d0fcfe2f337 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -390,6 +390,10 @@ String translateOption(ArgumentQueue argQueue) { builderOption += transformed.toString(); } + if (option.launcherOption) { + nativeImage.bundleLauncherArgs.add(argQueue.peek()); + } + return builderOption; } return null; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index fbaffaf75733..f2c1c665f016 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -833,8 +833,7 @@ private Path writeBundle() { if (nativeImage.buildExecutable) { Path runArgsFile = stageDir.resolve("run.json"); try (JsonWriter writer = new JsonWriter(runArgsFile)) { - List runArgs = new ArrayList<>(); - + List runArgs = new ArrayList<>(nativeImage.bundleLauncherArgs); boolean hasMainClassModule = nativeImage.mainClassModule != null && !nativeImage.mainClassModule.isEmpty(); boolean hasMainClass = nativeImage.mainClass != null && !nativeImage.mainClass.isEmpty(); if (hasMainClassModule) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 7e637a90d707..917d76fdfd2a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -267,6 +267,7 @@ private static String oR(OptionKey option) { static final String oHNumberOfThreads = oH(NativeImageOptions.NumberOfThreads); final Map imageBuilderEnvironment = new HashMap<>(); + final ArrayList bundleLauncherArgs = new ArrayList<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); private final LinkedHashSet imageBuilderModulePath = new LinkedHashSet<>(); private final LinkedHashSet imageBuilderClasspath = new LinkedHashSet<>(); From 186f0c6dbe0721c8c1121bae0e238fa1e446364a Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 19 Jul 2023 09:55:38 +0200 Subject: [PATCH 49/61] Tweak launcher option parsing --- .../com/oracle/svm/driver/APIOptionHandler.java | 16 ++++++++++++++-- .../src/com/oracle/svm/driver/BundleSupport.java | 3 ++- .../src/com/oracle/svm/driver/NativeImage.java | 1 - 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 7d0fcfe2f337..997994183c7b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -328,6 +328,7 @@ boolean consume(ArgumentQueue args) { String translateOption(ArgumentQueue argQueue) { OptionInfo option = null; + boolean whitespaceSeparated = false; String[] optionNameAndOptionValue = null; OptionOrigin argumentOrigin = OptionOrigin.from(argQueue.argumentOrigin); found: for (OptionInfo optionInfo : apiOptions.values()) { @@ -353,6 +354,7 @@ String translateOption(ArgumentQueue argQueue) { } option = optionInfo; optionNameAndOptionValue = new String[]{headArg, optionValue}; + whitespaceSeparated = true; break found; } else { boolean withSeparator = valueSeparator != APIOption.NO_SEPARATOR; @@ -390,8 +392,18 @@ String translateOption(ArgumentQueue argQueue) { builderOption += transformed.toString(); } - if (option.launcherOption) { - nativeImage.bundleLauncherArgs.add(argQueue.peek()); + if (nativeImage.useBundle() && option.launcherOption) { + String optionName = optionNameAndOptionValue[0]; + if (optionValue != null) { + if (whitespaceSeparated) { + nativeImage.bundleSupport.bundleLauncherArgs.add(optionName); + nativeImage.bundleSupport.bundleLauncherArgs.add(optionValue); + } else { + nativeImage.bundleSupport.bundleLauncherArgs.add(optionName + optionValue); + } + } else { + nativeImage.bundleSupport.bundleLauncherArgs.add(optionName); + } } return builderOption; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index f2c1c665f016..89fe9650acaa 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -99,6 +99,7 @@ final class BundleSupport { private final boolean forceBuilderOnClasspath; private final List nativeImageArgs; private List updatedNativeImageArgs; + final ArrayList bundleLauncherArgs = new ArrayList<>(); boolean loadBundle; boolean writeBundle; @@ -833,7 +834,7 @@ private Path writeBundle() { if (nativeImage.buildExecutable) { Path runArgsFile = stageDir.resolve("run.json"); try (JsonWriter writer = new JsonWriter(runArgsFile)) { - List runArgs = new ArrayList<>(nativeImage.bundleLauncherArgs); + List runArgs = new ArrayList<>(bundleLauncherArgs); boolean hasMainClassModule = nativeImage.mainClassModule != null && !nativeImage.mainClassModule.isEmpty(); boolean hasMainClass = nativeImage.mainClass != null && !nativeImage.mainClass.isEmpty(); if (hasMainClassModule) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 917d76fdfd2a..7e637a90d707 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -267,7 +267,6 @@ private static String oR(OptionKey option) { static final String oHNumberOfThreads = oH(NativeImageOptions.NumberOfThreads); final Map imageBuilderEnvironment = new HashMap<>(); - final ArrayList bundleLauncherArgs = new ArrayList<>(); private final ArrayList imageBuilderArgs = new ArrayList<>(); private final LinkedHashSet imageBuilderModulePath = new LinkedHashSet<>(); private final LinkedHashSet imageBuilderClasspath = new LinkedHashSet<>(); From 613e9ae9dcda17e6d78529116947e67a70855372 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 19 Jul 2023 11:07:54 +0200 Subject: [PATCH 50/61] Force containerized bundle build if bundled native image was built in a container, Tweak ContainerSupport --- .../svm/driver/launcher/BundleLauncher.java | 11 ++-- .../svm/driver/launcher/ContainerSupport.java | 4 +- .../com/oracle/svm/driver/BundleSupport.java | 55 ++++++++++++------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 52579985b862..4e3129fa1118 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -283,16 +283,16 @@ private static void parseBundleLauncherArgs(String[] args) { } else if (!System.getProperty("os.name").equals("Linux")) { throw new Error("container option is only supported for Linux"); } - Path dockerfile; + containerSupport = new ContainerSupport(stageDir, Error::new, BundleLauncher::showWarning, BundleLauncher::showMessage); if (arg.indexOf(',') != -1) { String option = arg.substring(arg.indexOf(',') + 1); arg = arg.substring(0, arg.indexOf(',')); if (option.startsWith("dockerfile")) { if (option.indexOf('=') != -1) { - dockerfile = Paths.get(option.substring(option.indexOf('=') + 1)); - if (!Files.isReadable(dockerfile)) { - throw new Error(String.format("Dockerfile '%s' is not readable", dockerfile.toAbsolutePath())); + containerSupport.dockerfile = Paths.get(option.substring(option.indexOf('=') + 1)); + if (!Files.isReadable(containerSupport.dockerfile)) { + throw new Error(String.format("Dockerfile '%s' is not readable", containerSupport.dockerfile.toAbsolutePath())); } } else { throw new Error("container option dockerfile requires a dockerfile argument. E.g. dockerfile=path/to/Dockerfile."); @@ -300,10 +300,7 @@ private static void parseBundleLauncherArgs(String[] args) { } else { throw new Error(String.format("Unknown option %s. Valid option is: dockerfile=path/to/Dockerfile.", option)); } - } else { - dockerfile = stageDir.resolve("Dockerfile"); } - containerSupport = new ContainerSupport(dockerfile, stageDir, Error::new, BundleLauncher::showWarning, BundleLauncher::showMessage); if (arg.indexOf('=') != -1) { containerSupport.tool = arg.substring(arg.indexOf('=') + 1); } diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java index 61430b726228..a022b9a84517 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/ContainerSupport.java @@ -61,13 +61,13 @@ public class ContainerSupport { private final Consumer warningPrinter; private final Consumer messagePrinter; - public ContainerSupport(Path dockerfile, Path bundleStageDir, BiFunction errorFunction, Consumer warningPrinter, Consumer messagePrinter) { - this.dockerfile = dockerfile; + public ContainerSupport(Path bundleStageDir, BiFunction errorFunction, Consumer warningPrinter, Consumer messagePrinter) { this.errorFunction = errorFunction; this.warningPrinter = warningPrinter; this.messagePrinter = messagePrinter; if (bundleStageDir != null) { + dockerfile = bundleStageDir.resolve("Dockerfile"); Path containerFile = bundleStageDir.resolve("container.json"); if (Files.exists(containerFile)) { try (Reader reader = Files.newBufferedReader(containerFile)) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 89fe9650acaa..8014a22eb0fb 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -212,15 +212,25 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma .skip(1) .forEach(bundleSupport::parseExtendedOption); + if (!bundleSupport.useContainer && bundleSupport.bundleProperties.forceContainerBuild()) { + if (!OS.LINUX.isCurrent()) { + LogUtils.warning(BUNDLE_INFO_MESSAGE_PREFIX, "Bundle was built in a container, but container builds are only supported for Linux."); + } else { + bundleSupport.useContainer = true; + bundleSupport.containerSupport = new ContainerSupport(bundleSupport.stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); + } + } + if (bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); bundleSupport.useContainer = false; } else if (nativeImage.isDryRun()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); + bundleSupport.useContainer = false; } else { - if (bundleSupport.containerSupport.dockerfile == null) { - bundleSupport.containerSupport.dockerfile = bundleSupport.createDockerfile(); + if (!Files.exists(bundleSupport.containerSupport.dockerfile)) { + bundleSupport.createDockerfile(bundleSupport.containerSupport.dockerfile); } int exitStatusCode = bundleSupport.containerSupport.initializeImage(); switch (ExitStatus.of(exitStatusCode)) { @@ -250,22 +260,18 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } - private Path createDockerfile() { - // take Dockerfile from bundle or create default if not available - Path dockerfile = stageDir.resolve("Dockerfile"); - if (!Files.exists(dockerfile)) { - String dockerfileText = DEFAULT_DOCKERFILE; - if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { - dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; - } - try { - Files.writeString(dockerfile, dockerfileText); - dockerfile.toFile().deleteOnExit(); - } catch (IOException e) { - throw NativeImage.showError("Failed to create default Dockerfile " + dockerfile); - } + private void createDockerfile(Path dockerfile) { + nativeImage.showVerboseMessage(nativeImage.isVerbose(), BUNDLE_INFO_MESSAGE_PREFIX + "Creating default Dockerfile for native-image bundle."); + String dockerfileText = DEFAULT_DOCKERFILE; + if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { + dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; + } + try { + Files.writeString(dockerfile, dockerfileText); + dockerfile.toFile().deleteOnExit(); + } catch (IOException e) { + throw NativeImage.showError("Failed to create default Dockerfile " + dockerfile); } - return dockerfile; } private void parseExtendedOption(String option) { @@ -287,7 +293,7 @@ private void parseExtendedOption(String option) { if (containerSupport != null) { throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); } - containerSupport = new ContainerSupport(null, stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); + containerSupport = new ContainerSupport(stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); useContainer = true; if (optionValue != null) { if (!ContainerSupport.SUPPORTED_TOOLS.contains(optionValue)) { @@ -796,10 +802,12 @@ private Path writeBundle() { Path dockerfilePath = stageDir.resolve("Dockerfile"); try { - if ((containerSupport == null || containerSupport.dockerfile == null) && !Files.exists(dockerfilePath)) { + if (containerSupport == null || !Files.exists(containerSupport.dockerfile)) { // if no Dockerfile was created yet create a new default Dockerfile - createDockerfile(); - } else if (containerSupport != null && containerSupport.dockerfile != null && !dockerfilePath.equals(containerSupport.dockerfile)) { + if (!Files.exists(dockerfilePath)) { + createDockerfile(dockerfilePath); + } + } else if (!dockerfilePath.equals(containerSupport.dockerfile)) { Files.copy(containerSupport.dockerfile, dockerfilePath); } } catch (IOException e) { @@ -995,6 +1003,11 @@ private boolean forceBuilderOnClasspath() { return Boolean.parseBoolean(properties.getOrDefault(PROPERTY_KEY_BUILDER_ON_CLASSPATH, Boolean.FALSE.toString())); } + private boolean forceContainerBuild() { + assert !properties.isEmpty() : "Needs to be called after loadAndVerify()"; + return Boolean.parseBoolean(properties.getOrDefault(PROPERTY_KEY_BUILT_WITH_CONTAINER, Boolean.FALSE.toString())); + } + private void write() { properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR, String.valueOf(BUNDLE_FILE_FORMAT_VERSION_MAJOR)); properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR, String.valueOf(BUNDLE_FILE_FORMAT_VERSION_MINOR)); From 7da5063f5aa8776b9d1239404eabd9153c3573df Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 19 Jul 2023 15:11:16 +0200 Subject: [PATCH 51/61] Fix muslc containerized builds, Rename forceContainerBuild --- .../Dockerfile_muslib_extension | 7 +++-- .../com/oracle/svm/driver/BundleSupport.java | 30 +++---------------- .../com/oracle/svm/driver/NativeImage.java | 28 +++++++++++++++++ 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension index 6d49a1a43fdb..6259862460aa 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension +++ b/substratevm/src/com.oracle.svm.driver/resources/container-default/Dockerfile_muslib_extension @@ -38,7 +38,7 @@ RUN echo "$TEMP_REGION" > /etc/dnf/vars/ociregion \ && wget $ZLIB_LOCATION && tar -xvf zlib-1.2.11.tar.gz \ && cd zlib-1.2.11 \ && ./configure --prefix=$TOOLCHAIN_DIR --static \ - && make && make install \ + && make && make install FROM base as final @@ -48,5 +48,6 @@ COPY --from=muslib /usr/local/musl /usr/local/musl RUN echo "" > /etc/dnf/vars/ociregion ENV TOOLCHAIN_DIR=/usr/local/musl \ - CC=$TOOLCHAIN_DIR/bin/gcc \ - PATH=$TOOLCHAIN_DIR/bin:$PATH \ No newline at end of file + CC=$TOOLCHAIN_DIR/bin/gcc + +ENV PATH=$TOOLCHAIN_DIR/bin:$PATH \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 8014a22eb0fb..181851f2272d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -67,7 +67,6 @@ import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.option.BundleMember; -import com.oracle.svm.core.util.ExitStatus; import com.oracle.svm.core.util.json.JsonPrinter; import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.driver.launcher.BundleLauncher; @@ -212,7 +211,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma .skip(1) .forEach(bundleSupport::parseExtendedOption); - if (!bundleSupport.useContainer && bundleSupport.bundleProperties.forceContainerBuild()) { + if (!bundleSupport.useContainer && bundleSupport.bundleProperties.requireContainerBuild()) { if (!OS.LINUX.isCurrent()) { LogUtils.warning(BUNDLE_INFO_MESSAGE_PREFIX, "Bundle was built in a container, but container builds are only supported for Linux."); } else { @@ -228,27 +227,6 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } else if (nativeImage.isDryRun()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); bundleSupport.useContainer = false; - } else { - if (!Files.exists(bundleSupport.containerSupport.dockerfile)) { - bundleSupport.createDockerfile(bundleSupport.containerSupport.dockerfile); - } - int exitStatusCode = bundleSupport.containerSupport.initializeImage(); - switch (ExitStatus.of(exitStatusCode)) { - case OK -> { - } - case BUILDER_ERROR -> - /* Exit, builder has handled error reporting. */ - throw NativeImage.showError(null, null, exitStatusCode); - case OUT_OF_MEMORY -> { - nativeImage.showOutOfMemoryWarning(); - throw NativeImage.showError(null, null, exitStatusCode); - } - default -> { - String message = String.format("Container build request for '%s' failed with exit status %d", - nativeImage.imageName, exitStatusCode); - throw NativeImage.showError(message, null, exitStatusCode); - } - } } } @@ -260,10 +238,10 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } } - private void createDockerfile(Path dockerfile) { + void createDockerfile(Path dockerfile) { nativeImage.showVerboseMessage(nativeImage.isVerbose(), BUNDLE_INFO_MESSAGE_PREFIX + "Creating default Dockerfile for native-image bundle."); String dockerfileText = DEFAULT_DOCKERFILE; - if (nativeImage.getNativeImageArgs().contains("--static") && nativeImage.getNativeImageArgs().contains("--libc=musl")) { + if (nativeImage.staticExecutable && nativeImage.libC.equals("musl")) { dockerfileText += System.lineSeparator() + DEFAULT_DOCKERFILE_MUSLIB; } try { @@ -1003,7 +981,7 @@ private boolean forceBuilderOnClasspath() { return Boolean.parseBoolean(properties.getOrDefault(PROPERTY_KEY_BUILDER_ON_CLASSPATH, Boolean.FALSE.toString())); } - private boolean forceContainerBuild() { + private boolean requireContainerBuild() { assert !properties.isEmpty() : "Needs to be called after loadAndVerify()"; return Boolean.parseBoolean(properties.getOrDefault(PROPERTY_KEY_BUILT_WITH_CONTAINER, Boolean.FALSE.toString())); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 7e637a90d707..241d1f43fc4c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -98,6 +98,7 @@ import com.oracle.svm.hosted.NativeImageGeneratorRunner; import com.oracle.svm.hosted.NativeImageOptions; import com.oracle.svm.hosted.NativeImageSystemClassLoader; +import com.oracle.svm.hosted.c.libc.HostedLibCFeature; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -249,6 +250,8 @@ private static String oR(OptionKey option) { final String oHClass = oH(SubstrateOptions.Class); final String oHName = oH(SubstrateOptions.Name); final String oHPath = oH(SubstrateOptions.Path); + final String oHUseLibC = oH(HostedLibCFeature.LibCOptions.UseLibC); + final String oHEnableStaticExecutable = oHEnabled(SubstrateOptions.StaticExecutable); final String oHEnableSharedLibraryFlagPrefix = oHEnabled + SubstrateOptions.SharedLibrary.getName(); final String oHEnableBuildOutputColorful = oHEnabledByDriver(SubstrateOptions.BuildOutputColorful); final String oHEnableBuildOutputProgress = oHEnabledByDriver(SubstrateOptions.BuildOutputProgress); @@ -1096,6 +1099,8 @@ private int completeImageBuild() { mainClass = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHClass); buildExecutable = imageBuilderArgs.stream().noneMatch(arg -> arg.startsWith(oHEnableSharedLibraryFlagPrefix)); + staticExecutable = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(oHEnableStaticExecutable)); + libC = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHUseLibC); boolean listModules = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(oH + "+" + "ListModules")); printFlags |= imageBuilderArgs.stream().anyMatch(arg -> arg.matches("-H:MicroArchitecture(@[^=]*)?=list")); @@ -1411,6 +1416,8 @@ private void addTargetArguments() { } boolean buildExecutable; + boolean staticExecutable; + String libC; String mainClass; String mainClassModule; String imageName; @@ -1572,6 +1579,27 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa Path builderArgFile = createImageBuilderArgumentFile(finalImageBuilderArgs); if (useBundle() && bundleSupport.useContainer) { + if (!Files.exists(bundleSupport.containerSupport.dockerfile)) { + bundleSupport.createDockerfile(bundleSupport.containerSupport.dockerfile); + } + int exitStatusCode = bundleSupport.containerSupport.initializeImage(); + switch (ExitStatus.of(exitStatusCode)) { + case OK -> { + } + case BUILDER_ERROR -> + /* Exit, builder has handled error reporting. */ + throw NativeImage.showError(null, null, exitStatusCode); + case OUT_OF_MEMORY -> { + showOutOfMemoryWarning(); + throw NativeImage.showError(null, null, exitStatusCode); + } + default -> { + String message = String.format("Container build request for '%s' failed with exit status %d", + imageName, exitStatusCode); + throw NativeImage.showError(message, null, exitStatusCode); + } + } + Map mountMapping = ContainerSupport.mountMappingFor(config.getJavaHome(), bundleSupport.inputDir, bundleSupport.outputDir); mountMapping.put(argFile, ContainerSupport.TargetPath.readonly(argFile)); mountMapping.put(builderArgFile, ContainerSupport.TargetPath.readonly(builderArgFile)); From a8212d78525c0d6801aec79ece1188af40d0071e Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 19 Jul 2023 15:22:56 +0200 Subject: [PATCH 52/61] Tweak launcher option parsing --- .../src/com/oracle/svm/driver/APIOptionHandler.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 997994183c7b..193c66879229 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -393,16 +393,10 @@ String translateOption(ArgumentQueue argQueue) { } if (nativeImage.useBundle() && option.launcherOption) { - String optionName = optionNameAndOptionValue[0]; - if (optionValue != null) { - if (whitespaceSeparated) { - nativeImage.bundleSupport.bundleLauncherArgs.add(optionName); - nativeImage.bundleSupport.bundleLauncherArgs.add(optionValue); - } else { - nativeImage.bundleSupport.bundleLauncherArgs.add(optionName + optionValue); - } + if (whitespaceSeparated) { + nativeImage.bundleSupport.bundleLauncherArgs.addAll(List.of(optionNameAndOptionValue)); } else { - nativeImage.bundleSupport.bundleLauncherArgs.add(optionName); + nativeImage.bundleSupport.bundleLauncherArgs.add(argQueue.peek()); } } From 5b50e02cea1d8d81ee1f0a557ce49994f23e827e Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Thu, 20 Jul 2023 10:38:22 +0200 Subject: [PATCH 53/61] Fix style issues and build error --- .../svm/core/RuntimeAssertionsSupport.java | 3 +- .../com/oracle/svm/core/SubstrateOptions.java | 20 ++++++++++++ .../com/oracle/svm/core/option/APIOption.java | 3 +- .../com/oracle/svm/driver/NativeImage.java | 3 +- .../svm/hosted/c/libc/HostedLibCFeature.java | 31 ++----------------- 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java index 3b2fc6a91e46..84d15285e169 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java @@ -93,7 +93,8 @@ public static class Options { @APIOption(name = {"-ea", "-enableassertions"}, launcherOption = true, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Enable.class, defaultValue = "", // customHelp = "also -ea[:[packagename]...|:classname] or -enableassertions[:[packagename]...|:classname]. Enable assertions with specified granularity at run time.")// - @APIOption(name = {"-da", "-disableassertions"}, launcherOption = true, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Disable.class, defaultValue = "", // + @APIOption(name = {"-da", + "-disableassertions"}, launcherOption = true, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Disable.class, defaultValue = "", // customHelp = "also -da[:[packagename]...|:classname] or -disableassertions[:[packagename]...|:classname]. Disable assertions with specified granularity at run time.")// @Option(help = "Enable or disable Java assert statements at run time") // public static final HostedOptionKey RuntimeAssertions = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 990309c15c6a..216b57fb3bf6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -116,6 +116,26 @@ public static boolean parseOnce() { @Option(help = "Build statically linked executable (requires static libc and zlib)")// public static final HostedOptionKey StaticExecutable = new HostedOptionKey<>(false); + @APIOption(name = "libc")// + @Option(help = "Selects the libc implementation to use. Available implementations: glibc, musl, bionic")// + public static final HostedOptionKey UseLibC = new HostedOptionKey<>(null) { + @Override + public String getValueOrDefault(UnmodifiableEconomicMap, Object> values) { + if (!values.containsKey(this)) { + return Platform.includedIn(Platform.ANDROID.class) + ? "bionic" + : System.getProperty("substratevm.HostLibC", "glibc"); + } + return (String) values.get(this); + } + + @Override + public String getValue(OptionValues values) { + assert checkDescriptorExists(); + return getValueOrDefault(values.getMap()); + } + }; + @APIOption(name = "target")// @Option(help = "Selects native-image compilation target (in - format). Defaults to host's OS-architecture pair.")// public static final HostedOptionKey TargetPlatform = new HostedOptionKey<>("") { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java index 69ddbe8dd53b..3392146d9fb9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java @@ -62,7 +62,8 @@ boolean extra() default false; /** - * This option should be stored in a native image bundle and passed to the jvm when executed with {@code com.oracle.svm.driver.launcher.BundleLauncher}. + * This option should be stored in a native image bundle and passed to the jvm when executed + * with {@code com.oracle.svm.driver.launcher.BundleLauncher}. */ boolean launcherOption() default false; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 241d1f43fc4c..3ab15e83a87b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -98,7 +98,6 @@ import com.oracle.svm.hosted.NativeImageGeneratorRunner; import com.oracle.svm.hosted.NativeImageOptions; import com.oracle.svm.hosted.NativeImageSystemClassLoader; -import com.oracle.svm.hosted.c.libc.HostedLibCFeature; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -250,7 +249,7 @@ private static String oR(OptionKey option) { final String oHClass = oH(SubstrateOptions.Class); final String oHName = oH(SubstrateOptions.Name); final String oHPath = oH(SubstrateOptions.Path); - final String oHUseLibC = oH(HostedLibCFeature.LibCOptions.UseLibC); + final String oHUseLibC = oH(SubstrateOptions.UseLibC); final String oHEnableStaticExecutable = oHEnabled(SubstrateOptions.StaticExecutable); final String oHEnableSharedLibraryFlagPrefix = oHEnabled + SubstrateOptions.SharedLibrary.getName(); final String oHEnableBuildOutputColorful = oHEnabledByDriver(SubstrateOptions.BuildOutputColorful); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/libc/HostedLibCFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/libc/HostedLibCFeature.java index 2b6657e1c974..dee959612abd 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/libc/HostedLibCFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/libc/HostedLibCFeature.java @@ -26,18 +26,13 @@ import java.util.ServiceLoader; -import org.graalvm.collections.UnmodifiableEconomicMap; -import org.graalvm.compiler.options.Option; -import org.graalvm.compiler.options.OptionKey; -import org.graalvm.compiler.options.OptionValues; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.c.libc.LibCBase; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; -import com.oracle.svm.core.option.APIOption; -import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.util.UserError; @AutomaticallyRegisteredFeature @@ -47,31 +42,9 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { return HostedLibCBase.isPlatformEquivalent(Platform.LINUX.class); } - public static class LibCOptions { - @APIOption(name = "libc")// - @Option(help = "Selects the libc implementation to use. Available implementations: glibc, musl, bionic")// - public static final HostedOptionKey UseLibC = new HostedOptionKey<>(null) { - @Override - public String getValueOrDefault(UnmodifiableEconomicMap, Object> values) { - if (!values.containsKey(this)) { - return Platform.includedIn(Platform.ANDROID.class) - ? "bionic" - : System.getProperty("substratevm.HostLibC", "glibc"); - } - return (String) values.get(this); - } - - @Override - public String getValue(OptionValues values) { - assert checkDescriptorExists(); - return getValueOrDefault(values.getMap()); - } - }; - } - @Override public void afterRegistration(AfterRegistrationAccess access) { - String targetLibC = LibCOptions.UseLibC.getValue(); + String targetLibC = SubstrateOptions.UseLibC.getValue(); ServiceLoader loader = ServiceLoader.load(HostedLibCBase.class); for (HostedLibCBase libc : loader) { if (libc.getName().equals(targetLibC)) { From 6e70a31b6584344e265cf8ec7697e54a8219cfac Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 24 Jul 2023 13:00:40 +0200 Subject: [PATCH 54/61] Fix logging errors --- .../src/com/oracle/svm/driver/BundleSupport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 181851f2272d..0abf97647687 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -213,7 +213,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma if (!bundleSupport.useContainer && bundleSupport.bundleProperties.requireContainerBuild()) { if (!OS.LINUX.isCurrent()) { - LogUtils.warning(BUNDLE_INFO_MESSAGE_PREFIX, "Bundle was built in a container, but container builds are only supported for Linux."); + LogUtils.warning(BUNDLE_INFO_MESSAGE_PREFIX + "Bundle was built in a container, but container builds are only supported for Linux."); } else { bundleSupport.useContainer = true; bundleSupport.containerSupport = new ContainerSupport(bundleSupport.stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); @@ -222,7 +222,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma if (bundleSupport.useContainer) { if (!OS.LINUX.isCurrent()) { - nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX, "Skipping containerized build, only supported for Linux."); + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping containerized build, only supported for Linux."); bundleSupport.useContainer = false; } else if (nativeImage.isDryRun()) { nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Skipping container creation for native-image bundle with dry-run option."); From 1dbe52a712b9574deffc137aa1da7529c08ff2af Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Mon, 24 Jul 2023 13:02:34 +0200 Subject: [PATCH 55/61] Suppress checkstyle rule for custom log messages in bundle launcher --- .../src/com/oracle/svm/driver/launcher/BundleLauncher.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java index 4e3129fa1118..70cb535cce9d 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java +++ b/substratevm/src/com.oracle.svm.driver.launcher/src/com/oracle/svm/driver/launcher/BundleLauncher.java @@ -321,6 +321,7 @@ private static void parseBundleLauncherArgs(String[] args) { } } + // Checkstyle: Allow raw info or warning printing - begin private static void showMessage(String msg) { System.out.println(msg); } @@ -328,6 +329,7 @@ private static void showMessage(String msg) { private static void showWarning(String msg) { System.out.println("Warning: " + msg); } + // Checkstyle: Allow raw info or warning printing - end private static final Path buildTimeJavaHome = Paths.get(System.getProperty("java.home")); From 1198a04f13bdd5a1e46e38f68c66cd7894d30940 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 9 Aug 2023 10:32:57 +0200 Subject: [PATCH 56/61] Fix bundle launcher help text --- .../com/oracle/svm/driver/launcher/BundleLauncherHelp.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt b/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt index 4f6aebd0192d..afe7f539be61 100644 --- a/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt +++ b/substratevm/src/com.oracle.svm.driver.launcher/resources/com/oracle/svm/driver/launcher/BundleLauncherHelp.txt @@ -1,4 +1,4 @@ -This tool can ahead-of-time compile Java code to native executables. +This native image bundle can be used to launch the bundled application. Usage: java -jar bundle-file [options] [bundle-application-options] @@ -15,7 +15,7 @@ where options include: from inside that container. Requires podman or rootless docker to be installed. If available, 'podman' is preferred and rootless 'docker' is the fallback. Specifying one or the other as '=' forces the use of a specific tool. - 'dockerfile=': Use a user provided 'Dockerfile' instead of the Dockerfile + 'dockerfile=': Use a user-provided 'Dockerfile' instead of the Dockerfile bundled with the application --verbose enable verbose output From afe057abba726923565327af2153d896e3aa4243 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 9 Aug 2023 10:33:34 +0200 Subject: [PATCH 57/61] Update Bundles.md with description of new features --- docs/reference-manual/native-image/Bundles.md | 166 ++++++++++++++++-- 1 file changed, 149 insertions(+), 17 deletions(-) diff --git a/docs/reference-manual/native-image/Bundles.md b/docs/reference-manual/native-image/Bundles.md index 593e6669a7de..3559eba67938 100644 --- a/docs/reference-manual/native-image/Bundles.md +++ b/docs/reference-manual/native-image/Bundles.md @@ -22,6 +22,7 @@ Using Native Image bundles is a safe solution to encapsulate all this input requ * [Building with Bundles](#building-with-bundles) * [Environment Variables](#capturing-environment-variables) * [Creating New Bundles from Existing Bundles](#combining---bundle-create-and---bundle-apply) +* [Executing the bundled application](#executing-the-bundled-application) * [Bundle File Format](#bundle-file-format) ## Creating Bundles @@ -31,13 +32,19 @@ This will cause `native-image` to create a _*.nib_ file in addition to the actua Here is the option description: ``` ---bundle-create[=new-bundle.nib] +--bundle-create[=new-bundle.nib][,dry-run][,container[=][,dockerfile=]] in addition to image building, create a Native Image bundle file (*.nib file) that allows rebuilding of that image again at a later point. If a bundle-file gets passed, the bundle will be created with the given name. Otherwise, the bundle-file name is derived from the image name. - Note both bundle options can be combined with --dry-run to only perform - the bundle operations without any actual image building. + Note both bundle options can be extended with ",dry-run" and ",container" + * 'dry-run': only perform the bundle operations without any actual image building. + * 'container': sets up a container image for image building and performs image building + from inside that container. Requires podman or rootless docker to be installed. + If available, 'podman' is preferred and rootless 'docker' is the fallback. Specifying + one or the other as '=' forces the use of a specific tool. + * 'dockerfile=': Use a user provided 'Dockerfile' instead of the default based on + Oracle Linux 8 base images for GraalVM (see https://github.com/graalvm/container) ``` For example, assuming a Micronaut application is built with Maven, make sure the `--bundle-create` option is used. @@ -82,6 +89,18 @@ $ jar tf micronautguide.nib META-INF/MANIFEST.MF META-INF/nibundle.properties output/default/micronautguide +com/oracle/svm/driver/launcher/BundleLauncherUtil.class +com/oracle/svm/driver/launcher/ContainerSupport$TargetPath.class +com/oracle/svm/driver/launcher/BundleLauncherHelp.txt +com/oracle/svm/driver/launcher/BundleLauncher.class +com/oracle/svm/driver/launcher/configuration/BundleConfigurationParser.class +com/oracle/svm/driver/launcher/configuration/BundleEnvironmentParser.class +com/oracle/svm/driver/launcher/configuration/BundleArgsParser.class +com/oracle/svm/driver/launcher/configuration/BundlePathMapParser.class +com/oracle/svm/driver/launcher/configuration/BundleContainerSettingsParser.class +com/oracle/svm/driver/launcher/ContainerSupport.class +com/oracle/svm/driver/launcher/json/BundleJSONParser.class +com/oracle/svm/driver/launcher/json/BundleJSONParserException.class input/classes/cp/micronaut-core-3.8.7.jar input/classes/cp/netty-buffer-4.1.87.Final.jar input/classes/cp/jackson-databind-2.14.1.jar @@ -93,10 +112,12 @@ input/classes/cp/micronaut-jdbc-4.7.2.jar input/classes/cp/jackson-core-2.14.0.jar input/classes/cp/micronaut-runtime-3.8.7.jar input/classes/cp/micronautguide-0.1.jar -input/stage/build.json -input/stage/environment.json input/stage/path_substitutions.json input/stage/path_canonicalizations.json +input/stage/build.json +input/stage/run.json +input/stage/environment.json +input/stage/Dockerfile ``` As you can see, a bundle is just a JAR file with a specific layout. @@ -113,11 +134,11 @@ target/micronautguide.output └── other ``` -### Combining --bundle-create with --dry-run +### Combining --bundle-create with dry-run As mentioned in the `--bundle-create` option description, it is also possible to let `native-image` build a bundle but not actually perform the image building. This might be useful if a user wants to move the bundle to a more powerful machine and build the image there. -Modify the above `native-maven-plugin` configuration to also contain the argument `--dry-run`. +Modify the `--bundle-create` argument in the `native-maven-plugin` configuration above to `--bundle-create,dry-run`. Then running `./mvnw package -Dpackaging=native-image` takes only seconds and the created bundle is much smaller: ``` Native Image Bundles: Bundle written to /home/testuser/micronaut-data-jdbc-repository-maven-java/target/micronautguide.nib @@ -147,7 +168,7 @@ Since no executable is created, no bundle build output is available. ## Building with Bundles -Assuming that the native executable is used in production and once in a while an unexpected exception is thrown at run time. +Assuming that the native executable is used in production and once in a while, an unexpected exception is thrown at run time. Since you still have the bundle that was used to create the executable, it is trivial to build a variant of that executable with debugging support. Use `--bundle-apply=micronautguide.nib` like this: ```shell @@ -186,24 +207,83 @@ You successfully rebuilt the application from the bundle with debug info enabled The full option help of `--bundle-apply` shows a more advanced use case that will be discussed [later](#combining---bundle-create-and---bundle-apply) in detail: ``` ---bundle-apply=some-bundle.nib +--bundle-apply=some-bundle.nib[,dry-run][,container[=][,dockerfile=]] an image will be built from the given bundle file with the exact same arguments and files that have been passed to native-image originally to create the bundle. Note that if an extra --bundle-create gets passed after --bundle-apply, a new bundle will be written based on the given - bundle args plus any additional arguments that haven been passed + bundle args plus any additional arguments that have been passed afterwards. For example: > native-image --bundle-apply=app.nib --bundle-create=app_dbg.nib -g creates a new bundle app_dbg.nib based on the given app.nib bundle. Both bundles are the same except the new one also uses the -g option. ``` + +### Building in a Container + +Another addition to the `--bundle-create` and `--bundle-apply` options, as mentioned above, is to perform image building inside a container image. +This ensures that during the image build `native-image` can not access any resources that were not explicitly specified via the classpath or module path. +Modify the `--bundle-create` argument in the `native-maven-plugin` configuration to `--bundle-create,container`. +This still creates the same bundle as before. However, a container image is built and then used for building the native image executable. +If the container image is newly created, you can also see the build output from the container tool. The name of the container image is the hash of the used Dockerfile. +If the container image already exists you will see the following line in the build output instead: + +```shell +Native Image Bundles: Reusing container image c253ca50f50b380da0e23b168349271976d57e4e. +``` + +For building in a container you require either _podman_ or _rootless docker_ to be available on your system. +Additionally, building in a container is currently only supported for Linux, if using any other OS native image will not create and use a container image. +The container tool used for running the image build can be specified with `--bundle-create,container=podman` or `--bundle-create,container=docker`. +If not specified, `native-image` uses one of the supported tools. If available, `podman` is preferred and rootless `docker` is the fallback. + +The Dockerfile used to build the container image may also be explicitly specified with `--bundle-create,container,dockerfile=`. +If no Dockerfile was specified, a default Dockerfile is used, which is based on the Oracle Linux 8 container images for GraalVM from [here](https://github.com/graalvm/container). +Whichever Dockerfile is finally used to build the container image is stored in the bundle. +Even if you do not use the `container` option, `native-image` creates a Dockerfile and stores it in the bundle. + +Other than creating a container image on the host system, building inside a container does not create any additional build output. +However, the created bundle contains some additional files: +```shell +$ jar tf micronautguide.nib +META-INF/MANIFEST.MF +META-INF/nibundle.properties +... +input/classes/cp/micronaut-management-3.8.7.jar +input/stage/path_substitutions.json +input/stage/path_canonicalizations.json +input/stage/build.json +input/stage/run.json +input/stage/environment.json +input/stage/Dockerfile +input/stage/container.json +``` +The bundle contains the Dockerfile used for building the container image and stores the used container tool, its version and the name of the container image in `container.json`: +```json +{ + "containerTool":"podman", + "containerToolVersion":"podman version 3.4.4", + "containerImage":"c253ca50f50b380da0e23b168349271976d57e4e" +} +``` + +The `container` option may also be combined with `dry-run`, in this case `native-image` does neither create an executable nor a container image. +It does not even check if the selected container tool is available. +In this case, _container.json_ is omitted, or, if you explicitly specified a container tool, just contains the _containerTool_ field without any additional information. + +Containerized builds are sticky, which means that if a bundle was created with `--bundle-create,container` the bundle is marked as a container build. +If you now use `--bundle-apply` with this bundle, it is automatically built in a container again. +However, this does not apply to [executing a bundle](#executing-the-bundled-application), a bundled application is still executed outside a container by default. + +The extended command line interface for containerized builds is shown in the option help texts for `--bundle-create` and `--bundle-apply` above. + ## Capturing Environment Variables Before bundle support was added, all environment variables were visible to the `native-image` builder. This approach does not work well with bundles and is problematic for image building without bundles. Consider having an environment variable that holds sensitive information from your build machine. -Due to Native Image's ability to run code at build time that can create data to be available at run time, it is very easy to build an image were you accidentally leak the contents of such variables. +Due to Native Image's ability to run code at build time that can create data to be available at run time, it is very easy to build an image where you accidentally leak the contents of such variables. Passing environment variables to `native-image` now requires explicit arguments. @@ -252,7 +332,7 @@ $ ls -lh *.iprof The file `default.iprof` contains the profiling information that was created because you ran the Micronaut application from the executable built with `--pgo-instrument`. Now you can create a new optimized bundle out of the existing one: ```shell -native-image --bundle-apply=micronautguide.nib --bundle-create=micronautguide-pgo-optimized.nib --dry-run --pgo +native-image --bundle-apply=micronautguide.nib --bundle-create=micronautguide-pgo-optimized.nib,dry-run --pgo ``` Now take a look how _micronautguide-pgo-optimized.nib_ is different from _micronautguide.nib_: @@ -297,6 +377,52 @@ $ native-image --bundle-apply=micronautguide-pgo-optimized.nib ... ``` +## Executing the bundled application + +As described later in [Bundle File Format](#bundle-file-format), a bundle file is a JAR file with a contained launcher for launching the bundled application. +This means you can use a native image bundle with any JDK and execute it as a JAR file with `/bin/java -jar [bundle-file.nib]`. +The launcher uses the command line arguments stored in _run.json_ and adds all JAR files and folders in _input/classes/cp_ and _input/classes/p_ to the classpath and module path respectively. + +The launcher also comes with a separate command line interface described in its help text: +``` +This native image bundle can be used to launch the bundled application. + +Usage: java -jar bundle-file [options] [bundle-application-options] + +where options include: + + --with-native-image-agent[,update-bundle[=]] + runs the application with a native-image-agent attached + 'update-bundle' adds the agents output to the bundle-files classpath. + '=' creates a new bundle with the agent output instead. + Note 'update-bundle' requires native-image to be installed + + --container[=][,dockerfile=] + sets up a container image for execution and executes the bundled application + from inside that container. Requires podman or rootless docker to be installed. + If available, 'podman' is preferred and rootless 'docker' is the fallback. Specifying + one or the other as '=' forces the use of a specific tool. + 'dockerfile=': Use a user provided 'Dockerfile' instead of the Dockerfile + bundled with the application + + --verbose enable verbose output + --help print this help message +``` + +Running the bundled application with the `--with-native-image-agent` argument requires a `native-image-agent` library to be available. +The output of the `native-image-agent` is written to _.output/launcher/META-INF/native-image/-agent_. +If native image agents output should be inserted into the bundle with `,update-bundle`, the launcher then also requires `native-image`. +The `update-bundle` option makes executes the command `native-image --bundle-apply=.nib --bundle-create=.nib -cp .output/launcher` after executing the bundled application with the `native-image-agent` attached. + +The `container` option realizes a similar behavior to [containerized image builds](#building-in-a-container). +However, the only exception is that in this case the application is executed inside the container instead of `native-image`. +Every bundle contains a Dockerfile which is used for executing the bundled application in a container. +However, this Dockerfile can be overwritten by adding `,dockerile=` to the `--container` argument. + +The bundle launcher only consumes options it knows, all other arguments are passed on to the bundled application. +If the bundle launcher parses ` -- ` without a specified option, the launcher stops parsing arguments. +All remaining arguments are then also passed on to the bundled application. + ## Bundle File Format A bundle file is a JAR file with a well-defined internal layout. @@ -310,6 +436,7 @@ Inside a bundle you can find the following inner structure: │ * Bundle format version (BundleFileVersion{Major,Minor}) │ * Platform and architecture the bundle was created on │ * GraalVM / Native-image version used for bundle creation +├── com.oracle.svm.driver.launcher <- launcher for executing the bundled application ├── input <- All information required to rebuild the image │ ├── auxiliary <- Contains auxiliary files passed to native-image via arguments │ │ (e.g. external `config-*.json` files or PGO `*.iprof`-files) @@ -318,11 +445,16 @@ Inside a bundle you can find the following inner structure: │ │ └── p │ └── stage │ ├── build.json <- Full native-image command line (minus --bundle options) +│ ├── container.json <- Containerization tool, tool version and container +│ │ image name (not available information is omitted) +│ ├── Dockerfile <- Dockerfile used for building the container image │ ├── environment.json <- Environment variables used in the image build │ ├── path_canonicalizations.json <- Record of path-canonicalizations that happened -│ │ during bundle creation for the input files -│ └── path_substitutions.json <- Record of path-substitutions that happened -│ during bundle creation for the input files +│ │ during bundle creation for the input files +│ ├── path_substitutions.json <- Record of path-substitutions that happened +│ │ during bundle creation for the input files +│ └── run.json <- Full command line for executing the bundled application +│ (minus classpath and module path) └── output ├── default │ ├── myimage <- Created image and other output created by the image builder @@ -355,12 +487,12 @@ These include: * `--dry-run` The state of environment variables that are relevant for the build are captured in _input/stage/environment.json_. -For every `-E` argument that were seen when the bundle was created, a snapshot of its key-value pair is recorded in the file. +For every `-E` argument that was seen when the bundle was created, a snapshot of its key-value pair is recorded in the file. The remaining files _path_canonicalizations.json_ and _path_substitutions.json_ contain a record of the file-path transformations that were performed by the `native-image` tool based on the input file paths as specified by the original command line arguments. ### output -If a native executable is built as part of building the bundle (for example, the `--dry-run` option was not used), you also have an _output_ directory in the bundle. +If a native executable is built as part of building the bundle (for example, the `,dry-run` option was not used), you also have an _output_ directory in the bundle. It contains the executable that was built along with any other files that were generated as part of building. Most output files are located in the directory _output/default_ (the executable, its debug info, and debug sources). Builder output files, that would have been written to arbitrary absolute paths if the executable had not been built in the bundle mode, can be found in _output/other_. From 77968a8f67ebf79cd2ee4210fa6ff029819ff596 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 9 Aug 2023 15:08:02 +0200 Subject: [PATCH 58/61] Fix typos in bundle documentation --- docs/reference-manual/native-image/Bundles.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/reference-manual/native-image/Bundles.md b/docs/reference-manual/native-image/Bundles.md index 3559eba67938..4231e248d31e 100644 --- a/docs/reference-manual/native-image/Bundles.md +++ b/docs/reference-manual/native-image/Bundles.md @@ -225,8 +225,10 @@ The full option help of `--bundle-apply` shows a more advanced use case that wil Another addition to the `--bundle-create` and `--bundle-apply` options, as mentioned above, is to perform image building inside a container image. This ensures that during the image build `native-image` can not access any resources that were not explicitly specified via the classpath or module path. Modify the `--bundle-create` argument in the `native-maven-plugin` configuration to `--bundle-create,container`. -This still creates the same bundle as before. However, a container image is built and then used for building the native image executable. -If the container image is newly created, you can also see the build output from the container tool. The name of the container image is the hash of the used Dockerfile. +This still creates the same bundle as before. +However, a container image is built and then used for building the native image executable. +If the container image is newly created, you can also see the build output from the container tool. +The name of the container image is the hash of the used Dockerfile. If the container image already exists you will see the following line in the build output instead: ```shell @@ -234,9 +236,11 @@ Native Image Bundles: Reusing container image c253ca50f50b380da0e23b168349271976 ``` For building in a container you require either _podman_ or _rootless docker_ to be available on your system. -Additionally, building in a container is currently only supported for Linux, if using any other OS native image will not create and use a container image. +Additionally, building in a container is currently only supported for Linux. +Using any other OS native image will not create and use a container image. The container tool used for running the image build can be specified with `--bundle-create,container=podman` or `--bundle-create,container=docker`. -If not specified, `native-image` uses one of the supported tools. If available, `podman` is preferred and rootless `docker` is the fallback. +If not specified, `native-image` uses one of the supported tools. +If available, `podman` is preferred and rootless `docker` is the fallback. The Dockerfile used to build the container image may also be explicitly specified with `--bundle-create,container,dockerfile=`. If no Dockerfile was specified, a default Dockerfile is used, which is based on the Oracle Linux 8 container images for GraalVM from [here](https://github.com/graalvm/container). @@ -412,12 +416,12 @@ where options include: Running the bundled application with the `--with-native-image-agent` argument requires a `native-image-agent` library to be available. The output of the `native-image-agent` is written to _.output/launcher/META-INF/native-image/-agent_. If native image agents output should be inserted into the bundle with `,update-bundle`, the launcher then also requires `native-image`. -The `update-bundle` option makes executes the command `native-image --bundle-apply=.nib --bundle-create=.nib -cp .output/launcher` after executing the bundled application with the `native-image-agent` attached. +The `update-bundle` option executes the command `native-image --bundle-apply=.nib --bundle-create=.nib -cp .output/launcher` after executing the bundled application with the `native-image-agent` attached. The `container` option realizes a similar behavior to [containerized image builds](#building-in-a-container). However, the only exception is that in this case the application is executed inside the container instead of `native-image`. Every bundle contains a Dockerfile which is used for executing the bundled application in a container. -However, this Dockerfile can be overwritten by adding `,dockerile=` to the `--container` argument. +However, this Dockerfile can be overwritten by adding `,dockerfile=` to the `--container` argument. The bundle launcher only consumes options it knows, all other arguments are passed on to the bundled application. If the bundle launcher parses ` -- ` without a specified option, the launcher stops parsing arguments. From cb9a2480739f6d4e8206d4033996729d76e1e027 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 9 Aug 2023 15:13:07 +0200 Subject: [PATCH 59/61] Update CHANGELOG.md with new features --- substratevm/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index 01abb6d5dba0..0d61c4dbd798 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -16,6 +16,8 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-46740) Add support for foreign downcalls (part of "Project Panama") on the AMD64 platform. * (GR-27034) Add `-H:ImageBuildID` option to generate Image Build ID, which is a 128-bit UUID string generated randomly, once per bundle or digest of input args when bundles are not used. * (GR-47647) Add `-H:±UnlockExperimentalVMOptions` for unlocking access to experimental options similar to HotSpot's `-XX:UnlockExperimentalVMOptions`. Explicit unlocking will be required in a future release, which can be tested with the env setting `NATIVE_IMAGE_EXPERIMENTAL_OPTIONS_ARE_FATAL=true`. For more details, see [issue #7105](https://github.com/oracle/graal/issues/7105). +* (GR-43920) Add support for executing native image bundles as jar files with extra options `--with-native-image-agent` and `--container`. +* (GR-43920) Add `,container[=]`, `,dockerfile=` and `,dry-run` options to `--bundle-create`and `--bundle-apply`. ## GraalVM for JDK 17 and GraalVM for JDK 20 (Internal Version 23.0.0) * (GR-40187) Report invalid use of SVM specific classes on image class- or module-path as error. As a temporary workaround, `-H:+AllowDeprecatedBuilderClassesOnImageClasspath` allows turning the error into a warning. From 18e33cfe9920454df3c879d09dc89f9e642f36ed Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Fri, 11 Aug 2023 10:15:02 +0200 Subject: [PATCH 60/61] Bugfixes --- substratevm/mx.substratevm/mx_substratevm.py | 5 ++--- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 1a9bd6563e92..0c0abaa6437e 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -1016,6 +1016,7 @@ def _native_image_launcher_extra_jvm_args(): '--features=com.oracle.svm.driver.APIOptionFeature', '--initialize-at-build-time=com.oracle.svm.driver', '--link-at-build-time=com.oracle.svm.driver,com.oracle.svm.driver.metainf', + '-H:IncludeResources=com/oracle/svm/driver/launcher/.*', ] + svm_experimental_options([ '-H:-ParseRuntimeOptions', ]) @@ -1056,9 +1057,7 @@ def _native_image_launcher_extra_jvm_args(): destination="bin/", jar_distributions=["substratevm:SVM_DRIVER"], main_class=_native_image_launcher_main_class(), - build_args=driver_build_args + [ - '-H:IncludeResources=com/oracle/svm/driver/launcher/.*', - ], + build_args=driver_build_args, extra_jvm_args=_native_image_launcher_extra_jvm_args(), home_finder=False, ), diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 0abf97647687..9485e7db220c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -786,7 +786,7 @@ private Path writeBundle() { createDockerfile(dockerfilePath); } } else if (!dockerfilePath.equals(containerSupport.dockerfile)) { - Files.copy(containerSupport.dockerfile, dockerfilePath); + Files.copy(containerSupport.dockerfile, dockerfilePath, StandardCopyOption.REPLACE_EXISTING); } } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + dockerfilePath, e); From 4ee078e36a46106ca810449937528f057a9eea58 Mon Sep 17 00:00:00 2001 From: dominikmascherbauer Date: Wed, 16 Aug 2023 23:56:35 +0200 Subject: [PATCH 61/61] Parse extended options with constant strings --- .../com/oracle/svm/driver/BundleSupport.java | 36 +++++-------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 9485e7db220c..5eb2678dec0c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -61,7 +61,6 @@ import java.util.jar.JarFile; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; -import java.util.stream.Collectors; import java.util.stream.Stream; import com.oracle.svm.core.OS; @@ -116,6 +115,9 @@ final class BundleSupport { private final BundleProperties bundleProperties; static final String BUNDLE_OPTION = "--bundle"; + private static final String DRY_RUN_OPTION = "dry-run"; + private static final String CONTAINER_OPTION = "container"; + private static final String DOCKERFILE_OPTION = "dockerfile"; static final String BUNDLE_FILE_EXTENSION = ".nib"; ContainerSupport containerSupport; @@ -137,21 +139,6 @@ String optionName() { } } - enum ExtendedBundleOptions { - dry_run, - container, - dockerfile; - - static ExtendedBundleOptions get(String name) { - return ExtendedBundleOptions.valueOf(name.replace('-', '_')); - } - - @Override - public String toString() { - return name().replace('_', '-'); - } - } - static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { try { String bundleFilename = null; @@ -265,9 +252,9 @@ private void parseExtendedOption(String option) { optionValue = null; } - switch (ExtendedBundleOptions.get(optionKey)) { - case dry_run -> nativeImage.setDryRun(true); - case container -> { + switch (optionKey) { + case DRY_RUN_OPTION -> nativeImage.setDryRun(true); + case CONTAINER_OPTION -> { if (containerSupport != null) { throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); } @@ -280,9 +267,9 @@ private void parseExtendedOption(String option) { containerSupport.tool = optionValue; } } - case dockerfile -> { + case DOCKERFILE_OPTION -> { if (containerSupport == null) { - throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, ExtendedBundleOptions.container)); + throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, CONTAINER_OPTION)); } if (optionValue != null) { containerSupport.dockerfile = Path.of(optionValue); @@ -293,12 +280,7 @@ private void parseExtendedOption(String option) { throw NativeImage.showError(String.format("native-image option %s requires a dockerfile argument. E.g. %s=path/to/Dockerfile.", optionKey, optionKey)); } } - default -> { - String suggestedOptions = Arrays.stream(ExtendedBundleOptions.values()) - .map(Enum::toString) - .collect(Collectors.joining(", ")); - throw NativeImage.showError(String.format("Unknown option %s. Valid options are: %s.", optionKey, suggestedOptions)); - } + default -> throw NativeImage.showError(String.format("Unknown option %s. Use --help-extra for usage instructions.", optionKey)); } }