From 27c55a1193541344b5d75b47799cbd4602624826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lazar=20Mitrovi=C4=87?= Date: Fri, 27 Aug 2021 14:28:53 +0200 Subject: [PATCH 1/2] Add @argFile option to the Native Image Driver --- .../com.oracle.svm.driver/resources/Help.txt | 3 + .../svm/driver/DefaultOptionHandler.java | 220 ++++++++++++++++++ .../com/oracle/svm/driver/NativeImage.java | 2 +- 3 files changed, 224 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/resources/Help.txt b/substratevm/src/com.oracle.svm.driver/resources/Help.txt index 932703c24937..983155abc018 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/Help.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/Help.txt @@ -7,7 +7,10 @@ Usage: native-image [options] class [imagename] [options] (to build an image for a class) or native-image [options] -jar jarfile [imagename] [options] (to build an image for a jar file) + where options include: + + @argument files one or more argument files containing options -cp -classpath --class-path diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 46fcaf8ec226..f0230af9818b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -25,9 +25,12 @@ package com.oracle.svm.driver; import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; @@ -58,6 +61,7 @@ class DefaultOptionHandler extends NativeImage.OptionHandler { } boolean useDebugAttach = false; + boolean disableAtFiles = false; private static void singleArgumentCheck(ArgumentQueue args, String arg) { if (!args.isEmpty()) { @@ -210,6 +214,10 @@ public boolean consume(ArgumentQueue args) { nativeImage.showNewline(); System.exit(0); return true; + case "--disable-@files": + args.poll(); + disableAtFiles = true; + return true; } String debugAttach = "--debug-attach"; @@ -296,9 +304,221 @@ public boolean consume(ArgumentQueue args) { } return true; } + if (headArg.startsWith("@") && !disableAtFiles) { + args.poll(); + headArg = headArg.substring(1); + Path argFile = Paths.get(headArg); + NativeImage.NativeImageArgsProcessor processor = nativeImage.new NativeImageArgsProcessor(argFile.toString()); + readArgFile(argFile).forEach(processor::accept); + List leftoverArgs = processor.apply(false); + if (leftoverArgs.size() > 0) { + NativeImage.showError("Found unrecognized options while parsing argument file '" + argFile + "':\n" + String.join("\n", leftoverArgs)); + } + return true; + } return false; } + // Ported from JDK11's java.base/share/native/libjli/args.c + enum PARSER_STATE { + FIND_NEXT, + IN_COMMENT, + IN_QUOTE, + IN_ESCAPE, + SKIP_LEAD_WS, + IN_TOKEN + } + + class CTX_ARGS { + PARSER_STATE state; + int cptr; + int eob; + char quoteChar; + List parts; + String options; + } + + // Ported from JDK11's java.base/share/native/libjli/args.c + private List readArgFile(Path file) { + List arguments = new ArrayList<>(); + // Use of the at sign (@) to recursively interpret files isn't supported. + arguments.add("--disable-@files"); + + String options = null; + try { + options = new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } catch (IOException e) { + NativeImage.showError("Error reading argument file", e); + } + + CTX_ARGS ctx = new CTX_ARGS(); + ctx.state = PARSER_STATE.FIND_NEXT; + ctx.parts = new ArrayList<>(4); + ctx.quoteChar = '"'; + ctx.cptr = 0; + ctx.eob = options.length(); + ctx.options = options; + + String token = nextToken(ctx); + while (token != null) { + arguments.add(token); + token = nextToken(ctx); + } + + // remaining partial token + if (ctx.state == PARSER_STATE.IN_TOKEN || ctx.state == PARSER_STATE.IN_QUOTE) { + if (ctx.parts.size() != 0) { + token = String.join("", ctx.parts); + arguments.add(token); + } + } + return arguments; + } + + // Ported from JDK11's java.base/share/native/libjli/args.c + @SuppressWarnings("fallthrough") + private static String nextToken(CTX_ARGS ctx) { + int nextc = ctx.cptr; + int eob = ctx.eob; + int anchor = nextc; + String token; + + for (; nextc < eob; nextc++) { + char ch = ctx.options.charAt(nextc); + + // Skip white space characters + if (ctx.state == PARSER_STATE.FIND_NEXT || ctx.state == PARSER_STATE.SKIP_LEAD_WS) { + while (ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\f') { + nextc++; + if (nextc >= eob) { + return null; + } + ch = ctx.options.charAt(nextc); + } + ctx.state = (ctx.state == PARSER_STATE.FIND_NEXT) ? PARSER_STATE.IN_TOKEN : PARSER_STATE.IN_QUOTE; + anchor = nextc; + // Deal with escape sequences + } else if (ctx.state == PARSER_STATE.IN_ESCAPE) { + // concatenation directive + if (ch == '\n' || ch == '\r') { + ctx.state = PARSER_STATE.SKIP_LEAD_WS; + } else { + // escaped character + char[] escaped = new char[2]; + escaped[1] = '\0'; + switch (ch) { + case 'n': + escaped[0] = '\n'; + break; + case 'r': + escaped[0] = '\r'; + break; + case 't': + escaped[0] = '\t'; + break; + case 'f': + escaped[0] = '\f'; + break; + default: + escaped[0] = ch; + break; + } + ctx.parts.add(String.valueOf(escaped)); + ctx.state = PARSER_STATE.IN_QUOTE; + } + // anchor to next character + anchor = nextc + 1; + continue; + // ignore comment to EOL + } else if (ctx.state == PARSER_STATE.IN_COMMENT) { + while (ch != '\n' && ch != '\r') { + nextc++; + if (nextc >= eob) { + return null; + } + ch = ctx.options.charAt(nextc); + } + anchor = nextc + 1; + ctx.state = PARSER_STATE.FIND_NEXT; + continue; + } + + assert (ctx.state != PARSER_STATE.IN_ESCAPE); + assert (ctx.state != PARSER_STATE.FIND_NEXT); + assert (ctx.state != PARSER_STATE.SKIP_LEAD_WS); + assert (ctx.state != PARSER_STATE.IN_COMMENT); + + switch (ch) { + case ' ': + case '\t': + case '\f': + if (ctx.state == PARSER_STATE.IN_QUOTE) { + continue; + } + // fall through + case '\n': + case '\r': + if (ctx.parts.size() == 0) { + token = ctx.options.substring(anchor, nextc); + } else { + ctx.parts.add(ctx.options.substring(anchor, nextc)); + token = String.join("", ctx.parts); + ctx.parts = new ArrayList<>(); + } + ctx.cptr = nextc + 1; + ctx.state = PARSER_STATE.FIND_NEXT; + return token; + case '#': + if (ctx.state == PARSER_STATE.IN_QUOTE) { + continue; + } + ctx.state = PARSER_STATE.IN_COMMENT; + anchor = nextc + 1; + break; + case '\\': + if (ctx.state != PARSER_STATE.IN_QUOTE) { + continue; + } + ctx.parts.add(ctx.options.substring(anchor, nextc)); + ctx.state = PARSER_STATE.IN_ESCAPE; + // anchor after backslash character + anchor = nextc + 1; + break; + case '\'': + case '"': + if (ctx.state == PARSER_STATE.IN_QUOTE && ctx.quoteChar != ch) { + // not matching quote + continue; + } + // partial before quote + if (anchor != nextc) { + ctx.parts.add(ctx.options.substring(anchor, nextc)); + } + // anchor after quote character + anchor = nextc + 1; + if (ctx.state == PARSER_STATE.IN_TOKEN) { + ctx.quoteChar = ch; + ctx.state = PARSER_STATE.IN_QUOTE; + } else { + ctx.state = PARSER_STATE.IN_TOKEN; + } + break; + default: + break; + } + } + + assert (nextc == eob); + // Only need partial token, not comment or whitespaces + if (ctx.state == PARSER_STATE.IN_TOKEN || ctx.state == PARSER_STATE.IN_QUOTE) { + if (anchor < nextc) { + // not yet return until end of stream, we have part of a token. + ctx.parts.add(ctx.options.substring(anchor, nextc)); + } + } + return null; + } + private void processClasspathArgs(String cpArgs) { for (String cp : cpArgs.split(File.pathSeparator, Integer.MAX_VALUE)) { /* Conform to `java` command empty cp entry handling. */ diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index be90a8c789cb..aeec8f8563ee 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -815,7 +815,7 @@ protected NativeImage(BuildConfiguration config) { /* Discover supported MacroOptions */ optionRegistry = new MacroOption.Registry(); - /* Default handler needs to be fist */ + /* Default handler needs to be first */ defaultOptionHandler = new DefaultOptionHandler(this); registerOptionHandler(defaultOptionHandler); apiOptionHandler = new APIOptionHandler(this); From 7c4a040b2a2b7fbe7ea29cf4c76ce429ea24bc68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lazar=20Mitrovi=C4=87?= Date: Fri, 27 Aug 2021 15:34:37 +0200 Subject: [PATCH 2/2] Use @argFiles for NativeImageGeneratorRunner's vm invocation on JDK9+ Fixes the issue where builder vm invocation would fail on Windows due to classpath length exceeding command line length limit. --- .../com/oracle/svm/driver/NativeImage.java | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index aeec8f8563ee..b73b7045d57b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1372,21 +1372,24 @@ protected static List createImageBuilderArgs(ArrayList imageArgs return result; } + protected static String createVMInvocationArgumentFile(List arguments) { + try { + Path argsFile = Files.createTempFile("vminvocation", ".args"); + String joinedOptions = String.join("\n", arguments); + Files.write(argsFile, joinedOptions.getBytes()); + argsFile.toFile().deleteOnExit(); + return "@" + argsFile; + } catch (IOException e) { + throw showError(e.getMessage()); + } + } + protected static String createImageBuilderArgumentFile(List imageBuilderArguments) { try { - Path argsFile = Files.createTempFile("native-image", "args"); + Path argsFile = Files.createTempFile("native-image", ".args"); String joinedOptions = String.join("\0", imageBuilderArguments); Files.write(argsFile, joinedOptions.getBytes()); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - try { - Files.delete(argsFile); - } catch (IOException e) { - System.err.println("Failed to delete temporary image builder arguments file: " + argsFile.toString()); - } - } - }); + argsFile.toFile().deleteOnExit(); return NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + argsFile.toString(); } catch (IOException e) { throw showError(e.getMessage()); @@ -1395,37 +1398,46 @@ public void run() { protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedHashSet cp, LinkedHashSet mp, ArrayList imageArgs, LinkedHashSet imagecp, LinkedHashSet imagemp) { - /* Construct ProcessBuilder command from final arguments */ - List command = new ArrayList<>(); - command.add(canonicalize(config.getJavaExecutable()).toString()); - command.addAll(javaArgs); + List arguments = new ArrayList<>(); + arguments.addAll(javaArgs); if (!bcp.isEmpty()) { - command.add(bcp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator, "-Xbootclasspath/a:", ""))); + arguments.add(bcp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator, "-Xbootclasspath/a:", ""))); } if (!cp.isEmpty()) { - command.addAll(Arrays.asList("-cp", cp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator)))); + arguments.addAll(Arrays.asList("-cp", cp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator)))); } if (!mp.isEmpty()) { List strings = Arrays.asList("--module-path", mp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))); - command.addAll(strings); + arguments.addAll(strings); } if (USE_NI_JPMS) { - command.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME)); + arguments.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME)); } else { - command.add(config.getGeneratorMainClass()); + arguments.add(config.getGeneratorMainClass()); } if (IS_AOT && OS.getCurrent().hasProcFS) { /* * GR-8254: Ensure image-building VM shuts down even if native-image dies unexpected * (e.g. using CTRL-C in Gradle daemon mode) */ - command.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID())); + arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID())); } List finalImageBuilderArgs = createImageBuilderArgs(imageArgs, imagecp, imagemp); - List completeCommandList = Stream.concat(command.stream(), finalImageBuilderArgs.stream()).collect(Collectors.toList()); + + /* Construct ProcessBuilder command from final arguments */ + List command = new ArrayList<>(); + command.add(canonicalize(config.getJavaExecutable()).toString()); + List completeCommandList = new ArrayList<>(command); + if (config.useJavaModules()) { // Only in JDK9+ 'java' executable supports @argFiles. + command.add(createVMInvocationArgumentFile(arguments)); + } else { + command.addAll(arguments); + } command.add(createImageBuilderArgumentFile(finalImageBuilderArgs)); + + completeCommandList.addAll(Stream.concat(arguments.stream(), finalImageBuilderArgs.stream()).collect(Collectors.toList())); final String commandLine = SubstrateUtil.getShellCommandString(completeCommandList, true); if (isDiagnostics()) { // write to the diagnostics dir