diff --git a/docs/reference-manual/native-image/Agent.md b/docs/reference-manual/native-image/Agent.md index 27ef0cd66d49..f8ffbeeffd68 100644 --- a/docs/reference-manual/native-image/Agent.md +++ b/docs/reference-manual/native-image/Agent.md @@ -236,8 +236,6 @@ This tool must first be built with: native-image --macro:native-image-configure-launcher ``` -> Note: The Native Image Configure Tool is only available if [`native-image` is built via `mx`](https://github.com/oracle/graal/blob/master/substratevm/SubstrateVM.md). This configuration tool is not part of any GraalVM distribution by default. - Then, the tool can be used to merge sets of configuration files as follows: ```shell native-image-configure generate --input-dir=/path/to/config-dir-0/ --input-dir=/path/to/config-dir-1/ --output-dir=/path/to/merged-config-dir/ diff --git a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java index e9af8becdc58..7063e1b52b74 100644 --- a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java +++ b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java @@ -27,15 +27,19 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileTime; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.ConcurrentModificationException; import java.util.Date; import java.util.List; import java.util.TimeZone; @@ -89,6 +93,8 @@ public final class NativeImageAgent extends JvmtiAgentBase handler = e -> { if (e instanceof NoSuchFileException) { warn("file " + ((NoSuchFileException) e).getFile() + " for merging could not be found, skipping"); @@ -283,11 +304,12 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c }; TraceProcessor processor = new TraceProcessor(advisor, mergeConfigs.loadJniConfig(handler), mergeConfigs.loadReflectConfig(handler), mergeConfigs.loadProxyConfig(handler), mergeConfigs.loadResourceConfig(handler), mergeConfigs.loadSerializationConfig(handler), - mergeConfigs.loadPredefinedClassesConfig(predefinedClassDestinationDirs, shouldExcludeClassesWithHash, handler), omittedConfigProcessor); + mergeConfigs.loadPredefinedClassesConfig(predefinedClassDestDirs, shouldExcludeClassesWithHash, handler), omittedConfigProcessor); ConfigurationResultWriter writer = new ConfigurationResultWriter(processor); tracer = writer; tracingResultWriter = writer; } + expectedConfigModifiedBefore = getMostRecentlyModified(configOutputDirPath, getMostRecentlyModified(configOutputLockFilePath, null)); } catch (Throwable t) { return error(2, t.toString()); } @@ -342,7 +364,7 @@ private static T error(T result, String message) { private static T usage(T result, String message) { inform(message); inform("Example usage: -agentlib:native-image-agent=config-output-dir=/path/to/config-dir/"); - inform("For details, please read BuildConfiguration.md or https://www.graalvm.org/reference-manual/native-image/BuildConfiguration/"); + inform("For details, please read Agent.md or https://www.graalvm.org/reference-manual/native-image/Agent/"); return result; } @@ -505,20 +527,39 @@ protected void onVMDeathCallback(JvmtiEnv jvmti, JNIEnvironment jni) { private static final int MAX_WARNINGS_FOR_WRITING_CONFIGS_FAILURES = 5; private static int currentFailuresWritingConfigs = 0; + private static int currentFailuresModifiedTargetDirectory = 0; private void writeConfigurationFiles() { + Path tempDirectory = null; try { - final Path tempDirectory = configOutputDirPath.toFile().exists() - ? Files.createTempDirectory(configOutputDirPath, "tempConfig-") - : Files.createTempDirectory("tempConfig-"); - List writtenFilePaths = tracingResultWriter.writeToDirectory(tempDirectory); - - for (Path writtenFilePath : writtenFilePaths) { - Path fileName = tempDirectory.relativize(writtenFilePath); - Path target = configOutputDirPath.resolve(fileName); - tryAtomicMove(writtenFilePath, target); + FileTime mostRecent = getMostRecentlyModified(configOutputDirPath, expectedConfigModifiedBefore); + + // Write files first before failing any modification checks + tempDirectory = Files.createTempDirectory(configOutputDirPath, transformPath("agent-pid{pid}-{datetime}.tmp")); + List tempFilePaths = tracingResultWriter.writeToDirectory(tempDirectory); + + if (!Files.exists(configOutputLockFilePath)) { + throw unexpectedlyModified(configOutputLockFilePath); + } + expectUnmodified(configOutputLockFilePath); + if (!mostRecent.equals(expectedConfigModifiedBefore)) { + throw unexpectedlyModified(configOutputDirPath); + } + + Path[] targetFilePaths = new Path[tempFilePaths.size()]; + for (int i = 0; i < tempFilePaths.size(); i++) { + Path fileName = tempDirectory.relativize(tempFilePaths.get(i)); + targetFilePaths[i] = configOutputDirPath.resolve(fileName); + expectUnmodified(targetFilePaths[i]); } + for (int i = 0; i < tempFilePaths.size(); i++) { + tryAtomicMove(tempFilePaths.get(i), targetFilePaths[i]); + mostRecent = getMostRecentlyModified(targetFilePaths[i], mostRecent); + } + mostRecent = getMostRecentlyModified(configOutputDirPath, mostRecent); + expectedConfigModifiedBefore = mostRecent; + /* * Note that sidecar files may be written directly to the final output directory, such * as the class files from predefined class tracking. However, such files generally @@ -527,8 +568,39 @@ private void writeConfigurationFiles() { compulsoryDelete(tempDirectory); } catch (IOException e) { - warnUpToLimit(currentFailuresWritingConfigs++, MAX_WARNINGS_FOR_WRITING_CONFIGS_FAILURES, "Error when writing configuration files: " + e.toString()); + warnUpToLimit(currentFailuresWritingConfigs++, MAX_WARNINGS_FOR_WRITING_CONFIGS_FAILURES, "Error when writing configuration files: " + e); + } catch (ConcurrentModificationException e) { + warnUpToLimit(currentFailuresModifiedTargetDirectory++, MAX_WARNINGS_FOR_WRITING_CONFIGS_FAILURES, + "file or directory '" + e.getMessage() + "' has been modified by another process. " + + "All output files remain in the temporary directory '" + configOutputDirPath.resolve("..").relativize(tempDirectory) + "'. " + + "Ensure that only one agent instance and no other processes are writing to the output directory '" + configOutputDirPath + "' at the same time. " + + "For running multiple processes with agents at the same time to create a single configuration, read Agent.md " + + "or https://www.graalvm.org/reference-manual/native-image/Agent/ on how to use the native-image-configure tool."); + } + } + + private void expectUnmodified(Path path) { + try { + if (Files.getLastModifiedTime(path).compareTo(expectedConfigModifiedBefore) > 0) { + throw unexpectedlyModified(path); + } + } catch (IOException ignored) { + // best effort + } + } + + private static ConcurrentModificationException unexpectedlyModified(Path path) { + throw new ConcurrentModificationException(path.getFileName().toString()); + } + + private static FileTime getMostRecentlyModified(Path path, FileTime other) { + FileTime modified; + try { + modified = Files.getLastModifiedTime(path); + } catch (IOException ignored) { + return other; // best effort } + return (other == null || other.compareTo(modified) < 0) ? modified : other; } private static void compulsoryDelete(Path pathToDelete) { @@ -559,7 +631,7 @@ private static void tryAtomicMove(final Path source, final Path target) throws I Files.move(source, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } catch (AtomicMoveNotSupportedException e) { warnUpToLimit(currentFailuresAtomicMove++, MAX_FAILURES_ATOMIC_MOVE, - String.format("Could not move temporary configuration profile from (%s) to (%s) atomically. " + + String.format("Could not move temporary configuration profile from '%s' to '%s' atomically. " + "This might result in inconsistencies.", source.toAbsolutePath(), target.toAbsolutePath())); Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); } @@ -585,6 +657,8 @@ protected int onUnloadCallback(JNIJavaVM vm) { if (tracingResultWriter.supportsOnUnloadTraceWriting()) { if (configOutputDirPath != null) { writeConfigurationFiles(); + compulsoryDelete(configOutputLockFilePath); + configOutputLockFilePath = null; configOutputDirPath = null; } } diff --git a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/OmitPreviousConfigTests.java b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/OmitPreviousConfigTests.java index 86e016b27993..70ad11164e6e 100644 --- a/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/OmitPreviousConfigTests.java +++ b/substratevm/src/com.oracle.svm.configure.test/src/com/oracle/svm/configure/test/config/OmitPreviousConfigTests.java @@ -67,10 +67,7 @@ private static TraceProcessor loadTraceProcessorFromResourceDirectory(String res try { String resourceName = resourceDirectory + "/" + resourceFileName; URL resourceURL = OmitPreviousConfigTests.class.getResource(resourceName); - if (resourceURL == null) { - Assert.fail("Configuration file " + resourceName + " does not exist. Make sure that the test or the config directory have not been moved."); - } - return resourceURL.toURI(); + return (resourceURL != null) ? resourceURL.toURI() : null; } catch (Exception e) { throw VMError.shouldNotReachHere("Unexpected error while locating the configuration files.", e); } diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/ConfigurationTool.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/ConfigurationTool.java index 60038af1020c..15449db3d29f 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/ConfigurationTool.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/ConfigurationTool.java @@ -38,8 +38,10 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -239,6 +241,7 @@ private static void generate(Iterator argsIter, boolean acceptTraceFileA break; } } + failIfAgentLockFilesPresent(inputSet, omittedInputSet, outputSet); RuleNode callersFilter = null; if (!builtinCallerFilter) { @@ -275,7 +278,9 @@ private static void generate(Iterator argsIter, boolean acceptTraceFileA null); List predefinedClassDestDirs = new ArrayList<>(); for (URI pathUri : outputSet.getPredefinedClassesConfigPaths()) { - predefinedClassDestDirs.add(Paths.get(pathUri).getParent().resolve(ConfigurationFile.PREDEFINED_CLASSES_AGENT_EXTRACTED_SUBDIR)); + Path subdir = Files.createDirectories(Paths.get(pathUri).getParent().resolve(ConfigurationFile.PREDEFINED_CLASSES_AGENT_EXTRACTED_SUBDIR)); + subdir = Files.createDirectories(subdir); + predefinedClassDestDirs.add(subdir); } Predicate shouldExcludeClassesWithHash = omittedInputTraceProcessor.getPredefinedClassesConfiguration()::containsClassWithHash; p = new TraceProcessor(advisor, inputSet.loadJniConfig(ConfigurationSet.FAIL_ON_EXCEPTION), inputSet.loadReflectConfig(ConfigurationSet.FAIL_ON_EXCEPTION), @@ -332,6 +337,24 @@ private static void generate(Iterator argsIter, boolean acceptTraceFileA } } + private static void failIfAgentLockFilesPresent(ConfigurationSet... sets) { + Set paths = null; + for (ConfigurationSet set : sets) { + for (URI path : set.getDetectedAgentLockPaths()) { + if (paths == null) { + paths = new HashSet<>(); + } + paths.add(path.toString()); + } + } + if (paths != null && !paths.isEmpty()) { + throw new UsageException("The following agent lock files were found in specified configuration directories, which means an agent is currently writing to them. " + + "The agent must finish execution before its configuration can be safely accessed. " + + "Unless a lock file is a leftover from an earlier process that terminated abruptly, it is unsafe to delete it." + System.lineSeparator() + + String.join(System.lineSeparator(), paths)); + } + } + private static void generateFilterRules(Iterator argsIter) throws IOException { Path outputPath = null; boolean reduce = false; diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ConfigurationSet.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ConfigurationSet.java index 4a7839d36c71..2438a7d7f011 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ConfigurationSet.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/ConfigurationSet.java @@ -26,10 +26,13 @@ import java.io.IOException; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashSet; +import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; @@ -51,6 +54,7 @@ public class ConfigurationSet { private final Set resourceConfigPaths = new LinkedHashSet<>(); private final Set serializationConfigPaths = new LinkedHashSet<>(); private final Set predefinedClassesConfigPaths = new LinkedHashSet<>(); + private Set lockFilePaths; public void addDirectory(Path path) { jniConfigPaths.add(path.resolve(ConfigurationFile.JNI.getFileName()).toUri()); @@ -59,15 +63,34 @@ public void addDirectory(Path path) { resourceConfigPaths.add(path.resolve(ConfigurationFile.RESOURCES.getFileName()).toUri()); serializationConfigPaths.add(path.resolve(ConfigurationFile.SERIALIZATION.getFileName()).toUri()); predefinedClassesConfigPaths.add(path.resolve(ConfigurationFile.PREDEFINED_CLASSES_NAME.getFileName()).toUri()); + detectAgentLock(path.resolve(ConfigurationFile.LOCK_FILE_NAME), Files::exists, Path::toUri); + } + + private void detectAgentLock(T location, Predicate exists, Function toUri) { + if (exists.test(location)) { + if (lockFilePaths == null) { + lockFilePaths = new LinkedHashSet<>(); + } + lockFilePaths.add(toUri.apply(location)); + } } public void addDirectory(Function fileResolver) { - jniConfigPaths.add(fileResolver.apply(ConfigurationFile.JNI.getFileName())); - reflectConfigPaths.add(fileResolver.apply(ConfigurationFile.REFLECTION.getFileName())); - proxyConfigPaths.add(fileResolver.apply(ConfigurationFile.DYNAMIC_PROXY.getFileName())); - resourceConfigPaths.add(fileResolver.apply(ConfigurationFile.RESOURCES.getFileName())); - serializationConfigPaths.add(fileResolver.apply(ConfigurationFile.SERIALIZATION.getFileName())); - predefinedClassesConfigPaths.add(fileResolver.apply(ConfigurationFile.PREDEFINED_CLASSES_NAME.getFileName())); + add(jniConfigPaths, fileResolver, ConfigurationFile.JNI); + add(reflectConfigPaths, fileResolver, ConfigurationFile.REFLECTION); + add(proxyConfigPaths, fileResolver, ConfigurationFile.DYNAMIC_PROXY); + add(resourceConfigPaths, fileResolver, ConfigurationFile.RESOURCES); + add(serializationConfigPaths, fileResolver, ConfigurationFile.SERIALIZATION); + add(predefinedClassesConfigPaths, fileResolver, ConfigurationFile.PREDEFINED_CLASSES_NAME); + detectAgentLock(fileResolver.apply(ConfigurationFile.LOCK_FILE_NAME), Objects::nonNull, Function.identity()); + } + + private static void add(Set set, Function resolver, ConfigurationFile kind) { + URI location = resolver.apply(kind.getFileName()); + if (location == null) { + throw new RuntimeException("Configuration file " + location + " does not exist."); + } + set.add(location); } public boolean isEmpty() { @@ -75,6 +98,10 @@ public boolean isEmpty() { resourceConfigPaths.isEmpty() && serializationConfigPaths.isEmpty() && predefinedClassesConfigPaths.isEmpty(); } + public Set getDetectedAgentLockPaths() { + return (lockFilePaths != null) ? lockFilePaths : Collections.emptySet(); + } + public Set getJniConfigPaths() { return jniConfigPaths; } diff --git a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/PredefinedClassesConfiguration.java b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/PredefinedClassesConfiguration.java index 786c8009c63b..9d6a2cc84b53 100644 --- a/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/PredefinedClassesConfiguration.java +++ b/substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/config/PredefinedClassesConfiguration.java @@ -52,7 +52,6 @@ public PredefinedClassesConfiguration(Path[] classDestinationDirs, Predicate findConfigurationFiles(String fileName) { List files = new ArrayList<>(); for (String directory : OptionUtils.flatten(",", Options.ConfigurationFileDirectories.getValue())) { + if (Files.exists(Paths.get(directory, ConfigurationFile.LOCK_FILE_NAME))) { + throw foundLockFile("Configuration file directory '" + directory + "'"); + } Path path = Paths.get(directory, fileName); if (Files.exists(path)) { files.add(path); @@ -117,6 +120,15 @@ public static List findConfigurationResources(String fileName, ClassLoader */ final String separator = "/"; // always for resources (not platform-dependent) String relativeRoot = Stream.of(root.split(separator)).filter(part -> !part.isEmpty() && !part.equals(".")).collect(Collectors.joining(separator)); + try { + String lockPath = relativeRoot.isEmpty() ? ConfigurationFile.LOCK_FILE_NAME + : (relativeRoot + '/' + ConfigurationFile.LOCK_FILE_NAME); + Enumeration resource = classLoader.getResources(lockPath); + if (resource != null && resource.hasMoreElements()) { + throw foundLockFile("Configuration resource root '" + root + "'"); + } + } catch (IOException ignored) { + } String relativePath = relativeRoot.isEmpty() ? fileName : (relativeRoot + '/' + fileName); try { for (Enumeration e = classLoader.getResources(relativePath); e.hasMoreElements();) { @@ -128,4 +140,11 @@ public static List findConfigurationResources(String fileName, ClassLoader } return resources; } + + private static UserError.UserException foundLockFile(String container) { + throw UserError.abort("%s contains file '%s', which means an agent is currently writing to it." + + "The agent must finish execution before its generated configuration can be used to build a native image." + + "Unless the lock file is a leftover from an earlier process that terminated abruptly, it is unsafe to delete it.", + container, ConfigurationFile.LOCK_FILE_NAME); + } }