[GR-54128] Re-import libsvm_container code and add change detection annotations

substratevm/ci/ci.jsonnet

@@ -117,7 +117,7 @@
 
   // START MAIN BUILD DEFINITION
   local task_dict = {
-    "style-fullbuild": mxgate("fullbuild,style,nativeimagehelp") + eclipse + jdt + maven + mx_build_exploded + gdb("10.2") + platform_spec(no_jobs) + platform_spec({
+    "style-fullbuild": mxgate("fullbuild,style,nativeimagehelp,check_libcontainer_annotations") + eclipse + jdt + maven + mx_build_exploded + gdb("10.2") + platform_spec(no_jobs) + platform_spec({
       // We could run the style gate on JDK 22 as well, and use old JDKs for running tools like StopBugs etc.,
       // but since we support JDK 21 anyways, there is not good reason to do so.
       "linux:amd64:jdk21": gate + t("30:00"),

substratevm/mx.substratevm/mx_substratevm.py

@@ -24,7 +24,9 @@
 #
 
 import os
+import pathlib
 import re
+import shutil
 import tempfile
 from glob import glob
 from contextlib import contextmanager
@@ -53,8 +55,6 @@
 
 import sys
 
-
-
 suite = mx.suite('substratevm')
 svmSuites = [suite]
 
@@ -206,6 +206,7 @@ def __getattr__(self, name):
     'hellomodule',
     'condconfig',
     'truffle_unittests',
+    'check_libcontainer_annotations'
 ])
 
 def vm_native_image_path(config=None):
@@ -449,6 +450,10 @@ def help_stdout_check(output):
 
             mx.log('mx native-image --help output check detected no errors.')
 
+    with Task('Check ContainerLibrary annotations', tasks, tags=[GraalTags.check_libcontainer_annotations]) as t:
+        if t:
+            mx.command_function("check-libcontainer-annotations")([])
+
     with Task('module build demo', tasks, tags=[GraalTags.hellomodule]) as t:
         if t:
             hellomodule(args.extra_image_builder_arguments)
@@ -2213,3 +2218,77 @@ def javac_image(args):
     def musl_helloworld(args, config=None):
         final_args = ['--static', '--libc=musl'] + args
         run_helloworld_command(final_args, config, 'muslhelloworld')
+
+
+def _get_libcontainer_files():
+    paths = []
+    libcontainer_project = mx.project("com.oracle.svm.native.libcontainer")
+    libcontainer_dir = libcontainer_project.dir
+    for src_dir in libcontainer_project.source_dirs():
+        for path, _, files in os.walk(src_dir):
+            for name in files:
+                abs_path = pathlib.PurePath(path, name)
+                rel_path = abs_path.relative_to(libcontainer_dir)
+                src_svm = pathlib.PurePath("src", "svm")
+                if src_svm in rel_path.parents:
+                    # replace "svm" with "hotspot"
+                    stripped_path = rel_path.relative_to(src_svm)
+                    if not stripped_path.as_posix().startswith("svm_container"):
+                        hotspot_path = pathlib.PurePath("src", "hotspot") / stripped_path
+                        paths.append(hotspot_path.as_posix())
+                else:
+                    paths.append(rel_path.as_posix())
+    return libcontainer_dir, paths
+
+
+@mx.command(suite, 'check-libcontainer-annotations')
+def check_libcontainer_annotations(args):
+    """Verifies that files from libcontainer that are copied from hotspot have a @BasedOnJDKFile annotation in ContainerLibrary."""
+
+    # collect paths to check
+
+    libcontainer_dir, paths = _get_libcontainer_files()
+
+    java_project = mx.project("com.oracle.svm.core")
+    container_library = pathlib.Path(java_project.dir, "src/com/oracle/svm/core/container/ContainerLibrary.java")
+    with open(container_library, "r") as fp:
+        annotation_lines = [x for x in fp.readlines() if "@BasedOnJDKFile" in x]
+
+    # check all files are in an annotation
+    for f in paths:
+        if not any((a for a in annotation_lines if f in a)):
+            mx.abort(f"file {f} not found in any annotation in {container_library}")
+
+    # check all annotations refer to a file
+    for a in annotation_lines:
+        if not any((f for f in paths if f in a)):
+            mx.abort(f"annotation {a} does not match any files in {libcontainer_dir}")
+
+
+reimport_libcontainer_files_cmd = "reimport-libcontainer-files"
+
+
+@mx.command(suite, reimport_libcontainer_files_cmd)
+def reimport_libcontainer_files(args):
+    parser = ArgumentParser(prog=f"mx {reimport_libcontainer_files_cmd}")
+    parser.add_argument("--jdk-repo", required=True, help="Path to the OpenJDK repo to import the files from.")
+    parsed_args = parser.parse_args(args)
+
+    libcontainer_dir, paths = _get_libcontainer_files()
+
+    libcontainer_path = pathlib.Path(libcontainer_dir)
+    jdk_path = pathlib.Path(parsed_args.jdk_repo)
+
+    missing = []
+
+    for path in paths:
+        jdk_file = jdk_path / path
+        svm_file = libcontainer_path / path
+        if jdk_file.is_file():
+            if mx.ask_yes_no(f"Should I update {path}"):
+                shutil.copyfile(jdk_file, svm_file)
+        else:
+            missing.append(jdk_file)
+            mx.warn(f"File not found: {jdk_file}")
+            if mx.ask_yes_no(f"Should I delete {path}"):
+                svm_file.unlink()

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/container/ContainerLibrary.java

@@ -33,9 +33,62 @@
 import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.util.BasedOnJDKFile;
 
+/**
+ * Provides Java-level access to the native {@code libsvm_container} implementation.
+ *
+ * The native code is base on the container implementation in the JDK. The {@link BasedOnJDKFile}
+ * annotations below allow us to track upstream changes. Note that the referenced revisions/tags do
+ * not necessarily denote the date when the file was last imported (although often that is the
+ * case), but rather the last time upstream changes where reviewed. If there are changes that are
+ * irrelevant for SVM, we might omit updating our copies. That said, full updates are done
+ * regularly. See also the README file in
+ * {@code substratevm/src/com.oracle.svm.native.libcontainer/README.md}.
+ */
 @CContext(ContainerLibraryDirectives.class)
 @CLibrary(value = "svm_container", requireStatic = true, dependsOn = "m")
+// The following annotations are for files in `src/hotspot`, which are copied from the JDK
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/java.base/share/native/include/jni.h")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/java.base/unix/native/include/jni_md.h")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupSubsystem_linux.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupSubsystem_linux.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupUtil_linux.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupUtil_linux.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/osContainer_linux.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/osContainer_linux.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/os_linux.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/os_linux.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/linux/os_linux.inline.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/posix/include/jvm_md.h")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/posix/os_posix.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/posix/os_posix.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/os/posix/os_posix.inline.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/memory/allocation.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/memory/allocation.inline.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/memory/allStatic.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/nmt/memflags.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/runtime/os.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/runtime/os.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/runtime/os.inline.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/checkedCast.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/compilerWarnings_gcc.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/compilerWarnings.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/globalDefinitions_gcc.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/globalDefinitions.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/macros.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/ostream.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/ostream.hpp")
+// The following annotations are for files in `src/svm`, which are completely customized for SVM
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/logging/log.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/memory/allocation.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/runtime/globals.hpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/debug.cpp")
+@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+5/src/hotspot/share/utilities/debug.hpp")
 class ContainerLibrary {
     static final int VERSION = 240100;
 

substratevm/src/com.oracle.svm.native.libcontainer/README.md

@@ -1,8 +1,7 @@
 # Native cgroup support for SVM
 
 This contains `libsvm_container`, the native cgroup support for SVM (libsvm_container).
-The C code is ported from the OpenJDK and currently based on:
-https://github.com/openjdk/jdk/tree/9049402a1b9394095b04287eef1f2d46c4da60e9/src/hotspot
+The C code is ported from the OpenJDK and update regularly (see "Updating" below).
 
 ## Building
 
@@ -28,12 +27,25 @@ custom. They only provide the minimal required functionality and are specific to
 
 ## Updating
 
-While the code in here is completely independent and does not need to be in sync with the OpenJDK,
-it should be updated regularly to profit from upstream fixes and improvements. To do so, replace
-the files in [`src/hotspot`](./src/hotspot) with those from the OpenJDK. Then reapply all the
-changes (`#ifdef` guards) using the diff tool of your choice. Finally, adopt the files in
-[`src/svm`](./src/svm) to provide new functionality, if needed. Don't forget to update the import
-revision mention in this file.
+While the code in `libsvm_container` is completely independent and does not need to be in sync with
+the OpenJDK, it should be updated regularly to profit from upstream fixes and improvements. To keep
+track of this, `ContainerLibrary.java` contains `@BasedOnJDKFile` annotations for each imported file,
+which links to the source version in the JDK. With this information, all upstream changes can be
+detected. Note that strictly speaking, the referenced version in the annotation does not necessarily
+mean that the file was imported from that revision. Rather that all changes have been reviewed. If
+there are changes that are irrelevant for `libsvm_container`, we might keep the file as is and still
+bump the version. That said, we plan to do full reimports regularly, at least once every for every
+release.
+
+To help keeping the `@BasedOnJDKFile` annotations up to date, the
+`mx gate --tags check_libcontainer_annotations` command ensures that the actual files and
+annotations are in sync.
+
+To do a full reimport, replace the files in [`src/hotspot`](./src/hotspot) with those from the OpenJDK.
+The `mx reimport-libcontainer-files --jdk-repo path/to/jdk` can help with that. Then reapply all the
+changes (`#ifdef` guards) using the diff tool of your choice. Then, adopt the files in
+[`src/svm`](./src/svm) to provide new functionality, if needed. Finally, update the `@BasedOnJDKFile`
+annotations in `ContainerLibrary.java` to reflect the import revision.
 
 ## Local Testing