[GR-63407] Flush instruction cache after trampoline preparation.

PullRequest: graal/20420

Author: fangerer

substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/TrampolineSet.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,19 +29,22 @@
 import java.util.BitSet;
 import java.util.List;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.PinnedObject;
 import org.graalvm.nativeimage.c.function.CFunctionPointer;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.PointerBase;
 import org.graalvm.word.UnsignedWord;
 
+import com.oracle.svm.core.code.AbstractRuntimeCodeInstaller.RuntimeCodeInstallerPlatformHelper;
+import com.oracle.svm.core.heap.VMOperationInfos;
 import com.oracle.svm.core.os.VirtualMemoryProvider;
+import com.oracle.svm.core.thread.JavaVMOperation;
 import com.oracle.svm.core.util.UnsignedUtils;
 import com.oracle.svm.core.util.VMError;
 
 import jdk.graal.compiler.core.common.NumUtil;
+import jdk.graal.compiler.word.Word;
 
 /**
  * A set of trampolines that can be assigned to specific upcall stubs with specific method handles.
@@ -154,6 +157,15 @@ private Pointer prepareTrampolines(PinnedObject mhsArray, PinnedObject stubsArra
         VMError.guarantee(memoryProvider.protect(page, pageSize, VirtualMemoryProvider.Access.EXECUTE) == 0,
                         "Error when making the trampoline allocation executable");
 
+        /*
+         * On some architectures, it is necessary to flush the instruction cache if new code was
+         * installed and/or to issue an instruction synchronization barrier on other cores currently
+         * running a thread that may execute the newly installed code.
+         */
+        if (RuntimeCodeInstallerPlatformHelper.singleton().needsInstructionCacheSynchronization()) {
+            new InstructionCacheOperation(page.rawValue(), pageSize.rawValue()).enqueue();
+        }
+
         return page;
     }
 
@@ -173,4 +185,20 @@ boolean tryFree() {
         }
         return true;
     }
+
+    private static class InstructionCacheOperation extends JavaVMOperation {
+        private final long codeStart;
+        private final long codeSize;
+
+        InstructionCacheOperation(long codeStart, long codeSize) {
+            super(VMOperationInfos.get(InstructionCacheOperation.class, "Prepare FFM API trampoline set", SystemEffect.SAFEPOINT));
+            this.codeStart = codeStart;
+            this.codeSize = codeSize;
+        }
+
+        @Override
+        protected void operate() {
+            RuntimeCodeInstallerPlatformHelper.singleton().performCodeSynchronization(codeStart, codeSize);
+        }
+    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/CodeSynchronizationOperations.java

@@ -32,6 +32,15 @@
 @CLibrary(value = "libchelper", requireStatic = true)
 public class CodeSynchronizationOperations {
 
+    /**
+     * Flushes the processor's instruction cache for the given region of memory if required.
+     *
+     * Some target architectures (e.g. aarch64) require that the instruction cache be flushed, after
+     * modifying memory containing code, in order to obtain deterministic behavior.
+     * 
+     * @param codeStart start address of the modified memory
+     * @param codeSize size of the modified memory
+     */
     @CFunction(value = "codeSynchronization_clearCache", transition = Transition.NO_TRANSITION)
     public static native void clearCache(long codeStart, long codeSize);
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/AbstractRuntimeCodeInstaller.java

@@ -24,7 +24,6 @@
  */
 package com.oracle.svm.core.code;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.word.Pointer;
@@ -37,6 +36,9 @@
 import com.oracle.svm.core.thread.JavaVMOperation;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.api.replacements.Fold;
+import jdk.graal.compiler.word.Word;
+
 public class AbstractRuntimeCodeInstaller {
     protected Pointer allocateCodeMemory(long size) {
         PointerBase result = RuntimeCodeInfoAccess.allocateCodeMemory(Word.unsigned(size));
@@ -77,10 +79,6 @@ protected static <E extends Throwable> RuntimeException rethrow(Throwable ex) th
         throw (E) ex;
     }
 
-    protected static RuntimeCodeInstallerPlatformHelper platformHelper() {
-        return ImageSingletons.lookup(RuntimeCodeInstallerPlatformHelper.class);
-    }
-
     private static class InstallCodeOperation extends JavaVMOperation {
         private final SharedMethod method;
         private final CodeInfo codeInfo;
@@ -102,7 +100,7 @@ protected void operate() {
                 UnsignedWord offset = CodeInfoAccess.getCodeEntryPointOffset(codeInfo);
                 installedCode.setAddress(codeStart.rawValue(), codeStart.rawValue() + offset.rawValue(), method);
                 CodeInfoTable.getRuntimeCodeCache().addMethod(codeInfo);
-                platformHelper().performCodeSynchronization(codeInfo);
+                RuntimeCodeInstallerPlatformHelper.singleton().performCodeSynchronization(codeStart.rawValue(), CodeInfoAccess.getCodeSize(codeInfo).rawValue());
                 VMError.guarantee(CodeInfoAccess.getState(codeInfo) == CodeInfo.STATE_CODE_CONSTANTS_LIVE && installedCode.isValid(), "The code can't be invalidated before the VM operation finishes");
             } catch (Throwable e) {
                 error = e;
@@ -113,11 +111,19 @@ protected void operate() {
     /** Methods which are platform specific. */
     public interface RuntimeCodeInstallerPlatformHelper {
 
+        @Fold
+        static RuntimeCodeInstallerPlatformHelper singleton() {
+            return ImageSingletons.lookup(RuntimeCodeInstallerPlatformHelper.class);
+        }
+
+        boolean needsInstructionCacheSynchronization();
+
         /**
          * Method to enable platforms to perform any needed operations before code becomes visible.
          *
-         * @param codeInfo the new code to be installed
+         * @param codeStart start address of the new code to be installed
+         * @param codeSize size of the new code to be installed
          */
-        void performCodeSynchronization(CodeInfo codeInfo);
+        void performCodeSynchronization(long codeStart, long codeSize);
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java

@@ -973,8 +973,21 @@ public static String toString(int safepointBehavior) {
     }
 
     /**
-     * A thread-local enum conveying any actions needed before thread begins executing Java code.
-     * Only used on aarch64 at the moment.
+     * A thread-local enum conveying any actions needed before thread begins executing Java code. At
+     * the moment, only used on aarch64 where it is necessary to issue an ISB (instruction
+     * synchronization barrier) if new code was made executable (see
+     * <a href="https://developer.arm.com/documentation/ddi0487/latest">ARM Architecture Reference
+     * Manual</a> Section B2.2.5).
+     * 
+     * <pre>
+     * For example, assume there are 4 cores and 2 Java threads:
+     * - Thread A runs on core 0
+     * - Thread B runs on core 1 and installs code. We initiate a safepoint and force all Java
+     *   threads to call ISB
+     * - After the safepoint, thread A still runs on core 0 and executes ISB
+     * - If the OS switches thread A to core 2, the context switch also executes an appropriate
+     *   instruction
+     * </pre>
      */
     public static class ActionOnTransitionToJavaSupport {
 

substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/aarch64/AArch64RuntimeCodeInstallerPlatformHelper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2020, 2020, Arm Limited. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -30,8 +30,6 @@
 
 import com.oracle.svm.core.CodeSynchronizationOperations;
 import com.oracle.svm.core.code.AbstractRuntimeCodeInstaller.RuntimeCodeInstallerPlatformHelper;
-import com.oracle.svm.core.code.CodeInfo;
-import com.oracle.svm.core.code.CodeInfoAccess;
 import com.oracle.svm.core.code.RuntimeCodeCache;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton;
 import com.oracle.svm.core.option.RuntimeOptionKey;
@@ -48,9 +46,20 @@ public class AArch64RuntimeCodeInstallerPlatformHelper implements RuntimeCodeIns
         }
     }
 
+    /**
+     * According to the <a href="https://developer.arm.com/documentation/ddi0487/latest">ARM
+     * Architecture Reference Manual</a> (see Section B2.2.5), it is necessary to flush the
+     * instruction cache and to issue an ISB (instruction synchronization barrier) if new code was
+     * made executable.
+     */
     @Override
-    public void performCodeSynchronization(CodeInfo codeInfo) {
-        CodeSynchronizationOperations.clearCache(CodeInfoAccess.getCodeStart(codeInfo).rawValue(), CodeInfoAccess.getCodeSize(codeInfo).rawValue());
+    public boolean needsInstructionCacheSynchronization() {
+        return true;
+    }
+
+    @Override
+    public void performCodeSynchronization(long codeStart, long codeSize) {
+        CodeSynchronizationOperations.clearCache(codeStart, codeSize);
         VMThreads.ActionOnTransitionToJavaSupport.requestAllThreadsSynchronizeCode();
     }
 }

substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/amd64/AMD64RuntimeCodeInstallerPlatformHelper.java

@@ -28,14 +28,18 @@
 import org.graalvm.nativeimage.Platforms;
 
 import com.oracle.svm.core.code.AbstractRuntimeCodeInstaller.RuntimeCodeInstallerPlatformHelper;
-import com.oracle.svm.core.code.CodeInfo;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton;
 
 @AutomaticallyRegisteredImageSingleton(RuntimeCodeInstallerPlatformHelper.class)
 @Platforms(Platform.AMD64.class)
 public class AMD64RuntimeCodeInstallerPlatformHelper implements RuntimeCodeInstallerPlatformHelper {
 
     @Override
-    public void performCodeSynchronization(CodeInfo codeInfo) {
+    public boolean needsInstructionCacheSynchronization() {
+        return false;
+    }
+
+    @Override
+    public void performCodeSynchronization(long codeStart, long codeSize) {
     }
 }