diff --git a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/lir/LIRValueUtil.java b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/lir/LIRValueUtil.java index 52708e6b6d49..67780f36e796 100644 --- a/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/lir/LIRValueUtil.java +++ b/compiler/src/jdk.internal.vm.compiler/src/org/graalvm/compiler/lir/LIRValueUtil.java @@ -144,7 +144,7 @@ public static boolean differentRegisters(Object... values) { Register r1 = registers.get(i); for (int j = 0; j < i; j++) { Register r2 = registers.get(j); - assert !r1.equals(r2) : r1 + " appears more than once"; + assert !r1.equals(r2) : r1 + " appears more than once: " + j + " and " + i; } } return true; diff --git a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/AbiUtils.java b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/AbiUtils.java index 047727bc0c85..41f141598e89 100644 --- a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/AbiUtils.java +++ b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/AbiUtils.java @@ -31,8 +31,13 @@ import java.lang.foreign.MemoryLayout; import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodType; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Stream; import org.graalvm.compiler.api.replacements.Fold; @@ -43,7 +48,6 @@ import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.SubstrateTargetDescription; -import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.graal.code.AssignedLocation; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.headers.WindowsAPIs; @@ -67,28 +71,201 @@ public abstract class AbiUtils { @Platforms(Platform.HOSTED_ONLY.class) - public abstract static class Adaptation { - public abstract Class apply(Class parameter); + public static final class Adapter { + private static boolean allEqual(int reference, int... values) { + return Arrays.stream(values).allMatch(v -> v == reference); + } - public abstract ValueNode apply(ValueNode parameter); - } + private Adapter() { + } - @Platforms(Platform.HOSTED_ONLY.class) - public static final class Reinterpret extends Adaptation { - public final JavaKind to; + public enum Extracted { + CallTarget, + CaptureBufferAddress + } - public Reinterpret(JavaKind to) { - this.to = to; + public record AdaptationResult( + Map extractedArguments, + List arguments, + List parametersAssignment, + List returnsAssignment, + MethodType callType) { + public ValueNode getArgument(Extracted id) { + return extractedArguments.get(id); + } } - @Override - public Class apply(Class parameter) { - return to.toJavaClass(); + public static AdaptationResult adapt(AbiUtils self, List adaptations, List originalArguments, NativeEntryPointInfo nep) { + AssignedLocation[] originalAssignment = self.toMemoryAssignment(nep.parametersAssignment(), false); + VMError.guarantee(allEqual(adaptations.size(), originalArguments.size(), nep.methodType().parameterCount(), originalAssignment.length)); + + Map extractedArguments = new EnumMap<>(Extracted.class); + List arguments = new ArrayList<>(); + List assignment = new ArrayList<>(); + List> argumentTypes = new ArrayList<>(); + + for (int i = 0; i < adaptations.size(); ++i) { + Adaptation adaptation = adaptations.get(i); + if (adaptation == null) { + adaptation = NOOP; + } + + arguments.addAll(adaptation.apply(originalArguments.get(i), extractedArguments)); + assignment.addAll(adaptation.apply(originalAssignment[i])); + argumentTypes.addAll(adaptation.apply(nep.methodType().parameterType(i))); + + VMError.guarantee(allEqual(arguments.size(), assignment.size(), argumentTypes.size())); + } + + // Sanity checks + VMError.guarantee(extractedArguments.containsKey(Extracted.CallTarget)); + VMError.guarantee(!nep.capturesCallState() || extractedArguments.containsKey(Extracted.CaptureBufferAddress)); + for (int i = 0; i < arguments.size(); ++i) { + VMError.guarantee(arguments.get(i) != null); + VMError.guarantee(!assignment.get(i).isPlaceholder() || (i == 0 && nep.needsReturnBuffer())); + } + + return new AdaptationResult(extractedArguments, arguments, assignment, Arrays.stream(self.toMemoryAssignment(nep.returnsAssignment(), true)).toList(), + MethodType.methodType(nep.methodType().returnType(), argumentTypes)); } - @Override - public ValueNode apply(ValueNode parameter) { - return ReinterpretNode.reinterpret(to, parameter); + /** + * Allow to define (and later apply) a transformation on a coordinate in arrays of parameter + * types, parameter assignments and concrete arguments at the same time. + * + * E.g. given call with parameter types {@code [long, long, long]}, assignments + * {@code [%rdi, %rsi, %rdx]} and concrete arguments {@code [ 0, 1, 2 ]}, one could define + * the following adaptions {@code [ NOOP, drop(), check(long.class) ]}, which after + * application would yield parameter types {@code [long, long]}, assignments + * {@code [%rdi, %rdx]} and concrete arguments {@code [ 0, 2 ]}. + * + * No real restrictions are set on the actual transformations. The only invariant the + * current implementation expects to hold is that all three methods of one object return the + * same number of elements. + */ + public abstract static class Adaptation { + public abstract List> apply(Class parameter); + + public abstract List apply(AssignedLocation parameter); + + public abstract List apply(ValueNode parameter, Map extractedArguments); + } + + private static final Adaptation NOOP = new Adaptation() { + @Override + public List> apply(Class parameter) { + return List.of(parameter); + } + + @Override + public List apply(AssignedLocation parameter) { + return List.of(parameter); + } + + @Override + public List apply(ValueNode parameter, Map extractedArguments) { + return List.of(parameter); + } + }; + + public static Adaptation check(Class type) { + return new CheckType(Objects.requireNonNull(type)); + } + + public static Adaptation extract(Extracted as, Class type) { + return new Extract(Objects.requireNonNull(as), Objects.requireNonNull(type)); + } + + public static Adaptation drop() { + return Extract.DROP; + } + + public static Adaptation reinterpret(JavaKind to) { + return new Reinterpret(to); + } + + private static final class CheckType extends Adaptation { + private final Class expected; + + private CheckType(Class expected) { + this.expected = expected; + } + + @Override + public List> apply(Class parameter) { + if (parameter != expected) { + throw new IllegalArgumentException("Expected type " + expected + ", got " + parameter); + } + return List.of(parameter); + } + + @Override + public List apply(AssignedLocation parameter) { + return List.of(parameter); + } + + @Override + public List apply(ValueNode parameter, Map extractedArguments) { + return List.of(parameter); + } + } + + private static final class Reinterpret extends Adaptation { + private final JavaKind to; + + private Reinterpret(JavaKind to) { + this.to = to; + } + + @Override + public List> apply(Class parameter) { + return List.of(to.toJavaClass()); + } + + @Override + public List apply(AssignedLocation parameter) { + return List.of(parameter); + } + + @Override + public List apply(ValueNode parameter, Map extractedArguments) { + return List.of(ReinterpretNode.reinterpret(to, parameter)); + } + } + + private static final class Extract extends Adaptation { + private static Extract DROP = new Extract(null, null); + private final Extracted as; + private final Class type; + + private Extract(Extracted as, Class type) { + this.as = as; + this.type = type; + } + + @Override + public List> apply(Class parameter) { + if (type != null && parameter != type) { + throw new IllegalArgumentException("Expected type " + type + ", got " + parameter); + } + return List.of(); + } + + @Override + public List apply(AssignedLocation parameter) { + return List.of(); + } + + @Override + public List apply(ValueNode parameter, Map extractedArguments) { + if (as != null) { + if (extractedArguments.containsKey(as)) { + throw new IllegalStateException("%s was already extracted (%s).".formatted(as, extractedArguments.get(as))); + } + extractedArguments.put(as, parameter); + } + return List.of(); + } } } @@ -106,14 +283,11 @@ public static AbiUtils singleton() { return ImageSingletons.lookup(AbiUtils.class); } - public abstract void methodTypeMatchAssignment(int savedValueMask, MethodType methodType, AssignedLocation[] assignments, AssignedLocation[] returnAssignment, FunctionDescriptor fd, - Linker.Option... options); - /** * This method re-implements a part of the logic from the JDK so that we can get the callee-type * (i.e. the ABI low-level type) of a function from its descriptor. */ - public abstract NativeEntryPointInfo makeEntrypoint(FunctionDescriptor desc, Linker.Option... options); + public abstract NativeEntryPointInfo makeNativeEntrypoint(FunctionDescriptor desc, Linker.Option... options); /** * Generate a register allocation for SubstrateVM from the one generated by and for HotSpot. @@ -121,19 +295,30 @@ public abstract void methodTypeMatchAssignment(int savedValueMask, MethodType me public abstract AssignedLocation[] toMemoryAssignment(VMStorage[] moves, boolean forReturn); /** - * Generate additional argument adaptations which are not done by HotSpot. Note that, unlike - * {@link AbiUtils#toMemoryAssignment}, this information is not part of - * {@link NativeEntryPointInfo}. The reason for that being that this information is not relevant - * at runtime, so we don't want ot include it in {@link NativeEntryPointInfo}, which lives at - * runtime. This information is only useful when generating the - * {@link com.oracle.svm.hosted.foreign.DowncallStub}. - *

- * For convenience, there is an adaptation for every argument (except the NEP itself) to the - * entrypoint, i.e. including the special prefix arguments (call address, return buffer, call - * state buffer), even though these should most likely never be adapted. + * Apply some ABI-specific transformations to an entrypoint (info) and arguments intended to be + * used to call said entrypoint. + */ + @Platforms(Platform.HOSTED_ONLY.class) + public final Adapter.AdaptationResult adapt(List arguments, NativeEntryPointInfo nep) { + return Adapter.adapt(this, generateAdaptations(nep), arguments, nep); + } + + /** + * Generate additional argument adaptations which are not done by HotSpot. */ @Platforms(Platform.HOSTED_ONLY.class) - public abstract Adaptation[] adaptArguments(NativeEntryPointInfo nep); + protected List generateAdaptations(NativeEntryPointInfo nep) { + List adaptations = new ArrayList<>(Collections.nCopies(nep.methodType().parameterCount(), null)); + int current = 0; + if (nep.needsReturnBuffer()) { + adaptations.set(current++, Adapter.check(long.class)); + } + adaptations.set(current++, Adapter.extract(Adapter.Extracted.CallTarget, long.class)); + if (nep.capturesCallState()) { + adaptations.set(current++, Adapter.extract(Adapter.Extracted.CaptureBufferAddress, long.class)); + } + return adaptations; + } @Platforms(Platform.HOSTED_ONLY.class) public abstract void checkLibrarySupport(); @@ -163,13 +348,7 @@ private String name() { } @Override - public void methodTypeMatchAssignment(int savedValueMask, MethodType methodType, AssignedLocation[] assignments, AssignedLocation[] returnAssignment, FunctionDescriptor fd, - Linker.Option... options) { - fail(); - } - - @Override - public NativeEntryPointInfo makeEntrypoint(FunctionDescriptor desc, Linker.Option... options) { + public NativeEntryPointInfo makeNativeEntrypoint(FunctionDescriptor desc, Linker.Option... options) { return fail(); } @@ -179,7 +358,7 @@ public AssignedLocation[] toMemoryAssignment(VMStorage[] moves, boolean forRetur } @Override - public Adaptation[] adaptArguments(NativeEntryPointInfo nep) { + protected List generateAdaptations(NativeEntryPointInfo nep) { return fail(); } @@ -195,80 +374,32 @@ public Map canonicalLayouts() { } private abstract static class X86_64 extends AbiUtils { - protected static Stream argMoveBindingsStream(CallingSequence callingSequence) { - return callingSequence.argumentBindings() - .filter(Binding.VMStore.class::isInstance) - .map(Binding.VMStore.class::cast); - } - - protected static Stream retMoveBindingsStream(CallingSequence callingSequence) { - return callingSequence.returnBindings().stream() - .filter(Binding.VMLoad.class::isInstance) - .map(Binding.VMLoad.class::cast); - } - - protected static Binding.VMLoad[] retMoveBindings(CallingSequence callingSequence) { - return retMoveBindingsStream(callingSequence).toArray(Binding.VMLoad[]::new); - } - - protected VMStorage[] toStorageArray(Binding.Move[] moves) { - return Arrays.stream(moves).map(Binding.Move::storage).toArray(VMStorage[]::new); - } - - protected abstract CallingSequence makeCallingSequence(MethodType type, FunctionDescriptor desc, boolean forUpcall, LinkerOptions options); - - protected boolean typeMatchRegister(AMD64 target, Class type, Register register, boolean isVararg) { - Register.RegisterCategory rc = register.getRegisterCategory(); - PlatformKind kind = target.getPlatformKind(JavaKind.fromJavaClass(type)); - return target.canStoreValue(rc, kind); - } - - @Override - public void methodTypeMatchAssignment(int savedValueMask, MethodType methodType, AssignedLocation[] assignments, AssignedLocation[] returnAssignment, FunctionDescriptor fd, - Linker.Option... options) { - if (!SubstrateUtil.assertionsEnabled()) { - return; + static class Downcalls { + protected static Stream argMoveBindingsStream(CallingSequence callingSequence) { + return callingSequence.argumentBindings() + .filter(Binding.VMStore.class::isInstance) + .map(Binding.VMStore.class::cast); } - int firstActualArgument = 0; - if (methodType.parameterType(firstActualArgument++) != long.class) { - throw new AssertionError("Address expected as first param: " + methodType); - } - if (returnAssignment != null && methodType.parameterType(firstActualArgument++) != long.class) { - throw new AssertionError("Return buffer address expected: " + methodType); - } - if (savedValueMask != 0 && methodType.parameterType(firstActualArgument++) != long.class) { - throw new AssertionError("Capture buffer address expected: " + methodType); + protected static Stream retMoveBindingsStream(CallingSequence callingSequence) { + return callingSequence.returnBindings().stream() + .filter(Binding.VMLoad.class::isInstance) + .map(Binding.VMLoad.class::cast); } - /* - * This assertion can fail if, the entrypoint is supposed to capture the call state, but - * the provided mask is 0, i.e. a capture call state option is provided but with an - * empty capture list. - */ - /* - * TODO: check for capture more uniformly; either always rely on the presence of the - * option (even if empty), or on the non-zeroness of the mask. Note that part of this - * mismatch might also be coming from JDK code, in which case we cannot do much. - */ - assert firstActualArgument + assignments.length == methodType.parameterCount() : assignments.length + " " + methodType.parameterCount(); - AMD64 target = (AMD64) ImageSingletons.lookup(SubstrateTargetDescription.class).arch; - LinkerOptions optionsSet = LinkerOptions.forDowncall(fd, options); - - for (int i = 0; i < assignments.length; ++i) { - var type = methodType.parameterType(firstActualArgument + i); - var assignment = assignments[i]; - assert !assignment.assignsToRegister() || - typeMatchRegister(target, type, assignment.register(), - optionsSet.isVarargsIndex(i)) : "Cannot put %s in %s.\nDescriptor & options: %s %s\nAssignment: %s\nMethod type (placeholders: %d): %s" - .formatted(type, assignment.register(), fd, Arrays.toString(options), Arrays.toString(assignments), firstActualArgument, methodType); + protected static Binding.VMLoad[] retMoveBindings(CallingSequence callingSequence) { + return retMoveBindingsStream(callingSequence).toArray(Binding.VMLoad[]::new); } - assert returnAssignment == null || methodType.returnType().equals(void.class); + protected static VMStorage[] toStorageArray(Binding.Move[] moves) { + return Arrays.stream(moves).map(Binding.Move::storage).toArray(VMStorage[]::new); + } } + protected abstract CallingSequence makeCallingSequence(MethodType type, FunctionDescriptor desc, boolean forUpcall, LinkerOptions options); + @Override - public NativeEntryPointInfo makeEntrypoint(FunctionDescriptor desc, Linker.Option... options) { + public NativeEntryPointInfo makeNativeEntrypoint(FunctionDescriptor desc, Linker.Option... options) { // Linker.downcallHandle implemented in // AbstractLinker.downcallHandle @@ -285,62 +416,55 @@ public NativeEntryPointInfo makeEntrypoint(FunctionDescriptor desc, Linker.Optio /* OS SPECIFIC ENDS */ // DowncallLinker.getBoundMethodHandle - var argMoves = toStorageArray(argMoveBindingsStream(callingSequence).toArray(Binding.VMStore[]::new)); - var returnMoves = toStorageArray(retMoveBindings(callingSequence)); + var argMoves = Downcalls.toStorageArray(Downcalls.argMoveBindingsStream(callingSequence).toArray(Binding.VMStore[]::new)); + var returnMoves = Downcalls.toStorageArray(Downcalls.retMoveBindings(callingSequence)); var boundaryType = callingSequence.calleeMethodType(); var needsReturnBuffer = callingSequence.needsReturnBuffer(); // NativeEntrypoint.make - var parametersAssignment = toMemoryAssignment(argMoves, false); - var returnBuffering = needsReturnBuffer ? toMemoryAssignment(returnMoves, true) : null; - AbiUtils.singleton().methodTypeMatchAssignment(callingSequence.capturedStateMask(), boundaryType, parametersAssignment, returnBuffering, desc, options); return NativeEntryPointInfo.make(argMoves, returnMoves, boundaryType, needsReturnBuffer, callingSequence.capturedStateMask(), callingSequence.needsTransition()); } @Override public AssignedLocation[] toMemoryAssignment(VMStorage[] argMoves, boolean forReturn) { - int size = 0; for (VMStorage move : argMoves) { - if (move.type() != X86_64Architecture.StorageType.PLACEHOLDER) { - // Placeholders are ignored; they will be handled further down the line - ++size; - } else { - // Placeholders are expected to be prefix arguments - assert size == 0 : "Placeholders are expected to be prefix arguments"; - } - - if (move.type() == X86_64Architecture.StorageType.X87) { - throw unsupportedFeature("Unsupported register kind: X87"); - } else if (move.type() == X86_64Architecture.StorageType.STACK && forReturn) { - throw unsupportedFeature("Unsupported register kind for return: STACK"); + switch (move.type()) { + case X86_64Architecture.StorageType.X87 -> + throw unsupportedFeature("Unsupported register kind: X87"); + case X86_64Architecture.StorageType.STACK -> { + if (forReturn) { + throw unsupportedFeature("Unsupported register kind for return: STACK"); + } + } + default -> { + } } } - AssignedLocation[] storages = new AssignedLocation[size]; + AssignedLocation[] storages = new AssignedLocation[argMoves.length]; int i = 0; for (VMStorage move : argMoves) { - if (move.type() != X86_64Architecture.StorageType.PLACEHOLDER) { - storages[i++] = switch (move.type()) { - case X86_64Architecture.StorageType.INTEGER -> { - Register reg = AMD64.cpuRegisters[move.indexOrOffset()]; - assert reg.name.equals(move.debugName()); - assert reg.getRegisterCategory().equals(AMD64.CPU); - yield AssignedLocation.forRegister(reg); - } - case X86_64Architecture.StorageType.VECTOR -> { - /* - * Only the first four xmm registers should ever be used; in particular, - * this means we never need to index in xmmRegistersAVX512 - */ - Register reg = AMD64.xmmRegistersSSE[move.indexOrOffset()]; - assert reg.name.equals(move.debugName()); - assert reg.getRegisterCategory().equals(AMD64.XMM); - yield AssignedLocation.forRegister(reg); - } - case X86_64Architecture.StorageType.STACK -> AssignedLocation.forStack(move.indexOrOffset()); - default -> throw unsupportedFeature("Unhandled VMStorage: " + move); - }; - } + storages[i++] = switch (move.type()) { + case X86_64Architecture.StorageType.PLACEHOLDER -> AssignedLocation.placeholder(); + case X86_64Architecture.StorageType.INTEGER -> { + Register reg = AMD64.cpuRegisters[move.indexOrOffset()]; + assert reg.name.equals(move.debugName()); + assert reg.getRegisterCategory().equals(AMD64.CPU); + yield AssignedLocation.forRegister(reg); + } + case X86_64Architecture.StorageType.VECTOR -> { + /* + * Only the first four xmm registers should ever be used; in particular, + * this means we never need to index in xmmRegistersAVX512 + */ + Register reg = AMD64.xmmRegistersSSE[move.indexOrOffset()]; + assert reg.name.equals(move.debugName()); + assert reg.getRegisterCategory().equals(AMD64.XMM); + yield AssignedLocation.forRegister(reg); + } + case X86_64Architecture.StorageType.STACK -> AssignedLocation.forStack(move.indexOrOffset()); + default -> throw unsupportedFeature("Unhandled VMStorage: " + move); + }; } assert i == storages.length; @@ -385,10 +509,12 @@ protected CallingSequence makeCallingSequence(MethodType type, FunctionDescripto } @Override - @Platforms(Platform.HOSTED_ONLY.class) - public Adaptation[] adaptArguments(NativeEntryPointInfo nep) { - // No adaptations needed - return new Adaptation[nep.linkMethodType().parameterCount()]; + protected List generateAdaptations(NativeEntryPointInfo nep) { + var adaptations = super.generateAdaptations(nep); + /* Drop the rax parametersAssignment */ + assert adaptations.get(adaptations.size() - 1) == null; + adaptations.set(adaptations.size() - 1, Adapter.drop()); + return adaptations; } @Override @@ -410,15 +536,6 @@ protected CallingSequence makeCallingSequence(MethodType type, FunctionDescripto return jdk.internal.foreign.abi.x64.windows.CallArranger.getBindings(type, desc, forUpcall, options).callingSequence(); } - @Override - protected boolean typeMatchRegister(AMD64 target, Class type, Register register, boolean isVararg) { - Register.RegisterCategory rc = register.getRegisterCategory(); - JavaKind kind = JavaKind.fromJavaClass(type); - PlatformKind platformKind = target.getPlatformKind(kind); - return target.canStoreValue(rc, platformKind) || (isVararg && - register.getRegisterCategory().equals(AMD64.CPU) && (kind.equals(JavaKind.Float) || kind.equals(JavaKind.Double))); - } - /** * The Win64 ABI allows one mismatch between register and value type: When a variadic * floating-point argument is among the first four parameters of the function, the argument @@ -430,42 +547,24 @@ protected boolean typeMatchRegister(AMD64 target, Class type, Register regist * assignments of float/double parameters to a cpu register. */ @Override - @Platforms(Platform.HOSTED_ONLY.class) - public Adaptation[] adaptArguments(NativeEntryPointInfo nep) { - /* - * This method also performs some sanity checks about the generated entrypoint. - */ - MethodType methodType = nep.linkMethodType(); - AssignedLocation[] assignments = nep.parametersAssignment(); - - int firstActualArgument = 0; - // Note that the following ifs do increment firstActualArgument ! - if (methodType.parameterType(firstActualArgument++) != long.class && SubstrateUtil.assertionsEnabled()) { - throw new AssertionError("Address expected as first param: " + methodType); - } - if (nep.returnsAssignment() != null && methodType.parameterType(firstActualArgument++) != long.class && SubstrateUtil.assertionsEnabled()) { - throw new AssertionError("Return buffer address expected: " + methodType); - } - if (nep.capturesCallState() && methodType.parameterType(firstActualArgument++) != long.class && SubstrateUtil.assertionsEnabled()) { - throw new AssertionError("Capture buffer address expected: " + methodType); - } + protected List generateAdaptations(NativeEntryPointInfo nep) { + List adaptations = super.generateAdaptations(nep); - assert firstActualArgument + assignments.length == methodType.parameterCount() : assignments.length + " " + methodType.parameterCount(); AMD64 target = (AMD64) ImageSingletons.lookup(SubstrateTargetDescription.class).arch; - Adaptation[] adaptations = new Adaptation[methodType.parameterCount()]; - - for (int i = firstActualArgument; i < adaptations.length; ++i) { - Class type = methodType.parameterType(i); - AssignedLocation assignment = assignments[i - firstActualArgument]; - - if (assignment.assignsToRegister()) { - Register.RegisterCategory rc = assignment.register().getRegisterCategory(); - PlatformKind kind = target.getPlatformKind(JavaKind.fromJavaClass(type)); - if (!(target.canStoreValue(rc, kind))) { - assert rc.equals(AMD64.CPU) && (kind.equals(target.getPlatformKind(JavaKind.Float)) || kind.equals(target.getPlatformKind(JavaKind.Double))); - adaptations[i] = new Reinterpret(JavaKind.Long); - } + boolean previousMatched = false; + PlatformKind previousKind = null; + for (int i = adaptations.size() - 1; i >= 0; --i) { + PlatformKind kind = target.getPlatformKind(JavaKind.fromJavaClass(nep.methodType().parameterType(i))); + if ((kind.equals(target.getPlatformKind(JavaKind.Float)) || kind.equals(target.getPlatformKind(JavaKind.Double))) && + nep.parametersAssignment()[i].type() == X86_64Architecture.StorageType.INTEGER) { + assert Objects.equals(previousKind, kind) && previousMatched; + assert adaptations.get(i) == null; + adaptations.set(i, Adapter.reinterpret(JavaKind.Long)); + previousMatched = false; + } else { + previousMatched = true; } + previousKind = kind; } return adaptations; diff --git a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/DowncallStubsHolder.java b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/DowncallStubsHolder.java index 80405242b6f9..e76cc2b45af9 100644 --- a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/DowncallStubsHolder.java +++ b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/DowncallStubsHolder.java @@ -53,14 +53,14 @@ public static ConstantPool getConstantPool(MetaAccessProvider metaAccess) { */ @Platforms(Platform.HOSTED_ONLY.class) public static String stubName(NativeEntryPointInfo nep) { - StringBuilder builder = new StringBuilder("downcall_("); - for (var param : nep.nativeMethodType().parameterArray()) { + StringBuilder builder = new StringBuilder("downcall_"); + for (var param : nep.methodType().parameterArray()) { builder.append(JavaKind.fromJavaClass(param).getTypeChar()); } - builder.append(")"); - builder.append(JavaKind.fromJavaClass(nep.nativeMethodType().returnType()).getTypeChar()); + builder.append("_"); + builder.append(JavaKind.fromJavaClass(nep.methodType().returnType()).getTypeChar()); - if (nep.returnsAssignment() != null) { + if (nep.needsReturnBuffer()) { builder.append("_r"); } if (nep.capturesCallState()) { diff --git a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/ForeignFunctionsRuntime.java b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/ForeignFunctionsRuntime.java index f98c4c4927c7..566369edce45 100644 --- a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/ForeignFunctionsRuntime.java +++ b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/ForeignFunctionsRuntime.java @@ -30,7 +30,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.function.CFunctionPointer; -import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.type.CIntPointer; import org.graalvm.word.LocationIdentity; @@ -51,15 +50,15 @@ public static ForeignFunctionsRuntime singleton() { return ImageSingletons.lookup(ForeignFunctionsRuntime.class); } - private final EconomicMap stubs = EconomicMap.create(); + private final EconomicMap downcallStubs = EconomicMap.create(); public ForeignFunctionsRuntime() { } @Platforms(Platform.HOSTED_ONLY.class) - public void addStubPointer(NativeEntryPointInfo nepi, CFunctionPointer ptr) { - VMError.guarantee(!stubs.containsKey(nepi), "Multiple stubs generated for " + nepi); - stubs.put(nepi, new FunctionPointerHolder(ptr)); + public void addDowncallStubPointer(NativeEntryPointInfo nep, CFunctionPointer ptr) { + VMError.guarantee(!downcallStubs.containsKey(nep), "Seems like multiple stubs were generated for " + nep); + VMError.guarantee(downcallStubs.put(nep, new FunctionPointerHolder(ptr)) == null); } /** @@ -68,26 +67,24 @@ public void addStubPointer(NativeEntryPointInfo nepi, CFunctionPointer ptr) { * {@link jdk.internal.foreign.abi.DowncallLinker#getBoundMethodHandle} and add information * about the descriptor there. */ - public CodePointer getStubPointer(NativeEntryPointInfo nep) { - FunctionPointerHolder pointer = stubs.get(nep); + public CFunctionPointer getDowncallStubPointer(NativeEntryPointInfo nep) { + FunctionPointerHolder pointer = downcallStubs.get(nep); if (pointer == null) { - throw new UnregisteredDowncallStubException(nep); + throw new UnregisteredForeignStubException(nep); } else { return pointer.functionPointer; } } @SuppressWarnings("serial") - public static class UnregisteredDowncallStubException extends RuntimeException { - private final NativeEntryPointInfo nep; + public static class UnregisteredForeignStubException extends RuntimeException { - UnregisteredDowncallStubException(NativeEntryPointInfo nep) { + UnregisteredForeignStubException(NativeEntryPointInfo nep) { super(generateMessage(nep)); - this.nep = nep; } private static String generateMessage(NativeEntryPointInfo nep) { - return "Cannot perform downcall with leaf type " + nep.nativeMethodType() + " as it was not registered at compilation time."; + return "Cannot perform downcall with leaf type " + nep.methodType() + " as it was not registered at compilation time."; } } @@ -138,6 +135,7 @@ public static void captureCallState(int statesToCapture, CIntPointer captureBuff ++i; } + @Platforms(Platform.HOSTED_ONLY.class)// public static final SnippetRuntime.SubstrateForeignCallDescriptor CAPTURE_CALL_STATE = SnippetRuntime.findForeignCall(ForeignFunctionsRuntime.class, "captureCallState", false, LocationIdentity.any()); } diff --git a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/NativeEntryPointInfo.java b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/NativeEntryPointInfo.java index 291097bd452f..3e5dca8ba38d 100644 --- a/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/NativeEntryPointInfo.java +++ b/substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/NativeEntryPointInfo.java @@ -28,17 +28,15 @@ import java.util.Arrays; import java.util.Objects; -import com.oracle.svm.core.graal.code.AssignedLocation; - import jdk.internal.foreign.abi.ABIDescriptor; import jdk.internal.foreign.abi.VMStorage; /** * Carries information about an entrypoint for foreign function calls. - * {@link ForeignFunctionsRuntime#getStubPointer} allows getting the associated function pointer at - * runtime (if it exists). + * {@link ForeignFunctionsRuntime#getDowncallStubPointer} allows getting the associated function + * pointer at runtime (if it exists). *

- * {@link NativeEntryPointInfo#linkMethodType} is of the form (<>: argument; []: optional argument) + * {@link NativeEntryPointInfo#methodType} is of the form (<>: argument; []: optional argument) * *

  * {@code
@@ -50,15 +48,19 @@
  */
 public final class NativeEntryPointInfo {
     private final MethodType methodType;
-    private final AssignedLocation[] parameterAssignments;
-    private final AssignedLocation[] returnBuffering;
+    private final VMStorage[] parameterAssignments;
+    private final VMStorage[] returnBuffering;
+    private final boolean needsReturnBuffer;
     private final boolean capturesState;
     private final boolean needsTransition;
 
-    private NativeEntryPointInfo(MethodType methodType, AssignedLocation[] cc, AssignedLocation[] returnBuffering, boolean capturesState, boolean needsTransition) {
+    private NativeEntryPointInfo(MethodType methodType, VMStorage[] cc, VMStorage[] returnBuffering, boolean needsReturnBuffer, boolean capturesState, boolean needsTransition) {
+        assert methodType.parameterCount() == cc.length;
+        assert needsReturnBuffer == (returnBuffering.length > 1);
         this.methodType = methodType;
         this.parameterAssignments = cc;
         this.returnBuffering = returnBuffering;
+        this.needsReturnBuffer = needsReturnBuffer;
         this.capturesState = capturesState;
         this.needsTransition = needsTransition;
     }
@@ -69,13 +71,10 @@ public static NativeEntryPointInfo make(
                     boolean needsReturnBuffer,
                     int capturedStateMask,
                     boolean needsTransition) {
-        if (returnMoves.length > 1 != needsReturnBuffer) {
+        if ((returnMoves.length > 1) != needsReturnBuffer) {
             throw new AssertionError("Multiple register return, but needsReturnBuffer was false");
         }
-
-        AssignedLocation[] parametersAssignment = AbiUtils.singleton().toMemoryAssignment(argMoves, false);
-        AssignedLocation[] returnBuffering = needsReturnBuffer ? AbiUtils.singleton().toMemoryAssignment(returnMoves, true) : null;
-        return new NativeEntryPointInfo(methodType, parametersAssignment, returnBuffering, capturedStateMask != 0, needsTransition);
+        return new NativeEntryPointInfo(methodType, argMoves, returnMoves, needsReturnBuffer, capturedStateMask != 0, needsTransition);
     }
 
     public static Target_jdk_internal_foreign_abi_NativeEntryPoint makeEntryPoint(
@@ -86,8 +85,8 @@ public static Target_jdk_internal_foreign_abi_NativeEntryPoint makeEntryPoint(
                     int capturedStateMask,
                     boolean needsTransition) {
         var info = make(argMoves, returnMoves, methodType, needsReturnBuffer, capturedStateMask, needsTransition);
-        long addr = ForeignFunctionsRuntime.singleton().getStubPointer(info).rawValue();
-        return new Target_jdk_internal_foreign_abi_NativeEntryPoint(info.linkMethodType(), addr, capturedStateMask);
+        long addr = ForeignFunctionsRuntime.singleton().getDowncallStubPointer(info).rawValue();
+        return new Target_jdk_internal_foreign_abi_NativeEntryPoint(info.methodType(), addr, capturedStateMask);
     }
 
     public int callAddressIndex() {
@@ -101,38 +100,23 @@ public int captureAddressIndex() {
         return callAddressIndex() + 1;
     }
 
-    /**
-     * Method type without any of the special arguments.
-     */
-    public MethodType nativeMethodType() {
-        if (capturesCallState()) {
-            return this.methodType.dropParameterTypes(0, captureAddressIndex() + 1);
-        } else {
-            return this.methodType.dropParameterTypes(0, callAddressIndex() + 1);
-        }
-    }
-
-    /**
-     * Method type with all special arguments.
-     */
-    public MethodType linkMethodType() {
+    public MethodType methodType() {
         return this.methodType;
     }
 
     public boolean needsReturnBuffer() {
-        return this.returnBuffering != null;
+        return needsReturnBuffer;
     }
 
     public boolean capturesCallState() {
         return capturesState;
     }
 
-    public AssignedLocation[] parametersAssignment() {
-        assert parameterAssignments.length == this.nativeMethodType().parameterCount() : Arrays.toString(parameterAssignments) + " ; " + nativeMethodType();
+    public VMStorage[] parametersAssignment() {
         return parameterAssignments;
     }
 
-    public AssignedLocation[] returnsAssignment() {
+    public VMStorage[] returnsAssignment() {
         return returnBuffering;
     }
 
@@ -149,12 +133,12 @@ public boolean equals(Object o) {
             return false;
         }
         NativeEntryPointInfo that = (NativeEntryPointInfo) o;
-        return capturesState == that.capturesState && needsTransition == that.needsTransition && Objects.equals(methodType, that.methodType) &&
+        return capturesState == that.capturesState && needsTransition == that.needsTransition && needsReturnBuffer == that.needsReturnBuffer && Objects.equals(methodType, that.methodType) &&
                         Arrays.equals(parameterAssignments, that.parameterAssignments) && Arrays.equals(returnBuffering, that.returnBuffering);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(methodType, capturesState, needsTransition, Arrays.hashCode(parameterAssignments), Arrays.hashCode(returnBuffering));
+        return Objects.hash(methodType, needsReturnBuffer, capturesState, needsTransition, Arrays.hashCode(parameterAssignments), Arrays.hashCode(returnBuffering));
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java
index addb3bb6239e..0c971310952f 100644
--- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java
+++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java
@@ -28,7 +28,6 @@
 import static com.oracle.svm.core.graal.code.SubstrateBackend.SubstrateMarkId.PROLOGUE_END;
 import static com.oracle.svm.core.util.VMError.shouldNotReachHere;
 import static com.oracle.svm.core.util.VMError.unsupportedFeature;
-import static jdk.vm.ci.amd64.AMD64.r10;
 import static jdk.vm.ci.amd64.AMD64.rax;
 import static jdk.vm.ci.amd64.AMD64.rbp;
 import static jdk.vm.ci.amd64.AMD64.rsp;
@@ -39,6 +38,7 @@
 import static org.graalvm.compiler.lir.LIRValueUtil.asConstantValue;
 import static org.graalvm.compiler.lir.LIRValueUtil.differentRegisters;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
@@ -278,9 +278,18 @@ public static class SubstrateAMD64IndirectCallOp extends AMD64Call.IndirectCallO
         @Temp({REG, OperandFlag.ILLEGAL}) private Value exceptionTemp;
         private final BiConsumer offsetRecorder;
 
+        @Def({REG}) private Value[] multipleResults;
+
         public SubstrateAMD64IndirectCallOp(ResolvedJavaMethod callTarget, Value result, Value[] parameters, Value[] temps, Value targetAddress,
                         LIRFrameState state, Value javaFrameAnchor, Value javaFrameAnchorTemp, int newThreadStatus, boolean destroysCallerSavedRegisters, Value exceptionTemp,
                         BiConsumer offsetRecorder) {
+            this(callTarget, result, parameters, temps, targetAddress, state, javaFrameAnchor, javaFrameAnchorTemp, newThreadStatus, destroysCallerSavedRegisters, exceptionTemp, offsetRecorder,
+                            new Value[0]);
+        }
+
+        public SubstrateAMD64IndirectCallOp(ResolvedJavaMethod callTarget, Value result, Value[] parameters, Value[] temps, Value targetAddress,
+                        LIRFrameState state, Value javaFrameAnchor, Value javaFrameAnchorTemp, int newThreadStatus, boolean destroysCallerSavedRegisters, Value exceptionTemp,
+                        BiConsumer offsetRecorder, Value[] multipleResults) {
             super(TYPE, callTarget, result, parameters, temps, targetAddress, state);
             this.newThreadStatus = newThreadStatus;
             this.javaFrameAnchor = javaFrameAnchor;
@@ -288,6 +297,7 @@ public SubstrateAMD64IndirectCallOp(ResolvedJavaMethod callTarget, Value result,
             this.destroysCallerSavedRegisters = destroysCallerSavedRegisters;
             this.exceptionTemp = exceptionTemp;
             this.offsetRecorder = offsetRecorder;
+            this.multipleResults = multipleResults;
 
             assert differentRegisters(parameters, temps, targetAddress, javaFrameAnchor, javaFrameAnchorTemp);
         }
@@ -853,21 +863,40 @@ protected void prologSetParameterNodes(StructuredGraph graph, Value[] params) {
         @Override
         public Value[] visitInvokeArguments(CallingConvention invokeCc, Collection arguments) {
             Value[] values = super.visitInvokeArguments(invokeCc, arguments);
-
             SubstrateCallingConventionType type = (SubstrateCallingConventionType) ((SubstrateCallingConvention) invokeCc).getType();
+
+            if (type.usesReturnBuffer()) {
+                /*
+                 * We save the return buffer so that it can be accessed after the call.
+                 */
+                assert values.length > 0;
+                Value returnBuffer = values[0];
+                Variable saved = gen.newVariable(returnBuffer.getValueKind());
+                gen.append(gen.getSpillMoveFactory().createMove(saved, returnBuffer));
+                values[0] = saved;
+            }
+
             if (type.nativeABI()) {
+                VMError.guarantee(values.length == invokeCc.getArgumentCount() - 1, "The last argument should be missing.");
+                AllocatableValue raxValue = invokeCc.getArgument(values.length);
+                VMError.guarantee(raxValue instanceof RegisterValue && ((RegisterValue) raxValue).getRegister().equals(rax));
+
+                values = Arrays.copyOf(values, values.length + 1);
+
                 // Native functions might have varargs, in which case we need to set %al to the
                 // number of XMM registers used for passing arguments
                 int xmmCount = 0;
-                for (Value v : values) {
+                for (int i = 0; i < values.length - 1; ++i) {
+                    Value v = values[i];
                     if (isRegister(v) && asRegister(v).getRegisterCategory().equals(AMD64.XMM)) {
                         xmmCount++;
                     }
                 }
                 assert xmmCount <= 8;
-                AllocatableValue xmmCountRegister = AMD64.rax.asValue(LIRKind.value(AMD64Kind.DWORD));
-                gen.emitMoveConstant(xmmCountRegister, JavaConstant.forInt(xmmCount));
+                gen.emitMoveConstant(raxValue, JavaConstant.forInt(xmmCount));
+                values[values.length - 1] = raxValue;
             }
+
             return values;
         }
 
@@ -889,41 +918,43 @@ public BiConsumer getOffsetRecorder(@Suppress
             return null;
         }
 
+        private static AllocatableValue asReturnedValue(AssignedLocation assignedLocation) {
+            assert assignedLocation.assignsToRegister();
+            Register.RegisterCategory category = assignedLocation.register().getRegisterCategory();
+            LIRKind kind;
+            if (category.equals(AMD64.CPU)) {
+                kind = LIRKind.value(AMD64Kind.QWORD);
+            } else if (category.equals(AMD64.XMM)) {
+                kind = LIRKind.value(AMD64Kind.V128_DOUBLE);
+            } else {
+                throw unsupportedFeature("Register category " + category + " should not be used for returns spanning multiple registers.");
+            }
+            return assignedLocation.register().asValue(kind);
+        }
+
         @Override
         protected void emitInvoke(LoweredCallTargetNode callTarget, Value[] parameters, LIRFrameState callState, Value result) {
+            var cc = (SubstrateCallingConventionType) callTarget.callType();
             verifyCallTarget(callTarget);
             if (callTarget instanceof ComputedIndirectCallTargetNode) {
+                assert !cc.customABI();
                 emitComputedIndirectCall((ComputedIndirectCallTargetNode) callTarget, result, parameters, AllocatableValue.NONE, callState);
             } else {
                 super.emitInvoke(callTarget, parameters, callState, result);
             }
 
-            var cc = (SubstrateCallingConventionType) callTarget.callType();
-            if (cc.returnSaving != null) {
-                // The pointer to the return buffer is passed as first argument
-                Value baseSaveLocation = parameters[0];
-                // Could be any x86 scratch register
-                RegisterValue scratch = r10.asValue(parameters[0].getValueKind());
-                gen.emitMove(scratch, baseSaveLocation);
+            if (cc.usesReturnBuffer()) {
+                /*
+                 * The buffer argument was saved in visitInvokeArguments, so that the value was not
+                 * killed by the call.
+                 */
+                Value returnBuffer = parameters[0];
                 long offset = 0;
                 for (AssignedLocation ret : cc.returnSaving) {
-                    Value saveLocation = gen.getArithmetic().emitAdd(scratch, gen.emitJavaConstant(JavaConstant.forLong(offset)), false);
-                    if (ret.assignsToStack()) {
-                        throw unsupportedFeature("Return should never happen on stack.");
-                    }
-                    var register = ret.register();
-                    LIRKind kind;
-                    // There might a better/more natural way to check this
-                    if (register.getRegisterCategory().equals(AMD64.CPU)) {
-                        kind = gen.getValueKind(JavaKind.Long);
-                        offset += 8;
-                    } else if (register.getRegisterCategory().equals(AMD64.XMM)) {
-                        kind = gen.getValueKind(JavaKind.Double);
-                        offset += 16;
-                    } else {
-                        throw unsupportedFeature("Cannot use register " + register + " for return.");
-                    }
-                    gen.getArithmetic().emitStore(kind, saveLocation, gen.emitReadRegister(register, kind), callState, MemoryOrderMode.PLAIN);
+                    Value saveLocation = gen.getArithmetic().emitAdd(returnBuffer, gen.emitJavaConstant(JavaConstant.forLong(offset)), false);
+                    AllocatableValue returnedValue = asReturnedValue(ret);
+                    gen.getArithmetic().emitStore(returnedValue.getValueKind(), saveLocation, returnedValue, callState, MemoryOrderMode.PLAIN);
+                    offset += returnedValue.getPlatformKind().getSizeInBytes();
                 }
             }
         }
@@ -949,9 +980,18 @@ protected void emitIndirectCall(IndirectCallTargetNode callTarget, Value result,
             gen.emitMove(targetAddress, operand(callTarget.computedAddress()));
             ResolvedJavaMethod targetMethod = callTarget.targetMethod();
             vzeroupperBeforeCall((SubstrateAMD64LIRGenerator) getLIRGeneratorTool(), parameters, callState, (SharedMethod) targetMethod);
+
+            Value[] multipleResults = new Value[0];
+            var cc = (SubstrateCallingConventionType) callTarget.callType();
+            if (cc.customABI() && cc.usesReturnBuffer()) {
+                multipleResults = Arrays.stream(cc.returnSaving)
+                                .map(SubstrateAMD64NodeLIRBuilder::asReturnedValue)
+                                .toList().toArray(new Value[0]);
+            }
+
             append(new SubstrateAMD64IndirectCallOp(targetMethod, result, parameters, temps, targetAddress, callState,
                             setupJavaFrameAnchor(callTarget), setupJavaFrameAnchorTemp(callTarget), getNewThreadStatus(callTarget),
-                            getDestroysCallerSavedRegisters(targetMethod), getExceptionTemp(callTarget), getOffsetRecorder(callTarget)));
+                            getDestroysCallerSavedRegisters(targetMethod), getExceptionTemp(callTarget), getOffsetRecorder(callTarget), multipleResults));
         }
 
         protected void emitComputedIndirectCall(ComputedIndirectCallTargetNode callTarget, Value result, Value[] parameters, Value[] temps, LIRFrameState callState) {
@@ -1367,7 +1407,9 @@ private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) {
     @Override
     public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterAllocationConfig registerAllocationConfig, StructuredGraph graph, Object stub) {
         SharedMethod method = (SharedMethod) graph.method();
-        CallingConvention callingConvention = CodeUtil.getCallingConvention(getCodeCache(), method.getCallingConventionKind().toType(false), method, this);
+        SubstrateCallingConventionKind ccKind = method.getCallingConventionKind();
+        SubstrateCallingConventionType ccType = ccKind.isCustom() ? method.getCustomCallingConventionType() : ccKind.toType(false);
+        CallingConvention callingConvention = CodeUtil.getCallingConvention(getCodeCache(), ccType, method, this);
         return new SubstrateLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerAllocationConfig.getRegisterConfig()), callingConvention, registerAllocationConfig, method);
     }
 
diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64RegisterConfig.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64RegisterConfig.java
index ddaee0d1193b..164a46414637 100644
--- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64RegisterConfig.java
+++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64RegisterConfig.java
@@ -32,6 +32,7 @@
 import static jdk.vm.ci.amd64.AMD64.k5;
 import static jdk.vm.ci.amd64.AMD64.k6;
 import static jdk.vm.ci.amd64.AMD64.k7;
+import static jdk.vm.ci.amd64.AMD64.r11;
 import static jdk.vm.ci.amd64.AMD64.r12;
 import static jdk.vm.ci.amd64.AMD64.r13;
 import static jdk.vm.ci.amd64.AMD64.r14;
@@ -64,9 +65,11 @@
 import static jdk.vm.ci.amd64.AMD64.xmm9;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 
+import org.graalvm.compiler.core.common.LIRKind;
 import org.graalvm.nativeimage.Platform;
 
 import com.oracle.svm.core.ReservedRegisters;
@@ -81,6 +84,7 @@
 import com.oracle.svm.core.util.VMError;
 
 import jdk.vm.ci.amd64.AMD64;
+import jdk.vm.ci.amd64.AMD64Kind;
 import jdk.vm.ci.code.CallingConvention;
 import jdk.vm.ci.code.CallingConvention.Type;
 import jdk.vm.ci.code.Register;
@@ -260,26 +264,33 @@ public CallingConvention getCallingConvention(Type t, JavaType returnType, JavaT
          * We have to reserve a slot between return address and outgoing parameters for the deopt
          * frame handle. Exception: calls to native methods.
          */
-        int currentStackOffset = (type.nativeABI() ? nativeParamsStackOffset : target.wordSize);
+        int currentStackOffset = type.nativeABI() ? nativeParamsStackOffset : target.wordSize;
 
         AllocatableValue[] locations = new AllocatableValue[parameterTypes.length];
-        int firstActualArgument = 0;
-
         JavaKind[] kinds = new JavaKind[locations.length];
 
-        if (type.returnSaving != null) {
+        int firstActualArgument = 0;
+        if (type.usesReturnBuffer()) {
+            VMError.guarantee(type.fixedParameterAssignment != null);
+            VMError.guarantee(type.fixedParameterAssignment[0].isPlaceholder());
             /*
              * returnSaving implies an additional (prefix) parameter pointing to the buffer to use
-             * for saving. We pretend it is the first stack argument to the function: this means the
-             * function will safely ignore it, but we will be able to access it right after the call
-             * concludes.
+             * for saving. This argument is not actually used by the function, so it will be ignored
+             * in the remainder of this method.
              */
             firstActualArgument = 1;
             /*
-             * The actual allocation is done after allocating all other parameters, as it must be
-             * done last (it needs to be placed first on the stack, and this is done in reverse
-             * order)
+             * Ideally, we would just pretend this argument never existed and would not give it a
+             * location. In practice, it is not so simple, as the generated calling convention is
+             * expected to match the arguments, and not just ignore one of them. It might be
+             * possible to implement this using some kind of "SinkValue" as the location of the
+             * argument. In the meantime, we put it in a scratch register. r10 contains the target,
+             * rax the number of vector args, so r11 is the only scratch register left.
              */
+            JavaKind kind = ObjectLayout.getCallSignatureKind(isEntryPoint, (ResolvedJavaType) parameterTypes[0], metaAccess, target);
+            kinds[0] = kind;
+            ValueKind paramValueKind = valueKindFactory.getValueKind(isEntryPoint ? kind : kind.getStackKind());
+            locations[0] = r11.asValue(paramValueKind);
         }
 
         if (type.fixedParameterAssignment == null) {
@@ -346,50 +357,59 @@ public CallingConvention getCallingConvention(Type t, JavaType returnType, JavaT
         } else {
             final int baseStackOffset = currentStackOffset;
             Set usedRegisters = new HashSet<>();
-            VMError.guarantee(parameterTypes.length == type.fixedParameterAssignment.length + firstActualArgument,
-                            "Parameters/assignments size mismatch.");
+            VMError.guarantee(parameterTypes.length == type.fixedParameterAssignment.length, "Parameters/assignments size mismatch.");
 
-            for (int i = firstActualArgument; i < parameterTypes.length; i++) {
+            for (int i = firstActualArgument; i < locations.length; i++) {
                 JavaKind kind = ObjectLayout.getCallSignatureKind(isEntryPoint, (ResolvedJavaType) parameterTypes[i], metaAccess, target);
                 kinds[i] = kind;
 
                 ValueKind paramValueKind = valueKindFactory.getValueKind(isEntryPoint ? kind : kind.getStackKind());
 
-                int actualArgumentIndex = i - firstActualArgument;
-                AssignedLocation storage = type.fixedParameterAssignment[actualArgumentIndex];
+                AssignedLocation storage = type.fixedParameterAssignment[i];
                 if (storage.assignsToRegister()) {
                     if (!kind.isNumericInteger() && !kind.isNumericFloat()) {
                         throw unsupportedFeature("Unsupported storage/kind pair - Storage: " + storage + " ; Kind: " + kind);
                     }
                     Register reg = storage.register();
-                    VMError.guarantee(target.arch.canStoreValue(reg.getRegisterCategory(),
-                                    paramValueKind.getPlatformKind()), "Cannot assign value to register.");
+                    VMError.guarantee(target.arch.canStoreValue(reg.getRegisterCategory(), paramValueKind.getPlatformKind()), "Cannot assign value to register.");
                     locations[i] = reg.asValue(paramValueKind);
-                    VMError.guarantee(!usedRegisters.contains(reg),
-                                    "Register was already used.");
+                    VMError.guarantee(!usedRegisters.contains(reg), "Register was already used.");
                     usedRegisters.add(reg);
-                } else {
+                } else if (storage.assignsToStack()) {
                     /*
                      * There should be no "empty spaces" between arguments on the stack. This
                      * assertion checks so, but assumes that stack arguments are encountered
                      * "in order". This assumption might not be necessary, but simplifies the check
                      * tremendously.
                      */
-                    VMError.guarantee(currentStackOffset == baseStackOffset + storage.stackOffset(),
-                                    "Potential stack ``completeness'' violation.");
+                    VMError.guarantee(currentStackOffset == baseStackOffset + storage.stackOffset(), "Potential stack ``completeness'' violation.");
                     locations[i] = StackSlot.get(paramValueKind, currentStackOffset, !type.outgoing);
                     currentStackOffset += Math.max(paramValueKind.getPlatformKind().getSizeInBytes(), target.wordSize);
+                } else {
+                    VMError.shouldNotReachHere("Placeholder assignment.");
                 }
             }
         }
 
-        if (type.returnSaving != null) {
-            assert type.fixedParameterAssignment == null || type.fixedParameterAssignment.length + 1 == locations.length;
-            assert parameterTypes[0].getJavaKind() == JavaKind.Long;
-            JavaKind kind = ObjectLayout.getCallSignatureKind(isEntryPoint, (ResolvedJavaType) parameterTypes[0], metaAccess, target);
-            ValueKind paramValueKind = valueKindFactory.getValueKind(isEntryPoint ? kind : kind.getStackKind());
-            locations[0] = StackSlot.get(paramValueKind, currentStackOffset, !type.outgoing);
-            currentStackOffset += paramValueKind.getPlatformKind().getSizeInBytes();
+        /*
+         * Inject a pseudo-argument for rax Its value (which is the number for xmm registers
+         * containing an argument) will be injected in
+         * SubstrateAMD64NodeLIRBuilder.visitInvokeArguments. This information can be useful for
+         * functions taking a variable number of arguments (varargs).
+         */
+        if (type.nativeABI()) {
+            kinds = Arrays.copyOf(kinds, kinds.length + 1);
+            locations = Arrays.copyOf(locations, locations.length + 1);
+            kinds[kinds.length - 1] = JavaKind.Int;
+            locations[locations.length - 1] = AMD64.rax.asValue(LIRKind.value(AMD64Kind.DWORD));
+            if (type.customABI()) {
+                var extendsParametersAssignment = Arrays.copyOf(type.fixedParameterAssignment, type.fixedParameterAssignment.length + 1);
+                extendsParametersAssignment[extendsParametersAssignment.length - 1] = AssignedLocation.forRegister(rax);
+                type = SubstrateCallingConventionType.makeCustom(
+                                type.outgoing,
+                                extendsParametersAssignment,
+                                type.returnSaving);
+            }
         }
 
         JavaKind returnKind = returnType == null ? JavaKind.Void : ObjectLayout.getCallSignatureKind(isEntryPoint, (ResolvedJavaType) returnType, metaAccess, target);
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/AssignedLocation.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/AssignedLocation.java
index bb08caf1e4e1..93ff169af6ff 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/AssignedLocation.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/AssignedLocation.java
@@ -32,7 +32,9 @@
  * Represent a register or a stack offset.
  */
 public final class AssignedLocation {
+
     private static final int NONE = -1;
+    private static final AssignedLocation PLACEHOLDER = new AssignedLocation();
 
     private static boolean isValidOffset(int i) {
         return i >= 0 || i == NONE;
@@ -42,20 +44,29 @@ private void checkClassInvariant() {
         if (!isValidOffset(this.stackOffset)) {
             throw new IllegalStateException("Stack offset cannot be < 0 (and not NONE).");
         }
-        if ((this.register == null) == (this.stackOffset == NONE)) {
-            throw new IllegalStateException("Must assign to either a register or stack (but not both).");
+        if (assignsToStack() == assignsToRegister()) {
+            throw new IllegalStateException("Cannot assign to both register and stack.");
         }
     }
 
     private final Register register;
     private final int stackOffset;
 
+    private AssignedLocation() {
+        this.register = null;
+        this.stackOffset = NONE;
+    }
+
     private AssignedLocation(Register register, int stackOffset) {
         this.register = register;
         this.stackOffset = stackOffset;
         checkClassInvariant();
     }
 
+    public static AssignedLocation placeholder() {
+        return PLACEHOLDER;
+    }
+
     public static AssignedLocation forRegister(Register register) {
         return new AssignedLocation(register, NONE);
     }
@@ -67,6 +78,10 @@ public static AssignedLocation forStack(int offset) {
         return new AssignedLocation(null, offset);
     }
 
+    public boolean isPlaceholder() {
+        return !assignsToRegister() && !assignsToStack();
+    }
+
     public boolean assignsToRegister() {
         return register != null;
     }
@@ -77,14 +92,14 @@ public boolean assignsToStack() {
 
     public Register register() {
         if (register == null) {
-            throw new IllegalStateException("Cannot get register index of a stack location.");
+            throw new IllegalStateException("Not a register assignment.");
         }
         return register;
     }
 
     public int stackOffset() {
         if (stackOffset == NONE) {
-            throw new IllegalStateException("Cannot get stack offset of a register location.");
+            throw new IllegalStateException("Not a stack assignment.");
         }
         return stackOffset;
     }
@@ -93,9 +108,10 @@ public int stackOffset() {
     public String toString() {
         if (assignsToRegister()) {
             return "r-" + register;
-        } else {
-            assert assignsToStack();
+        } else if (assignsToStack()) {
             return "s-" + stackOffset;
+        } else {
+            return "p";
         }
     }
 
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/CustomCallingConventionMethod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/CustomCallingConventionMethod.java
new file mode 100644
index 000000000000..ed321f1dd9c4
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/CustomCallingConventionMethod.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.graal.code;
+
+/**
+ * See {@link SubstrateCallingConventionType}.
+ */
+public interface CustomCallingConventionMethod {
+    SubstrateCallingConventionType getCallingConvention();
+}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionKind.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionKind.java
index e8954cac61ac..e0e0927052b4 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionKind.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionKind.java
@@ -37,9 +37,24 @@ public enum SubstrateCallingConventionKind {
     /**
      * A call between Java and native code, which must use the platform ABI.
      */
-    Native;
+    Native,
+    /**
+     * A call between Java and native code, which uses a method-specific ABI. This can be used to
+     * pass values which are not easily representable, e.g. a structure which is split across
+     * multiple registers.
+     *
+     * Methods using this calling convention should implement {@link CustomCallingConventionMethod}.
+     */
+    Custom;
 
     public SubstrateCallingConventionType toType(boolean outgoing) {
+        if (isCustom()) {
+            throw new IllegalArgumentException("Custom calling conventions cannot be created using toType.");
+        }
         return (outgoing ? SubstrateCallingConventionType.outgoingTypes : SubstrateCallingConventionType.incomingTypes).get(this);
     }
+
+    public boolean isCustom() {
+        return this == Custom;
+    }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionType.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionType.java
index ab48afd112c6..2b660520f4ee 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionType.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/SubstrateCallingConventionType.java
@@ -50,8 +50,12 @@ public final class SubstrateCallingConventionType implements CallingConvention.T
         outgoingTypes = new EnumMap<>(SubstrateCallingConventionKind.class);
         incomingTypes = new EnumMap<>(SubstrateCallingConventionKind.class);
         for (SubstrateCallingConventionKind kind : SubstrateCallingConventionKind.values()) {
-            outgoingTypes.put(kind, new SubstrateCallingConventionType(kind, true));
-            incomingTypes.put(kind, new SubstrateCallingConventionType(kind, false));
+            if (kind.isCustom()) {
+                // Custom conventions cannot be enumerated this way
+                continue;
+            }
+            outgoingTypes.put(kind, new SubstrateCallingConventionType(kind, true, null, null));
+            incomingTypes.put(kind, new SubstrateCallingConventionType(kind, false, null, null));
         }
     }
 
@@ -62,37 +66,34 @@ private SubstrateCallingConventionType(SubstrateCallingConventionKind kind, bool
         this.returnSaving = returnSaving;
     }
 
-    private SubstrateCallingConventionType(SubstrateCallingConventionKind kind, boolean outgoing) {
-        this(kind, outgoing, null, null);
-    }
-
     /**
-     * Allows to manually assign which location (i.e. which register or stack location) to use for
-     * each argument.
+     * Create a calling convention with custom parameter/return assignment. The return value might
+     * get buffered, see {@link SubstrateCallingConventionType#usesReturnBuffer}.
+     *
+     * Methods using this calling convention should implement {@link CustomCallingConventionMethod}.
      */
-    public SubstrateCallingConventionType withParametersAssigned(AssignedLocation[] fixedRegisters) {
-        assert nativeABI();
-        return new SubstrateCallingConventionType(this.kind, this.outgoing, fixedRegisters, returnSaving);
+    public static SubstrateCallingConventionType makeCustom(boolean outgoing, AssignedLocation[] parameters, AssignedLocation[] returns) {
+        return new SubstrateCallingConventionType(SubstrateCallingConventionKind.Custom, outgoing, Objects.requireNonNull(parameters), Objects.requireNonNull(returns));
     }
 
-    /**
-     * Allows to retrieve the return of a function. When said return is more than one word long, we
-     * have no way of representing it as a value. Thus, this value will instead be stored from the
-     * registers containing it into a buffer provided (as a pointer) as a prefix argument to the
-     * function.
-     * 

- * Note that, even if used in conjunction with - * {@link SubstrateCallingConventionType#withParametersAssigned}, the location of the extra - * argument (i.e. the pointer to the return buffer) should not be assigned to a location, as - * this will be handled by the backend. - */ - public SubstrateCallingConventionType withReturnSaving(AssignedLocation[] newReturnSaving) { - assert nativeABI(); - return new SubstrateCallingConventionType(this.kind, this.outgoing, this.fixedParameterAssignment, newReturnSaving); + public boolean nativeABI() { + return kind == SubstrateCallingConventionKind.Native || customABI(); } - public boolean nativeABI() { - return kind == SubstrateCallingConventionKind.Native; + public boolean customABI() { + return kind == SubstrateCallingConventionKind.Custom; + } + + /** + * In the case of an outgoing call with multiple returned values, or if the return is more than + * one quadword long, there is no way to represent them in Java and the returns need special + * treatment. This is done using an extra prefix argument which is interpreted as a pointer to + * buffer where the values will be stored. + * + * This is currently only allowed in custom conventions. + */ + public boolean usesReturnBuffer() { + return outgoing && returnSaving != null && returnSaving.length >= 2; } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SharedRuntimeMethod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SharedRuntimeMethod.java index 9ddb3d2d49e1..e22b9918523e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SharedRuntimeMethod.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/meta/SharedRuntimeMethod.java @@ -24,7 +24,10 @@ */ package com.oracle.svm.core.graal.meta; +import static com.oracle.svm.core.util.VMError.intentionallyUnimplemented; + import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; import com.oracle.svm.core.meta.SharedMethod; /** @@ -38,6 +41,11 @@ default SharedRuntimeMethod getOriginal() { return this; } + @Override + default SubstrateCallingConventionType getCustomCallingConventionType() { + throw intentionallyUnimplemented(); // ExcludeFromJacocoGeneratedReport + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getDeoptOffsetInImage(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java index f13abcd839f3..854e765c6550 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java @@ -27,6 +27,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.graal.code.SubstrateCallingConventionKind; +import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -51,6 +52,8 @@ public interface SharedMethod extends ResolvedJavaMethod { SubstrateCallingConventionKind getCallingConventionKind(); + SubstrateCallingConventionType getCustomCallingConventionType(); + boolean hasCalleeSavedRegisters(); SharedMethod[] getImplementations(); diff --git a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/DowncallStub.java b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/DowncallStub.java index 3c64951d969a..3fdee5dd38f7 100644 --- a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/DowncallStub.java +++ b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/DowncallStub.java @@ -24,10 +24,8 @@ */ package com.oracle.svm.hosted.foreign; -import java.lang.invoke.MethodType; import java.util.List; -import org.graalvm.collections.Pair; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.java.FrameStateBuilder; @@ -45,13 +43,12 @@ import com.oracle.svm.core.foreign.LinkToNativeSupportImpl; import com.oracle.svm.core.foreign.NativeEntryPointInfo; import com.oracle.svm.core.foreign.Target_jdk_internal_foreign_abi_NativeEntryPoint; -import com.oracle.svm.core.graal.code.SubstrateCallingConventionKind; +import com.oracle.svm.core.graal.code.AssignedLocation; import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; import com.oracle.svm.core.graal.snippets.CFunctionSnippets; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.hosted.code.NonBytecodeMethod; import com.oracle.svm.hosted.code.SimpleSignature; -import com.oracle.svm.hosted.phases.HostedGraphKit; import com.oracle.svm.util.ReflectionUtil; import jdk.vm.ci.meta.JavaKind; @@ -67,22 +64,20 @@ * handles (or specialized classes); *

  • Unbox the arguments (the arguments are in an array of Objects, due to funneling through * {@link LinkToNativeSupportImpl#linkToNative}) --- done by - * {@link DowncallStub#adaptArguments};
  • + * {@link ForeignGraphKit#unboxArguments}; *
  • Further adapt arguments as to satisfy SubstrateVM's backends --- done by - * {@link DowncallStub#adaptArguments}, see {@link AbiUtils.Adaptation}
  • - *
  • Transform HotSpot's memory moves into ones for SubstrateVM --- done by - * {@link AbiUtils#toMemoryAssignment}
  • + * {@link AbiUtils.adapt} *
  • Perform a C-function call:
  • *
      *
    • Setup the frame anchor and capture call state --- Implemented in - * {@link CFunctionSnippets#prologueSnippet}, {@link CFunctionSnippets#epilogueSnippet} and + * {@link CFunctionSnippets#prologueSnippet}, {@link CFunctionSnippets#captureSnippet}, + * {@link CFunctionSnippets#epilogueSnippet} and * {@link ForeignFunctionsRuntime#captureCallState};
    • - *
    • Setup registers, stack and return buffer according to the aforementioned assignments --- - * Implemented in the backend, e.g. + *
    • Setup registers, stack and return buffer --- Implemented in the backend, e.g. * {@link com.oracle.svm.core.graal.amd64.SubstrateAMD64RegisterConfig#getCallingConvention} and * {@link com.oracle.svm.core.graal.amd64.SubstrateAMD64Backend.SubstrateAMD64NodeLIRBuilder#emitInvoke}
    • *
    - *
  • If needed, box the return --- done by {@link DowncallStub#adaptReturnValue}.
  • + *
  • If needed, box the return --- done by {@link ForeignGraphKit#boxAndReturn}.
  • * * Call state capture is done in the call epilogue to prevent the runtime environment from modifying * the call state, which could happen if a safepoint was inserted between the downcall and the @@ -112,23 +107,19 @@ public static Signature createSignature(MetaAccessProvider metaAccess) { */ @Override public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method, HostedProviders providers, Purpose purpose) { - HostedGraphKit kit = new HostedGraphKit(debug, providers, method, purpose); + ForeignGraphKit kit = new ForeignGraphKit(debug, providers, method, purpose); FrameStateBuilder state = kit.getFrameState(); boolean deoptimizationTarget = MultiMethod.isDeoptTarget(method); List arguments = kit.loadArguments(getSignature().toParameterTypes(null)); - AbiUtils.Adaptation[] adaptations = AbiUtils.singleton().adaptArguments(nep); assert arguments.size() == 1; - var argumentsAndNep = adaptArguments(kit, nep.linkMethodType(), arguments.get(0), adaptations); + var argumentsAndNep = kit.unpackArgumentsAndExtractNEP(arguments.get(0), nep.methodType()); arguments = argumentsAndNep.getLeft(); ValueNode runtimeNep = argumentsAndNep.getRight(); - MethodType callType = adaptMethodType(nep, adaptations); - assert callType.parameterCount() == arguments.size(); + AbiUtils.Adapter.AdaptationResult adapted = AbiUtils.singleton().adapt(kit.unboxArguments(arguments, this.nep.methodType()), this.nep); - int callAddressIndex = nep.callAddressIndex(); - ValueNode callAddress = arguments.remove(callAddressIndex); - callType = callType.dropParameterTypes(callAddressIndex, callAddressIndex + 1); + ValueNode callAddress = adapted.getArgument(AbiUtils.Adapter.Extracted.CallTarget); ForeignCallDescriptor captureFunction = null; ValueNode captureMask = null; @@ -136,30 +127,19 @@ public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method, if (nep.capturesCallState()) { captureFunction = ForeignFunctionsRuntime.CAPTURE_CALL_STATE; captureMask = kit.createLoadField(runtimeNep, kit.getMetaAccess().lookupJavaField(ReflectionUtil.lookupField(Target_jdk_internal_foreign_abi_NativeEntryPoint.class, "captureMask"))); - - assert nep.captureAddressIndex() > nep.callAddressIndex(); - /* - * We already removed 1 element from the list, so we must offset the index by one as - * well - */ - int captureAddressIndex = nep.captureAddressIndex() - 1; - captureAddress = arguments.remove(captureAddressIndex); - callType = callType.dropParameterTypes(captureAddressIndex, captureAddressIndex + 1); + captureAddress = adapted.getArgument(AbiUtils.Adapter.Extracted.CaptureBufferAddress); } - assert callType.parameterCount() == arguments.size(); state.clearLocals(); - SubstrateCallingConventionType cc = SubstrateCallingConventionKind.Native.toType(true) - .withParametersAssigned(nep.parametersAssignment()) - /* Assignment might be null, in which case this is a no-op */ - .withReturnSaving(nep.returnsAssignment()); + SubstrateCallingConventionType cc = SubstrateCallingConventionType.makeCustom(true, adapted.parametersAssignment().toArray(new AssignedLocation[0]), + adapted.returnsAssignment().toArray(new AssignedLocation[0])); CFunction.Transition transition = nep.skipsTransition() ? CFunction.Transition.NO_TRANSITION : CFunction.Transition.TO_NATIVE; ValueNode returnValue = kit.createCFunctionCallWithCapture( callAddress, - arguments, - SimpleSignature.fromMethodType(callType, kit.getMetaAccess()), + adapted.arguments(), + SimpleSignature.fromMethodType(adapted.callType(), kit.getMetaAccess()), VMThreads.StatusSupport.getNewThreadStatus(transition), deoptimizationTarget, cc, @@ -167,61 +147,8 @@ public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method, captureMask, captureAddress); - returnValue = adaptReturnValue(kit, nep.linkMethodType(), returnValue); - - kit.createReturn(returnValue, JavaKind.Object); + kit.boxAndReturn(returnValue, nep.methodType()); return kit.finalizeGraph(); } - - private static ValueNode adaptArgument(HostedGraphKit kit, ValueNode argument, Class type, AbiUtils.Adaptation adaptation) { - argument = kit.createUnboxing(argument, JavaKind.fromJavaClass(type)); - if (adaptation != null) { - argument = adaptation.apply(argument); - } - return argument; - } - - private static MethodType adaptMethodType(NativeEntryPointInfo nep, AbiUtils.Adaptation[] adaptations) { - MethodType mt = nep.linkMethodType(); - - Class[] parameters = new Class[mt.parameterCount()]; - for (int i = 0; i < mt.parameterCount(); ++i) { - Class parameterType = mt.parameterType(i); - assert parameterType.isPrimitive() : parameterType; - - AbiUtils.Adaptation adaptation = adaptations[i]; - if (adaptation != null) { - parameterType = adaptation.apply(parameterType); - } - - parameters[i] = parameterType; - } - - return MethodType.methodType(mt.returnType(), parameters); - } - - private static Pair, ValueNode> adaptArguments(HostedGraphKit kit, MethodType signature, ValueNode argumentsArray, AbiUtils.Adaptation[] adaptations) { - var args = kit.loadArrayElements(argumentsArray, JavaKind.Object, signature.parameterCount() + 1); - assert adaptations.length == signature.parameterCount() : adaptations.length + " " + signature.parameterCount(); - assert args.size() == signature.parameterCount() + 1 : args.size() + " " + (signature.parameterCount() + 1); - // We have to drop the NEP, which is the last argument - ValueNode nep = args.remove(args.size() - 1); - for (int i = 0; i < args.size(); ++i) { - args.set(i, adaptArgument(kit, args.get(i), signature.parameterType(i), adaptations[i])); - } - return Pair.create(args, nep); - } - - private static ValueNode adaptReturnValue(HostedGraphKit kit, MethodType signature, ValueNode invokeValue) { - ValueNode returnValue = invokeValue; - JavaKind returnKind = JavaKind.fromJavaClass(signature.returnType()); - if (returnKind.equals(JavaKind.Void)) { - return kit.createObject(null); - } - - var boxed = kit.getMetaAccess().lookupJavaType(returnKind.toBoxedJavaClass()); - returnValue = kit.createBoxing(returnValue, returnKind, boxed); - return returnValue; - } } diff --git a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsConfigurationParser.java b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsConfigurationParser.java index 4a8a1d341807..227c6ce5b55f 100644 --- a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsConfigurationParser.java +++ b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsConfigurationParser.java @@ -29,6 +29,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.function.BiConsumer; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -53,22 +54,24 @@ public ForeignFunctionsConfigurationParser(RuntimeForeignAccessSupport access) { @Override public void parseAndRegister(Object json, URI origin) { var topLevel = asMap(json, "first level of document must be a map"); - checkAttributes(topLevel, "foreign methods categories", List.of("downcalls")); - for (Object downcall : asList(topLevel.get("downcalls"), "downcalls must be an array of method signatures")) { - parseDowncall(downcall); + checkAttributes(topLevel, "foreign methods categories", List.of(), List.of("downcalls")); + + var downcalls = asList(topLevel.get("downcalls", List.of()), "downcalls must be an array of method signatures"); + for (Object downcall : downcalls) { + parseAndRegisterForeignCall(downcall, (descriptor, options) -> accessSupport.registerForDowncall(ConfigurationCondition.alwaysTrue(), descriptor, options)); } } - private void parseDowncall(Object downcall) { - var map = asMap(downcall, "a downcall must be a map"); - checkAttributes(map, "downcall", List.of("descriptor"), List.of("options")); - var descriptor = parseDowncallSignatures(map.get("descriptor")); + private void parseAndRegisterForeignCall(Object call, BiConsumer register) { + var map = asMap(call, "a foreign call must be a map"); + checkAttributes(map, "foreign call", List.of("descriptor"), List.of("options")); + var descriptor = parseDescriptor(map.get("descriptor")); var options = parseOptions(map.get("options", null)); - accessSupport.registerForDowncall(ConfigurationCondition.alwaysTrue(), descriptor, options.toArray()); + register.accept(descriptor, options.toArray()); } - private FunctionDescriptor parseDowncallSignatures(Object signature) { - String input = asString(signature, "downcalls's elements must be function descriptors"); + private FunctionDescriptor parseDescriptor(Object signature) { + String input = asString(signature, "a function descriptor must be a string"); return FunctionDescriptorParser.parse(input); } diff --git a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsFeature.java b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsFeature.java index a9f6ae1b2411..55ec6162ce62 100644 --- a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignFunctionsFeature.java @@ -26,13 +26,13 @@ import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; import org.graalvm.collections.Pair; import org.graalvm.compiler.api.replacements.Fold; @@ -40,6 +40,7 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.type.CIntPointer; import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeForeignAccessSupport; @@ -56,7 +57,6 @@ import com.oracle.svm.core.foreign.AbiUtils; import com.oracle.svm.core.foreign.ForeignFunctionsRuntime; import com.oracle.svm.core.foreign.LinkToNativeSupportImpl; -import com.oracle.svm.core.foreign.NativeEntryPointInfo; import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider; import com.oracle.svm.core.meta.MethodPointer; import com.oracle.svm.core.util.UserError; @@ -97,8 +97,9 @@ private static boolean isPreviewEnabled() { private boolean sealed = false; private final RuntimeForeignAccessSupportImpl accessSupport = new RuntimeForeignAccessSupportImpl(); - private final Set> registeredDowncalls = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final Map>> downcallsMapping = new HashMap<>(); + + private final Set> registeredDowncalls = ConcurrentHashMap.newKeySet(); + private int downcallCount = 0; @Fold public static ForeignFunctionsFeature singleton() { @@ -148,31 +149,38 @@ public void duringSetup(DuringSetupAccess a) { ConfigurationParser parser = new ForeignFunctionsConfigurationParser(accessSupport); ConfigurationParserUtils.parseAndRegisterConfigurations(parser, access.getImageClassLoader(), "panama foreign", ConfigurationFiles.Options.ForeignConfigurationFiles, ConfigurationFiles.Options.ForeignResources, ConfigurationFile.FOREIGN.getFileName()); + } + private void createDowncallStubs(FeatureImpl.BeforeAnalysisAccessImpl access) { + this.downcallCount = createStubs( + registeredDowncalls, + access, + AbiUtils.singleton()::makeNativeEntrypoint, + n -> new DowncallStub(n, access.getMetaAccess().getWrapped()), + ForeignFunctionsRuntime.singleton()::addDowncallStubPointer); } - private int createDowncallStubs(FeatureImpl.BeforeAnalysisAccessImpl access) { - Map created = new HashMap<>(); - for (Pair fdOptionsPair : registeredDowncalls) { - NativeEntryPointInfo nepi = AbiUtils.singleton().makeEntrypoint(fdOptionsPair.getLeft(), fdOptionsPair.getRight()); + private int createStubs( + Set> source, + FeatureImpl.BeforeAnalysisAccessImpl access, + BiFunction stubGenerator, + Function wrapper, + BiConsumer register) { + + Map created = new HashMap<>(); + + for (Pair fdOptionsPair : source) { + S nepi = stubGenerator.apply(fdOptionsPair.getLeft(), fdOptionsPair.getRight()); if (!created.containsKey(nepi)) { - ResolvedJavaMethod stub = new DowncallStub(nepi, access.getMetaAccess().getWrapped()); + ResolvedJavaMethod stub = wrapper.apply(nepi); AnalysisMethod analysisStub = access.getUniverse().lookup(stub); - access.getBigBang().addRootMethod(analysisStub, false, "Foreign downcall stub, registered in " + ForeignFunctionsFeature.class); + access.getBigBang().addRootMethod(analysisStub, false, "Foreign stub, registered in " + ForeignFunctionsFeature.class); created.put(nepi, analysisStub); - ForeignFunctionsRuntime.singleton().addStubPointer( - nepi, - new MethodPointer(analysisStub)); - downcallsMapping.put(analysisStub.getName(), new ArrayList<>()); + register.accept(nepi, new MethodPointer(analysisStub)); } - - String stubName = created.get(nepi).getName(); - downcallsMapping.get(stubName).add(fdOptionsPair); - - assert created.size() == downcallsMapping.size(); } - registeredDowncalls.clear(); + source.clear(); return created.size(); } @@ -197,8 +205,8 @@ public void beforeAnalysis(BeforeAnalysisAccess a) { access.registerAsRoot(ReflectionUtil.lookupMethod(ForeignFunctionsRuntime.class, "captureCallState", int.class, CIntPointer.class), false, "Runtime support, registered in " + ForeignFunctionsFeature.class); - int downcallStubsCount = createDowncallStubs(access); - ProgressReporter.singleton().setForeignFunctionsInfo(downcallStubsCount); + createDowncallStubs(access); + ProgressReporter.singleton().setForeignFunctionsInfo(getCreatedDowncallStubsCount()); } @Override @@ -208,13 +216,8 @@ public void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) { /* Testing interface */ - public int getCreatedStubsCount() { - assert sealed; - return downcallsMapping.size(); - } - - public Map>> getCreatedStubsMap() { + public int getCreatedDowncallStubsCount() { assert sealed; - return downcallsMapping; + return this.downcallCount; } } diff --git a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignGraphKit.java b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignGraphKit.java new file mode 100644 index 000000000000..ad788a879fbe --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/ForeignGraphKit.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.foreign; + +import java.lang.invoke.MethodType; +import java.util.List; + +import org.graalvm.collections.Pair; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.nodes.ValueNode; + +import com.oracle.graal.pointsto.infrastructure.GraphProvider; +import com.oracle.graal.pointsto.meta.HostedProviders; +import com.oracle.svm.hosted.phases.HostedGraphKit; + +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaMethod; + +class ForeignGraphKit extends HostedGraphKit { + ForeignGraphKit(DebugContext debug, HostedProviders providers, ResolvedJavaMethod method, GraphProvider.Purpose purpose) { + super(debug, providers, method, purpose); + } + + Pair, ValueNode> unpackArgumentsAndExtractNEP(ValueNode argumentsArray, MethodType methodType) { + List args = loadArrayElements(argumentsArray, JavaKind.Object, methodType.parameterCount() + 1); + ValueNode nep = args.remove(args.size() - 1); + return Pair.create(args, nep); + } + + List unboxArguments(List args, MethodType methodType) { + assert args.size() == methodType.parameterCount() : args.size() + " " + methodType.parameterCount(); + for (int i = 0; i < args.size(); ++i) { + ValueNode argument = args.get(i); + argument = createUnboxing(argument, JavaKind.fromJavaClass(methodType.parameterType(i))); + args.set(i, argument); + } + return args; + } + + public ValueNode boxAndReturn(ValueNode returnValue, MethodType methodType) { + JavaKind returnKind = JavaKind.fromJavaClass(methodType.returnType()); + if (returnKind.equals(JavaKind.Void)) { + return createReturn(createObject(null), JavaKind.Object); + } + + var boxed = getMetaAccess().lookupJavaType(returnKind.toBoxedJavaClass()); + return createReturn(createBoxing(returnValue, returnKind, boxed), JavaKind.Object); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/StronglyTypedRuntimeForeignAccessSupport.java b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/StronglyTypedRuntimeForeignAccessSupport.java index 62879ea3c4ce..e1fb4905edef 100644 --- a/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/StronglyTypedRuntimeForeignAccessSupport.java +++ b/substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/StronglyTypedRuntimeForeignAccessSupport.java @@ -34,22 +34,41 @@ * Convenience interface until {@link RuntimeForeignAccessSupport} can be strongly typed. */ public interface StronglyTypedRuntimeForeignAccessSupport extends RuntimeForeignAccessSupport { - @Override - default void registerForDowncall(ConfigurationCondition condition, Object descO, Object... optionsO) { - if (!(descO instanceof FunctionDescriptor)) { - throw new IllegalArgumentException("Desc must be an instance of " + FunctionDescriptor.class); + private static FunctionDescriptor castDesc(Object descO) { + if (descO instanceof FunctionDescriptor desc) { + return desc; } - FunctionDescriptor desc = (FunctionDescriptor) descO; - Linker.Option[] options = new Linker.Option[optionsO.length]; + throw new IllegalArgumentException("Desc must be an instance of " + FunctionDescriptor.class + "; was " + descO.getClass()); + } + private static Linker.Option[] castOptions(Object... optionsO) { + Linker.Option[] options = new Linker.Option[optionsO.length]; for (int i = 0; i < optionsO.length; ++i) { if (!(optionsO[i] instanceof Linker.Option)) { throw new IllegalArgumentException("Option at position " + i + " must be an instance of " + Linker.Option.class); } options[i] = (Linker.Option) optionsO[i]; } + return options; + } - registerForDowncall(condition, desc, options); + @FunctionalInterface + interface Recorder { + void apply(ConfigurationCondition condition, FunctionDescriptor desc, Linker.Option... options); + } + + static StronglyTypedRuntimeForeignAccessSupport make(Recorder forDowncalls) { + return new StronglyTypedRuntimeForeignAccessSupport() { + @Override + public void registerForDowncall(ConfigurationCondition condition, FunctionDescriptor desc, Linker.Option... options) { + forDowncalls.apply(condition, desc, options); + } + }; + } + + @Override + default void registerForDowncall(ConfigurationCondition condition, Object descO, Object... optionsO) { + registerForDowncall(condition, castDesc(descO), castOptions(optionsO)); } void registerForDowncall(ConfigurationCondition condition, FunctionDescriptor desc, Linker.Option... options); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java index a72aeab4cdbc..b4a9d122c56b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java @@ -62,9 +62,11 @@ import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.graal.code.CustomCallingConventionMethod; import com.oracle.svm.core.graal.code.ExplicitCallingConvention; import com.oracle.svm.core.graal.code.StubCallingConvention; import com.oracle.svm.core.graal.code.SubstrateCallingConventionKind; +import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; import com.oracle.svm.core.graal.phases.SubstrateSafepointInsertionPhase; import com.oracle.svm.core.meta.MethodPointer; import com.oracle.svm.core.meta.SharedMethod; @@ -351,6 +353,13 @@ public SubstrateCallingConventionKind getCallingConventionKind() { return ExplicitCallingConvention.Util.getCallingConventionKind(wrapped, isEntryPoint()); } + @Override + public SubstrateCallingConventionType getCustomCallingConventionType() { + VMError.guarantee(getCallingConventionKind().isCustom(), "%s does not have a custom calling convention.", name); + VMError.guarantee(wrapped.getWrapped() instanceof CustomCallingConventionMethod, "%s has a custom calling convention but doesn't implement %s", name, CustomCallingConventionMethod.class); + return ((CustomCallingConventionMethod) wrapped.getWrapped()).getCallingConvention(); + } + @Override public boolean hasCalleeSavedRegisters() { return StubCallingConvention.Utils.hasStubCallingConvention(this);