[GR-48374] Better error messages when dependencies are missing, use NativeModules=false for multiprocessing workers.

Author: steve-s

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java

@@ -55,6 +55,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.graalvm.collections.EconomicMap;
@@ -526,7 +527,8 @@ public int getId() {
      * This represents whether the current process has already loaded an instance of the native CAPI
      * extensions - this can only be loaded once per process.
      */
-    private static boolean nativeCAPILoaded = false;
+    private static AtomicBoolean nativeCAPILoaded = new AtomicBoolean();
+    private static AtomicBoolean warnedSecondContexWithNativeCAPI = new AtomicBoolean();
 
     @TruffleBoundary
     public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, TruffleString name, TruffleString path) throws IOException, ImportException, ApiInitException {
@@ -540,11 +542,27 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context,
             TruffleFile capiFile = homePath.resolve(libName);
             try {
                 SourceBuilder capiSrcBuilder;
-                boolean useNative = PythonOptions.NativeModules.getValue(env.getOptions()) && !nativeCAPILoaded;
-                nativeCAPILoaded |= useNative;
+                final boolean useNative;
+                if (PythonOptions.NativeModules.getValue(env.getOptions())) {
+                    useNative = nativeCAPILoaded.compareAndSet(false, true);
+                    if (!useNative && warnedSecondContexWithNativeCAPI.compareAndSet(false, true)) {
+                        LOGGER.warning("GraalPy option 'NativeModules' is set to true, " +
+                                        "but only one context in the process can use native modules, " +
+                                        "second and other contexts fallback to NativeModules=false and " +
+                                        "will use LLVM bitcode execution via GraalVM LLVM.");
+                    }
+                } else {
+                    useNative = false;
+                }
                 if (useNative) {
+                    if (!env.getInternalLanguages().containsKey(J_NFI_LANGUAGE)) {
+                        throw PRaiseNode.raiseUncached(node, PythonBuiltinClassType.SystemError, ErrorMessages.NFI_NOT_AVAILABLE, "NativeModules", "true");
+                    }
                     capiSrcBuilder = Source.newBuilder(J_NFI_LANGUAGE, "load(RTLD_GLOBAL) \"" + capiFile.getAbsoluteFile().getPath() + "\"", "<libpython>");
                 } else {
+                    if (!env.getInternalLanguages().containsKey(J_LLVM_LANGUAGE)) {
+                        throw PRaiseNode.raiseUncached(node, PythonBuiltinClassType.SystemError, ErrorMessages.LLVM_NOT_AVAILABLE);
+                    }
                     capiSrcBuilder = Source.newBuilder(J_LLVM_LANGUAGE, capiFile);
                 }
                 if (!context.getLanguage().getEngineOption(PythonOptions.ExposeInternalSources)) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java

@@ -1485,6 +1485,13 @@ public abstract class ErrorMessages {
     public static final TruffleString HPY_CANNOT_SPECIFY_LEG_SLOTS_WO_SETTING_LEG = tsLiteral("cannot specify .legacy_slots without setting .builtin_shape=HPyType_BuiltinShape_Legacy");
     public static final TruffleString HPY_CANNOT_USE_CALL_WITH_LEGACY = tsLiteral("Cannot use HPy call protocol with legacy types that inherit the struct. " +
                     "Either set the basicsize to a non-zero value or use legacy slot 'Py_tp_call'.");
+    public static final TruffleString NFI_NOT_AVAILABLE = tsLiteral("GraalPy option '%s' is set to '%s, but the 'nfi' language, which is required for this feature, is not available. " +
+                    "If this is a GraalPy standalone distribution: this indicates internal error. If GraalPy was used as a Maven dependency: " +
+                    "are you missing a runtime dependency 'org.graalvm.truffle:truffle-nfi-libffi', which should be a dependency of 'org.graalvm.polyglot:python{-community}'?");
+
+    public static final TruffleString LLVM_NOT_AVAILABLE = tsLiteral("GraalPy option 'NativeModules' is set to false, but the 'llvm' language, which is required for this feature, is not available. " +
+                    "If this is a GraalPy standalone distribution: this indicates internal error. If GraalPy was used as a Maven dependency: are you missing a runtime dependency " +
+                    "'org.graalvm.polyglot:llvm{-community}'?");
     private static final String HPY_NON_DEFAULT_MESSAGE = "This is not allowed because custom HPy_mod_create slot cannot return a builtin module object and cannot make any use of any other " +
                     "data defined in the HPyModuleDef. Either do not define HPy_mod_create slot and let the runtime create a builtin module object from the provided HPyModuleDef, or do not " +
                     "define anything else but the HPy_mod_create slot.";

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIPosixSupport.java

@@ -76,7 +76,10 @@
 import org.graalvm.nativeimage.ImageInfo;
 
 import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.objects.exception.OSErrorEnum;
+import com.oracle.graal.python.nodes.ErrorMessages;
+import com.oracle.graal.python.nodes.PRaiseNode;
 import com.oracle.graal.python.runtime.PosixSupportLibrary.AcceptResult;
 import com.oracle.graal.python.runtime.PosixSupportLibrary.AddrInfoCursor;
 import com.oracle.graal.python.runtime.PosixSupportLibrary.AddrInfoCursorLibrary;
@@ -349,6 +352,12 @@ private static String getLibPath(PythonContext context) {
         private static void loadLibrary(NFIPosixSupport posix) {
             String path = getLibPath(posix.context);
             String backend = posix.nfiBackend.toJavaStringUncached();
+            Env env = posix.context.getEnv();
+
+            if (!env.getInternalLanguages().containsKey(J_NFI_LANGUAGE)) {
+                throw PRaiseNode.raiseUncached(null, PythonBuiltinClassType.SystemError, ErrorMessages.NFI_NOT_AVAILABLE, "PosixModuleBackend", "native");
+            }
+
             String withClause = backend.equals(J_NATIVE) ? "" : "with " + backend;
             String src = String.format("%sload (RTLD_LOCAL) \"%s\"", withClause, path);
             Source loadSrc = Source.newBuilder(J_NFI_LANGUAGE, src, "load:" + SUPPORTING_NATIVE_LIB_NAME).internal(true).build();
@@ -357,9 +366,13 @@ private static void loadLibrary(NFIPosixSupport posix) {
                 LOGGER.fine(String.format("Loading native library: %s", src));
             }
             try {
-                posix.nfiLibrary = posix.context.getEnv().parseInternal(loadSrc).call();
+                posix.nfiLibrary = env.parseInternal(loadSrc).call();
             } catch (Throwable e) {
-                throw CompilerDirectives.shouldNotReachHere("Unable to load native posix support library", e);
+                throw new UnsupportedOperationException(String.format("""
+                                Could not load posix support library from path '%s'. Troubleshooting:\s
+                                Check permissions of the file.
+                                Missing runtime Maven dependency 'org.graalvm.truffle:truffle-nfi-libffi' (should be a dependency of `org.graalvm.polyglot:python{-community}`)?""",
+                                path));
             }
         }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -1151,6 +1151,7 @@ public long spawnTruffleContext(int fd, int sentinel, int[] fdsToKeep) {
                         forceSharing(getOption(PythonOptions.ForceSharingForInnerContexts)).//
                         inheritAllAccess(true).//
                         initializeCreatorContext(true).//
+                        option("python.NativeModules", "false").//
                         // TODO always force java posix in spawned: test_multiprocessing_spawn fails
                         // with that. Gives "OSError: [Errno 9] Bad file number"
                         // option("python.PosixModuleBackend", "java").//