[GR-42007] Basic windows build

PullRequest: graalpython/2499

Author: timfel

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "b7f2a8abd1005a8ef57418f3f20a81c3696f9cfe" }
+{ "overlay": "0382241d2334bc7a645e23bbeb33b5775bc7a7a5" }

graalpython/com.oracle.graal.python.cext/CEXT-WINDOWS-README.md

@@ -0,0 +1 @@
+C extensions on Windows not supported yet

graalpython/com.oracle.graal.python.cext/include/Python.h

@@ -75,12 +75,19 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#ifndef _MSC_VER
 #include <sys/dir.h>
 #include <dirent.h>
-#include <locale.h>
 #include <langinfo.h>
-#include <assert.h>
 #include <unistd.h>
+#else
+#define HAVE_COPYSIGN
+#define HAVE_ROUND
+#define HAVE_HYPOT
+#define NT_THREADS
+#endif
+#include <locale.h>
+#include <assert.h>
 #include <math.h>
 
 #include "pyport.h"

graalpython/com.oracle.graal.python.cext/setup.py

@@ -46,7 +46,7 @@
 
 import _sysconfig
 
-__dir__ = __file__.rpartition("/")[0]
+__dir__ = os.path.dirname(__file__)
 cflags_warnings = [ "-Wno-int-to-pointer-cast"
                   , "-Wno-int-conversion"
                   , "-Wno-void-pointer-to-int-cast"
@@ -60,13 +60,15 @@
 libposix_name = "libposix"
 
 MACOS = sys.platform == "darwin"
+WIN32 = sys.platform == "win32"
 verbosity = '--verbose' if sys.flags.verbose else '--quiet'
 darwin_native = MACOS and __graalpython__.platform_id == "native"
+win32_native = WIN32 and __graalpython__.platform_id == "native"
 relative_rpath = "@loader_path" if darwin_native else r"$ORIGIN"
 so_ext = get_config_var("EXT_SUFFIX")
 SOABI = get_config_var("SOABI")
 is_managed = 'managed' in SOABI
-lib_ext = 'dylib' if MACOS else 'so'
+lib_ext = 'dylib' if MACOS else ('pyd' if WIN32 else 'so')
 
 # configure logger
 logger = logging.getLogger(__name__)

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -70,7 +70,7 @@
             "genericpath",
             "ntpath",
             "posixpath",
-            # We must explicitly mark os.path as a frozen module
+            # We must explicitly mark os.path as a frozen module, it is also special cased below
             ("ntpath" if os.name == "nt" else "posixpath") + " : os.path",
             "os",
             "site",
@@ -279,7 +279,7 @@ def frozenid(self):
 
     @property
     def modname(self):
-        if self.pyfile.startswith(self.stdlib_path):
+        if os.path.normpath(self.pyfile).startswith(os.path.normpath(self.stdlib_path)):
             return self.id
         return None
 
@@ -369,7 +369,7 @@ def _iter_sources(modules):
 
 
 def _get_checksum(filename):
-    with open(filename, "rb") as infile:
+    with open(filename, "rb", encoding="utf-8") as infile:
         contents = infile.read()
     m = sha256()
     m.update(contents)
@@ -528,11 +528,13 @@ def lower_camel_case(str):
  */
 package com.oracle.graal.python.builtins.objects.module;
 
+import com.oracle.graal.python.builtins.PythonOS;
+
 public final class FrozenModules {"""
 
 
 def freeze_module(src):
-    with open(src.pyfile, "r") as src_file, open(src.binaryfile, "wb") as binary_file:
+    with open(src.pyfile, "r", encoding="utf-8") as src_file, open(src.binaryfile, "wb") as binary_file:
         code_obj = compile(src_file.read(), f"<frozen {src.id}>", "exec")
         marshal.dump(code_obj, binary_file)
 
@@ -561,6 +563,9 @@ def write_frozen_lookup(out_file, modules):
             out_file.write(
                 f'                return Map.{module.symbol}.asPackage({"true" if module.ispkg else "false"});\n'
             )
+        elif module.name == "os.path": # Special case for os.path
+            out_file.write(u'            case "os.path":\n')
+            out_file.write(u'                return PythonOS.getPythonOS() != PythonOS.PLATFORM_WIN32 ? Map.POSIXPATH : Map.NTPATH;\n')
         else:
             out_file.write(f'            case "{module.name}":\n')
             out_file.write(f"                return Map.{module.symbol};\n")
@@ -572,21 +577,21 @@ def write_frozen_lookup(out_file, modules):
 
 def write_frozen_module_file(file, modules):
     if os.path.exists(file):
-        with open(file, "r") as f:
+        with open(file, "r", encoding="utf-8") as f:
             content = f.read()
         stat_result = os.stat(file)
         atime, mtime = stat_result.st_atime, stat_result.st_mtime
     else:
         content = None
     os.makedirs(os.path.dirname(file), exist_ok=True)
-    with open(file, "w") as out_file:
+    with open(file, "w", encoding="utf-8") as out_file:
         out_file.write(FROZEN_MODULES_HEADER)
         out_file.write("\n\n")
         write_frozen_modules_map(out_file, modules)
         out_file.write("\n")
         write_frozen_lookup(out_file, modules)
         out_file.write("}\n")
-    with open(file, "r") as f:
+    with open(file, "r", encoding="utf-8") as f:
         new_content = f.read()
     if new_content == content:
         # set mtime to the old one, if we didn't change anything

graalpython/com.oracle.graal.python.jni/JNI-WINDOWS-README.md

@@ -0,0 +1 @@
+C extensions on Windows not supported yet

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -663,6 +663,11 @@ protected void launch(Builder contextBuilder) {
             }
         }
 
+        String osName = System.getProperty("os.name");
+        if (osName != null && osName.toLowerCase().contains("windows")) {
+            contextBuilder.option("python.PosixModuleBackend", "java");
+        }
+
         if (multiContext) {
             contextBuilder.engine(Engine.newBuilder().allowExperimentalOptions(true).options(enginePolyglotOptions).build());
         }
@@ -711,6 +716,10 @@ protected void launch(Builder contextBuilder) {
     }
 
     private static String toAbsolutePath(String executable) {
+        if (executable.contains(":")) {
+            // this is either already an absolute windows path, or not a single executable
+            return executable;
+        }
         return Paths.get(executable).toAbsolutePath().toString();
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -25,20 +25,24 @@
  */
 package com.oracle.graal.python;
 
+import static com.oracle.graal.python.builtins.PythonOS.PLATFORM_WIN32;
+import static com.oracle.graal.python.builtins.PythonOS.getPythonOS;
 import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION;
 import static com.oracle.graal.python.nodes.StringLiterals.T_PY_EXTENSION;
 import static com.oracle.graal.python.nodes.truffle.TruffleStringMigrationHelpers.isJavaString;
 import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING;
 import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
 
 import java.io.IOException;
+import java.nio.file.InvalidPathException;
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
 import java.util.logging.Level;
 
+import org.graalvm.nativeimage.ImageInfo;
 import org.graalvm.options.OptionDescriptors;
 import org.graalvm.options.OptionKey;
 import org.graalvm.options.OptionValues;
@@ -523,7 +527,7 @@ private CallTarget parseForBytecodeInterpreter(ParsingRequest request) {
     private Source tryLoadSource(PythonContext context, CodeUnit code, boolean internal, String path) {
         try {
             return Source.newBuilder(PythonLanguage.ID, context.getEnv().getPublicTruffleFile(path)).name(code.name.toJavaStringUncached()).internal(internal).build();
-        } catch (IOException | SecurityException | UnsupportedOperationException e) {
+        } catch (IOException | SecurityException | UnsupportedOperationException | InvalidPathException e) {
             return null;
         }
     }
@@ -952,7 +956,7 @@ public static Source newSource(PythonContext ctxt, TruffleString tsrc, TruffleSt
                             sourceBuilder.content(src);
                         }
                     }
-                } catch (SecurityException | IOException e) {
+                } catch (SecurityException | IOException | InvalidPathException e) {
                     sourceBuilder = null;
                 }
             }
@@ -973,6 +977,11 @@ public static Source newSource(PythonContext ctxt, TruffleFile src, String name)
     }
 
     private static Source newSource(PythonContext context, SourceBuilder srcBuilder) throws IOException {
+        if (getPythonOS() == PLATFORM_WIN32 && ImageInfo.inImageBuildtimeCode()) {
+            // canonicalization on windows means something else than on linux and causes issues
+            // with paths
+            srcBuilder.canonicalizePath(false);
+        }
         if (shouldMarkSourceInternal(context)) {
             srcBuilder.internal(true);
         }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -60,6 +60,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.oracle.graal.python.builtins.modules.WinregModuleBuiltins;
 import com.oracle.graal.python.builtins.objects.itertools.PairwiseBuiltins;
 import org.graalvm.nativeimage.ImageInfo;
 
@@ -93,6 +94,7 @@
 import com.oracle.graal.python.builtins.modules.MarshalModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.MathModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.MultiprocessingModuleBuiltins;
+import com.oracle.graal.python.builtins.modules.NtModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.OperatorModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.PolyglotModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins;
@@ -439,12 +441,14 @@ private static TruffleString[] initializeCoreFiles() {
 
     private static void filterBuiltins(List<PythonBuiltins> builtins) {
         PythonOS currentOs = PythonOS.getPythonOS();
+        List<PythonBuiltins> toRemove = new ArrayList<>();
         for (PythonBuiltins builtin : builtins) {
             CoreFunctions annotation = builtin.getClass().getAnnotation(CoreFunctions.class);
             if (annotation.os() != PythonOS.PLATFORM_ANY && annotation.os() != currentOs) {
-                builtins.remove(builtin);
+                toRemove.add(builtin);
             }
         }
+        builtins.removeAll(toRemove);
     }
 
     private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed) {
@@ -511,6 +515,8 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
                         new PropertyBuiltins(),
                         new BaseExceptionBuiltins(),
                         new PosixModuleBuiltins(),
+                        new NtModuleBuiltins(),
+                        new WinregModuleBuiltins(),
                         new CryptModuleBuiltins(),
                         new ScandirIteratorBuiltins(),
                         new DirEntryBuiltins(),
@@ -871,7 +877,10 @@ private void initializeJavaCore() {
     }
 
     private void initializeImportlib() {
-        PythonModule bootstrap = ImpModuleBuiltins.importFrozenModuleObject(this, T__FROZEN_IMPORTLIB, false);
+        PythonModule bootstrap = null;
+        if (!ImageInfo.inImageBuildtimeCode()) {
+            bootstrap = ImpModuleBuiltins.importFrozenModuleObject(this, T__FROZEN_IMPORTLIB, false);
+        }
         PythonModule bootstrapExternal;
 
         PyObjectCallMethodObjArgs callNode = PyObjectCallMethodObjArgs.getUncached();
@@ -945,8 +954,7 @@ public final void postInitialize() {
              * access is never allowed during native image build time.
              */
             if (ImageInfo.inImageCode() && !getContext().isNativeAccessAllowed()) {
-                builtinModules.remove(BuiltinNames.T_BZ2);
-                sysModules.delItem(BuiltinNames.T_BZ2);
+                removeBuiltinModule(BuiltinNames.T_BZ2);
             }
 
             globalScopeObject = PythonMapScope.createTopScope(getContext());
@@ -955,6 +963,16 @@ public final void postInitialize() {
         }
     }
 
+    @TruffleBoundary
+    public final void removeBuiltinModule(TruffleString name) {
+        assert !initialized : "can only remove builtin modules before initialization is finished";
+        builtinModules.remove(name);
+        if (sysModules != null) {
+            // may already be published
+            sysModules.delItem(name);
+        }
+    }
+
     @TruffleBoundary
     public final PythonModule lookupBuiltinModule(TruffleString name) {
         return builtinModules.get(name);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltins.java

@@ -163,7 +163,6 @@ public void initialize(Python3Core core) {
             final boolean declaresExplicitSelf;
             PythonBuiltinClassType constructsClass = builtin.constructsClass();
             if ((annotation.defineModule().length() > 0 || annotation.extendsModule().length() > 0) && constructsClass == PythonBuiltinClassType.nil) {
-                assert annotation.defineModule().isEmpty() && !annotation.extendsModule().isEmpty() || !annotation.defineModule().isEmpty() && annotation.extendsModule().isEmpty();
                 assert !builtin.isGetter();
                 assert !builtin.isSetter();
                 assert annotation.extendClasses().length == 0;