[GR-68952] Backport to 25.0: Fix wrong line endings on windows when downloading patches

PullRequest: graalpython/3989

Author: ilyass-elmazidi

graalpython/com.oracle.graal.python.pegparser.generator/main_parser_gen.py

@@ -41,6 +41,7 @@
 import os
 import sys
 import tokenize
+import io
 
 from pegen.build import generate_token_definitions
 from pegen.grammar_parser import GeneratedParser as GrammarParser
@@ -71,34 +72,29 @@ def main():
 
     # Determine which line separator to use
     if os.path.exists(args.output_file):
-        stat_result = os.stat(args.output_file)
-        atime, mtime = stat_result.st_atime, stat_result.st_mtime
-        with open(args.output_file, "r", encoding="utf-8", newline=os.linesep) as f:
-            content = f.read()
-        if os.linesep != "\n":
-            unix_content = content.replace(os.linesep, "\n")
-            if unix_content == content:
-                # Windows file has Unix line endings
-                linesep = "\n"
-                content = unix_content
+        with open(args.output_file, "rb") as f:
+            raw_old_content = f.read()
+            if b'\r\n' in raw_old_content:
+                linesep = '\r\n'
             else:
-                linesep = os.linesep
-        else:
-            linesep = "\n"
+                linesep = '\n'
+        with open(args.output_file, "r", encoding="utf-8") as f:
+            old_content = f.read()
     else:
-        content = None
+        old_content = None
         linesep = os.linesep
 
-    with open(args.output_file, "w", encoding="utf-8", newline=linesep) as file:
-        gen = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, file, debug=args.debug)
-        gen.generate(os.path.basename(args.grammar_file))
+    out = io.StringIO()
+    class_name = os.path.splitext(os.path.basename(args.output_file))[0]
+    gen = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, out, class_name, debug=args.debug)
+    gen.generate(os.path.basename(args.grammar_file))
 
-    with open(args.output_file, "r", encoding="utf-8", newline=linesep) as file:
-        new_content = file.read()
-
-    if content == new_content:
+    if out.getvalue() != old_content:
+        print(f"Writing new {args.output_file}")
+        with open(args.output_file, "w", encoding="utf-8", newline=linesep) as f:
+            f.write(out.getvalue())
+    else:
         print(f"{args.output_file} not modified")
-        os.utime(args.output_file, (atime, mtime))
 
 
 if __name__ == '__main__':

graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py

@@ -874,10 +874,12 @@ def __init__(
         exact_tokens: Dict[str, int],
         non_exact_tokens: Set[str],
         file: Optional[IO[Text]],
+        class_name: str,
         debug: bool = True,
         skip_actions: bool = False,
     ):
         super().__init__(grammar, set(tokens.values()), file)
+        self.class_name = class_name
         self.typingvisitor = TypingVisitor(self) # Java type hack
         self.callmakervisitor = JavaCallMakerVisitor(self, exact_tokens, non_exact_tokens, self.print)
         self.lookahead_functions: Dict[str, FunctionCall] = {}
@@ -930,10 +932,9 @@ def generate(self, filename: str) -> None:
         self.print(f"// Generated from {filename} by pegen")
         self.print("package com.oracle.graal.python.pegparser;")
         self.print(IMPORTS)
-        className = os.path.splitext(os.path.basename(self.file.name))[0]
         self.print('@SuppressWarnings({"all", "cast"})')
         self.print('@SuppressFBWarnings')
-        self.print("public final class %s extends AbstractParser {" % className)
+        self.print("public final class %s extends AbstractParser {" % self.class_name)
         # Java needs a few fields declarations. Also, we're now in a class
         self.level += 1
         self.print()
@@ -945,12 +946,12 @@ def generate(self, filename: str) -> None:
             self.print(f"private static final int {rulename.upper()}_ID = {i};{comment}")
         self.print()
         # Java needs a constructor
-        self.print("public %s(String source, SourceRange sourceRange, ParserCallbacks parserCb, InputType startRule, EnumSet<Flags> flags, int featureVersion) {" % className)
+        self.print("public %s(String source, SourceRange sourceRange, ParserCallbacks parserCb, InputType startRule, EnumSet<Flags> flags, int featureVersion) {" % self.class_name)
         with self.indent():
             self.print("super(source, sourceRange, parserCb, startRule, flags, featureVersion);")
         self.print("}")
         self.print()
-        self.print("public %s(String source, ParserCallbacks parserCb, InputType startRule, EnumSet<Flags> flags, int featureVersion) {" % className)
+        self.print("public %s(String source, ParserCallbacks parserCb, InputType startRule, EnumSet<Flags> flags, int featureVersion) {" % self.class_name)
         with self.indent():
             self.print("super(source, null, parserCb, startRule, flags, featureVersion);")
         self.print("}")

graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/interop/JavaInteropTest.java

@@ -48,7 +48,6 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -107,6 +106,14 @@ public void tearDown() {
             context.getEngine().close();
         }
 
+        private String getOutString() {
+            return out.toString(StandardCharsets.UTF_8).replaceAll("\r\n", "\n");
+        }
+
+        private String getErrString() {
+            return err.toString(StandardCharsets.UTF_8).replaceAll("\r\n", "\n");
+        }
+
         @Test
         public void evalFailsOnError() {
             boolean didFail = false;
@@ -217,8 +224,8 @@ public void testHostException() {
                 Assert.assertTrue(e.getMessage(), e.getMessage().contains("divide by zero"));
             }
 
-            String outString = out.toString(StandardCharsets.UTF_8);
-            String errString = err.toString(StandardCharsets.UTF_8);
+            String outString = getOutString();
+            String errString = getErrString();
             Assert.assertTrue(outString, outString.isEmpty());
             Assert.assertTrue(errString, errString.isEmpty());
         }
@@ -257,7 +264,7 @@ def is_none(x):
         }
 
         @Test
-        public void testPassingFloats() throws UnsupportedEncodingException {
+        public void testPassingFloats() {
             String source = "import polyglot\n" +
                             "@polyglot.export_value\n" +
                             "def foo(x, y):\n" +
@@ -266,11 +273,11 @@ public void testPassingFloats() throws UnsupportedEncodingException {
             context.eval(script);
             Value main = context.getPolyglotBindings().getMember("foo");
             main.execute((float) 1.0, (float) 2.0);
-            assertEquals("2.0\n", out.toString("UTF-8"));
+            assertEquals("2.0\n", getOutString());
         }
 
         @Test
-        public void testPassingBigIntegers() throws UnsupportedEncodingException {
+        public void testPassingBigIntegers() {
             String source = "import polyglot\n" +
                             "@polyglot.export_value\n" +
                             "def foo(x, y):\n" +
@@ -279,20 +286,20 @@ public void testPassingBigIntegers() throws UnsupportedEncodingException {
             context.eval(script);
             Value main = context.getPolyglotBindings().getMember("foo");
             main.execute(BigInteger.valueOf(Long.MAX_VALUE).multiply(BigInteger.TWO), BigInteger.valueOf(7));
-            assertEquals("129127208515966861298\n", out.toString("UTF-8"));
+            assertEquals("129127208515966861298\n", getOutString());
             out.reset();
             main.execute(Long.MAX_VALUE, 14);
-            assertEquals("129127208515966861298\n", out.toString("UTF-8"));
+            assertEquals("129127208515966861298\n", getOutString());
             out.reset();
             main.execute(6, 7);
-            assertEquals("42\n", out.toString("UTF-8"));
+            assertEquals("42\n", getOutString());
             out.reset();
             main.execute(true, true);
-            assertEquals("1\n", out.toString("UTF-8"));
+            assertEquals("1\n", getOutString());
         }
 
         @Test
-        public void testBigIntegersAdd() throws UnsupportedEncodingException {
+        public void testBigIntegersAdd() {
             String source = "import polyglot\n" +
                             "@polyglot.export_value\n" +
                             "def foo(x, y):\n" +
@@ -301,20 +308,20 @@ public void testBigIntegersAdd() throws UnsupportedEncodingException {
             context.eval(script);
             Value main = context.getPolyglotBindings().getMember("foo");
             main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(7));
-            assertEquals("60847228810955011271841753858055\n", out.toString("UTF-8"));
+            assertEquals("60847228810955011271841753858055\n", getOutString());
             out.reset();
             main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(24).shiftLeft(101));
-            assertEquals("121694457621910022543683507716096\n", out.toString("UTF-8"));
+            assertEquals("121694457621910022543683507716096\n", getOutString());
             out.reset();
             main.execute(6, 7);
-            assertEquals("13\n", out.toString("UTF-8"));
+            assertEquals("13\n", getOutString());
             out.reset();
             main.execute(true, true);
-            assertEquals("2\n", out.toString("UTF-8"));
+            assertEquals("2\n", getOutString());
         }
 
         @Test
-        public void testBigIntegersEg() throws UnsupportedEncodingException {
+        public void testBigIntegersEg() {
             String source = "import polyglot\n" +
                             "@polyglot.export_value\n" +
                             "def foo(x, y):\n" +
@@ -323,26 +330,26 @@ public void testBigIntegersEg() throws UnsupportedEncodingException {
             context.eval(script);
             Value main = context.getPolyglotBindings().getMember("foo");
             main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(7));
-            assertEquals("False\n", out.toString("UTF-8"));
+            assertEquals("False\n", getOutString());
             out.reset();
             main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(24).shiftLeft(101));
-            assertEquals("True\n", out.toString("UTF-8"));
+            assertEquals("True\n", getOutString());
             out.reset();
             main.execute(6, 7);
-            assertEquals("False\n", out.toString("UTF-8"));
+            assertEquals("False\n", getOutString());
             out.reset();
             main.execute(6, 6);
-            assertEquals("True\n", out.toString("UTF-8"));
+            assertEquals("True\n", getOutString());
             out.reset();
             main.execute(true, BigInteger.ONE);
-            assertEquals("True\n", out.toString("UTF-8"));
+            assertEquals("True\n", getOutString());
             out.reset();
             main.execute(true, BigInteger.ZERO);
-            assertEquals("False\n", out.toString("UTF-8"));
+            assertEquals("False\n", getOutString());
         }
 
         @Test
-        public void testAsFunction() throws UnsupportedEncodingException {
+        public void testAsFunction() {
             String source = "import polyglot\n" +
                             "@polyglot.export_value\n" +
                             "def foo():\n" +
@@ -351,11 +358,11 @@ public void testAsFunction() throws UnsupportedEncodingException {
             context.eval(script);
             Value main = context.getPolyglotBindings().getMember("foo");
             main.execute();
-            assertEquals("Called\n", out.toString("UTF-8"));
+            assertEquals("Called\n", getOutString());
         }
 
         @Test
-        public void testAsFunctionVarArgs() throws UnsupportedEncodingException {
+        public void testAsFunctionVarArgs() {
             String source = "import polyglot\n" +
                             "@polyglot.export_value\n" +
                             "def foo(a, b):\n" +
@@ -364,28 +371,28 @@ public void testAsFunctionVarArgs() throws UnsupportedEncodingException {
             context.eval(script);
             Value main = context.getPolyglotBindings().getMember("foo");
             main.execute("Hello", "World");
-            assertEquals("Hello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\n", getOutString());
         }
 
         @Test
-        public void mainFunctionsAreImplicitlyImporteable() throws UnsupportedEncodingException {
+        public void mainFunctionsAreImplicitlyImporteable() {
             String source = "def foo(a, b):\n" +
                             "    print(a, b)\n\n";
             Source script = Source.create("python", source);
             context.eval(script);
             Value main = context.getBindings("python").getMember("foo");
             main.execute("Hello", "World");
-            assertEquals("Hello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\n", getOutString());
         }
 
         @Test
-        public void builtinFunctionsAreImporteable() throws UnsupportedEncodingException {
+        public void builtinFunctionsAreImporteable() {
             String source = "pass";
             Source script = Source.create("python", source);
             context.eval(script);
             Value main = context.getBindings("python").getMember("__builtins__").getMember("print");
             main.execute("Hello", "World");
-            assertEquals("Hello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\n", getOutString());
         }
 
         @Test
@@ -394,22 +401,22 @@ public void enumeratingMainBindingsWorks() throws Exception {
         }
 
         @Test
-        public void testMultipleInvocationsAreInSameScope() throws UnsupportedEncodingException {
+        public void testMultipleInvocationsAreInSameScope() {
             String source = "def foo(a, b):\n" +
                             "    print(a, b)\n" +
                             "foo";
             Source script = Source.create("python", source);
             Value foo = context.eval(script);
             foo.execute("Hello", "World");
-            assertEquals("Hello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\n", getOutString());
 
             source = "def bar(a, b):\n" +
                             "    foo(a, b)\n" +
                             "bar";
             script = Source.create("python", source);
             Value bar = context.eval(script);
             bar.execute("Hello", "World");
-            assertEquals("Hello World\nHello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\nHello World\n", getOutString());
 
             source = "invalid syntax";
             script = Source.create("python", source);
@@ -418,9 +425,9 @@ public void testMultipleInvocationsAreInSameScope() throws UnsupportedEncodingEx
             } catch (Throwable t) {
             }
             bar.execute("Hello", "World");
-            assertEquals("Hello World\nHello World\nHello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\nHello World\nHello World\n", getOutString());
             foo.execute("Hello", "World");
-            assertEquals("Hello World\nHello World\nHello World\nHello World\n", out.toString("UTF-8"));
+            assertEquals("Hello World\nHello World\nHello World\nHello World\n", getOutString());
         }
 
         @Test
@@ -568,7 +575,7 @@ public void accessJavaObjectFields() throws IOException {
                             "5.0 <class 'float'>\n" +
                             "6.0 <class 'float'>\n" +
                             "True <class 'bool'>\n" +
-                            "c <class 'polyglot.ForeignString'>\n", out.toString("UTF-8"));
+                            "c <class 'polyglot.ForeignString'>\n", getOutString());
         }
 
         @Test
@@ -595,7 +602,7 @@ public void accessJavaObjectGetters() throws IOException {
                             "5.0 <class 'float'>\n" +
                             "6.0 <class 'float'>\n" +
                             "True <class 'bool'>\n" +
-                            "c <class 'polyglot.ForeignString'>\n", out.toString("UTF-8"));
+                            "c <class 'polyglot.ForeignString'>\n", getOutString());
         }
 
         @Test
@@ -612,7 +619,7 @@ public void javaStringsAndPythonStrings() throws IOException {
             assertEquals("" +
                             "<class 'polyglot.ForeignDict'>\n" +
                             "True\n" +
-                            "True\n", out.toString("UTF-8"));
+                            "True\n", getOutString());
         }
 
         @Test

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -899,7 +899,7 @@ def parse_config(cls, config_path: Path):
             if config_tags_dir := settings.get('tags_dir'):
                 tags_dir = (config_path.parent / config_tags_dir).resolve()
             # Temporary hack for Bytecode DSL development in master branch:
-            if IS_GRAALPY and __graalpython__.is_bytecode_dsl_interpreter and tags_dir:
+            if IS_GRAALPY and getattr(__graalpython__, 'is_bytecode_dsl_interpreter', False) and tags_dir:
                 new_tags_dir = (config_path.parent / (config_tags_dir + '_bytecode_dsl')).resolve()
                 if new_tags_dir.exists():
                     tags_dir = new_tags_dir

graalpython/com.oracle.graal.python.test/src/tests/conftest.toml

@@ -22,7 +22,6 @@ partial_splits_individual_tests = true
 exclude_on = ['win32']
 selector = [
     "test_multiprocessing_graalpy.py", # import _winapi
-    "test_patched_pip.py",
     "test_pathlib.py",
     "test_posix.py", # import posix
     "test_pyio.py", # pyio imports msvcrt