[GR-38632] [GR-31236] [GR-17176] Fix SL parser and add gate checks for generating parsers and for TODOs.

PullRequest: graal/11728

Author: boris-spas

truffle/external_repos/simplelanguage/generate_parser.sh

@@ -40,5 +40,5 @@
 # SOFTWARE.
 #
 
-curl -O https://www.antlr.org/download/antlr-4.7.1-complete.jar
-java -cp antlr-4.7.1-complete.jar org.antlr.v4.Tool -package com.oracle.truffle.sl.parser -no-listener language/src/main/java/com/oracle/truffle/sl/parser/SimpleLanguage.g4
+curl -O https://www.antlr.org/download/antlr-4.9.2-complete.jar
+java -cp antlr-4.9.2-complete.jar org.antlr.v4.Tool -package com.oracle.truffle.sl.parser -no-listener language/src/main/java/com/oracle/truffle/sl/parser/SimpleLanguage.g4

truffle/mx.truffle/mx_truffle.py

@@ -256,6 +256,8 @@ def _truffle_gate_runner(args, tasks):
         with Task('Truffle DSL max state bit tests', tasks) as t:
             if t:
                 _truffle_gate_state_bitwidth_tests()
+    with Task('Validate parsers', tasks) as t:
+        if t: validate_parsers()
 
 # The Truffle DSL specialization state bit width computation is complicated and
 # rarely used as the default maximum bit width of 32 is rarely exceeded. Therefore
@@ -680,6 +682,30 @@ def create_parser(grammar_project, grammar_package, grammar_name, copyright_temp
         with open(filename, 'w') as content_file:
             content_file.write(content)
 
+def validate_parsers(args=None, out=None):
+    validate_parser("com.oracle.truffle.sl", "com/oracle/truffle/sl/parser/SimpleLanguage.g4", create_sl_parser)
+    validate_parser("com.oracle.truffle.dsl.processor", "com/oracle/truffle/dsl/processor/expression/Expression.g4", create_dsl_parser)
+
+def validate_parser(grammar_project, grammar_path, create_command, args=None, out=None):
+    def read_file(path):
+        with open(path, "r") as f:
+            return f.readlines()
+    parser_path = grammar_path.replace(".g4", "Parser.java")
+    lexer_path = grammar_path.replace(".g4", "Lexer.java")
+    parser = mx.project(grammar_project).source_dirs()[0] + "/" + parser_path
+    lexer = mx.project(grammar_project).source_dirs()[0] + "/" + lexer_path
+    parser_before = read_file(parser)
+    lexer_before = read_file(lexer)
+    create_command([], out)
+    parser_after = read_file(parser)
+    lexer_after = read_file(lexer)
+    if (parser_before != parser_after or lexer_before != lexer_after):
+        with open(parser, "w") as f:
+            f.writelines(parser_before)
+        with open(lexer, "w") as f:
+            f.writelines(lexer_before)
+        mx.abort("Parser generated from " + grammar_path + " does not match content of " + parser_path + " or " + lexer_path + "." +
+            " Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx.")
 
 class LibffiBuilderProject(mx.AbstractNativeProject, mx_native.NativeDependency):  # pylint: disable=too-many-ancestors
     """Project for building libffi from source.

truffle/mx.truffle/suite.py

@@ -336,7 +336,7 @@
         "mx:JUNIT",
         "sdk:POLYGLOT_TCK"
       ],
-      "checkstyle" : "com.oracle.truffle.sl",
+      "checkstyle" : "com.oracle.truffle.api",
       "javaCompliance" : "11+",
       "workingSets" : "SimpleLanguage,Test",
       "jacoco" : "exclude",
@@ -872,7 +872,7 @@
         "jdk.unsupported", # GR-36880
       ],
       "javaCompliance" : "11+",
-      "checkstyleVersion" : "8.36.1",
+      "checkstyle" : "com.oracle.truffle.api",
       "annotationProcessors" : ["TRUFFLE_DSL_PROCESSOR"],
       "workingSets" : "Truffle,SimpleLanguage",
     },
@@ -883,7 +883,7 @@
       "dependencies" : [
         "sdk:GRAAL_SDK",
       ],
-      "checkstyle" : "com.oracle.truffle.sl",
+      "checkstyle" : "com.oracle.truffle.api",
       "javaCompliance" : "11+",
       "workingSets" : "Truffle,SimpleLanguage",
     },
@@ -899,7 +899,7 @@
       "requires" : [
         "java.logging",
       ],
-      "checkstyle" : "com.oracle.truffle.sl",
+      "checkstyle" : "com.oracle.truffle.api",
       "javaCompliance" : "11+",
       "workingSets" : "Truffle,SimpleLanguage,Test",
       "annotationProcessors" : ["TRUFFLE_DSL_PROCESSOR", "mx:JMH_1_21"],
@@ -927,9 +927,9 @@
         "com.oracle.truffle.st"
       ],
       "javaCompliance" : "11+",
-      "checkstyleVersion" : "8.36.1",
+      "checkstyle" : "com.oracle.truffle.api",
       "annotationProcessors" : ["TRUFFLE_DSL_PROCESSOR"],
-      "workingSets" : "Truffle,SimpleLanguage",
+      "workingSets" : "Truffle",
       "testProject" : True,
       "jacoco" : "exclude",
     },

truffle/src/com.oracle.truffle.api.debug/src/com/oracle/truffle/api/debug/Debugger.java

@@ -349,18 +349,18 @@ public static Debugger find(TruffleLanguage.Env env) {
     static final class AccessorDebug extends Accessor {
 
         /*
-         * TODO get rid of this access and replace it with an API in {@link TruffleInstrument.Env}.
-         * I don't think {@link CallTarget} is the right return type here as we want to make it
-         * embeddable into the current AST.
+         * TODO GR-38632 get rid of this access and replace it with an API in {@link
+         * TruffleInstrument.Env}. I don't think {@link CallTarget} is the right return type here as
+         * we want to make it embeddable into the current AST.
          */
         protected CallTarget parse(Source code, Node context, String... argumentNames) {
             RootNode rootNode = context.getRootNode();
             return languageSupport().parse(engineSupport().getEnvForInstrument(rootNode.getLanguageInfo()), code, context, argumentNames);
         }
 
         /*
-         * TODO I initially moved this to TruffleInstrument.Env but decided against as a new API for
-         * inline parsing might replace it.
+         * TODO GR-38632 I initially moved this to TruffleInstrument.Env but decided against as a
+         * new API for inline parsing might replace it.
          */
         protected Object evalInContext(Source source, Node node, MaterializedFrame frame) {
             return languageSupport().evalInContext(source, node, frame);

truffle/src/com.oracle.truffle.api.dsl.test/src/com/oracle/truffle/api/dsl/test/ImplicitCastTest.java

@@ -135,7 +135,7 @@ abstract static class ImplicitCastTypedExecuteNode extends Node {
 
         public abstract Object execute(int value);
 
-        // TODO: this should not be an error
+        // TODO:GR-38632 this should not be an error
         @ExpectError("Method signature (boolean) does not match to the expected signature: %")
         @Specialization
         public boolean op1(boolean value) {
@@ -192,7 +192,7 @@ public void testImplicitCast1() {
 
     @TypeSystemReference(ImplicitCast0Types.class)
     @NodeChildren({@NodeChild(value = "operand0", type = ImplicitCast2Node.class), @NodeChild(value = "operand1", type = ImplicitCast2Node.class, executeWith = "operand0")})
-    // TODO temporary workaround
+    // TODO GR-38632 temporary workaround
     abstract static class ImplicitCast2Node extends ValueNode {
 
         @Specialization

truffle/src/com.oracle.truffle.api.dsl.test/src/com/oracle/truffle/api/dsl/test/SharedCachedTest.java

@@ -59,7 +59,7 @@
 @SuppressWarnings("unused")
 public class SharedCachedTest {
 
-    // TODO how to share primitive caches? maybe through a specialization class?
+    // TODO GR-38632 how to share primitive caches? maybe through a specialization class?
     abstract static class UnboundCachedPrimitiveNode extends Node {
 
         abstract Object execute(Object arg);

truffle/src/com.oracle.truffle.api.dsl.test/src/com/oracle/truffle/api/dsl/test/SpecializationFallthroughTest.java

@@ -123,7 +123,7 @@ public void afterExecution(TestRootNode<? extends ValueNode> node, int index, Ob
                         });
     }
 
-    /* TODO assert falltrough do1 before do2 */
+    /* TODO GR-38632 assert falltrough do1 before do2 */
     @NodeChildren({@NodeChild("a")})
     static class FallthroughTest1 extends ValueNode {
 

truffle/src/com.oracle.truffle.api.library.test/src/com/oracle/truffle/api/library/test/ReflectionTest.java

@@ -141,7 +141,7 @@ public void testReflectionOnObject() throws Exception {
 
         Message primitive = Message.resolve(ReflectionTestLibrary.class, "primitive");
 
-        // TODO more tests necessary
+        // TODO GR-38632 more tests necessary
 
         Assert.assertEquals(11, (int) reflection.send(object, primitive, 11));
 

truffle/src/com.oracle.truffle.api.library.test/src/com/oracle/truffle/api/library/test/examples/ReflectiveExportExample.java

@@ -82,7 +82,7 @@ Object send(Message message, Object[] args) throws Exception {
             if (message == MESSAGE) {
                 return "reflectiveExport";
             } else {
-                // TODO how to invoke the super implementation?
+                // TODO GR-38632 how to invoke the super implementation?
                 throw new AbstractMethodError();
             }
         }
@@ -99,7 +99,7 @@ public void runExample() throws Exception {
         try {
             assertEquals("message1", library.message1(value));
         } catch (AbstractMethodError e) {
-            // TODO currently throws abstract method error but should return default value
+            // TODO GR-38632 currently throws abstract method error but should return default value
             // "message1".
         }
     }

truffle/src/com.oracle.truffle.api.test/src/com/oracle/truffle/api/test/host/TestMemberAccess.java

@@ -422,7 +422,7 @@ public void testOverloadedConstructor2() throws InteropException {
         testObj = (TruffleObject) INTEROP.instantiate(testClass, (short) 42);
         assertEquals(int.class.getName(), INTEROP.readMember(testObj, "ctor"));
         testObj = (TruffleObject) INTEROP.instantiate(testClass, 4.2f);
-        // TODO prioritize conversion from double to float over double to int
+        // TODO GR-38632 prioritize conversion from double to float over double to int
         // assertEquals(float.class.getName(), INTEROP.readMember(testObj, "ctor"));
     }