Use tokens (instead of trees) to highlight literals

Tokens position is less fragile. Specially when code does not parse

Author: allanrenucci

compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala

@@ -57,13 +57,17 @@ object SyntaxHighlighting {
         scanner.nextToken()
         val end = scanner.lastOffset
 
-        if (alphaKeywords.contains(token))
+        // Branch order is important. For example,
+        // `true` is at the same time a keyword and a literal
+        if (literalTokens.contains(token))
+          highlightRange(start, end, LiteralColor)
+        else if (alphaKeywords.contains(token))
           highlightRange(start, end, KeywordColor)
         else if (token == IDENTIFIER && name == nme.???)
           highlightRange(start, end, Console.RED_B)
       }
 
-      val treeHighlighter = new untpd.UntypedTreeTraverser {
+      object TreeHighlighter extends untpd.UntypedTreeTraverser {
         import untpd._
 
         def ignored(tree: NameTree) = {
@@ -72,6 +76,9 @@ object SyntaxHighlighting {
           name == nme.ERROR || name == nme.CONSTRUCTOR
         }
 
+        def highlight(trees: List[Tree])(implicit ctx: Context): Unit =
+          trees.foreach(traverse)
+
         def traverse(tree: Tree)(implicit ctx: Context): Unit = {
           tree match {
             case tree: NameTree if ignored(tree) =>
@@ -85,8 +92,6 @@ object SyntaxHighlighting {
               highlightPosition(tree.pos, TypeColor)
             case _: TypTree =>
               highlightPosition(tree.pos, TypeColor)
-            case _: Literal =>
-              highlightPosition(tree.pos, LiteralColor)
             case _ =>
           }
           traverseChildren(tree)
@@ -95,8 +100,7 @@ object SyntaxHighlighting {
 
       val parser = new Parser(source)
       val trees = parser.blockStatSeq()
-      for (tree <- trees)
-        treeHighlighter.traverse(tree)
+      TreeHighlighter.highlight(trees)
 
       val highlighted = new StringBuilder()
 

compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala

@@ -30,7 +30,7 @@ class SyntaxHighlightingTests extends DottyTest {
   @Test
   @Ignore("Comments are currently not supported")
   def comments = {
-    test("//a", "<C|//a>")
+    test("// a", "<C|// a>")
     test("/** a */", "<C|/** a */>")
     test("/* a */", "<C|/* a */>")
   }
@@ -50,7 +50,7 @@ class SyntaxHighlightingTests extends DottyTest {
     test("1.1", "<L|1.1>")
     test("1.1.toString", "<L|1.1>.toString")
     test("1L", "<L|1L>")
-    test("1Lx", "1Lx")
+    test("1Lx", "<L|1L>x")
     test("1f", "<L|1f>")
     test("1.1f", "<L|1.1f>")
     test("1.1fx", "1.1fx")
@@ -94,9 +94,9 @@ class SyntaxHighlightingTests extends DottyTest {
     test("var",                "<K|var>")
     test("var foo",            "<K|var> <V|foo>")
     test("var foo:",           "<K|var> <V|foo>:")
-    test("var foo: Int",       "<K|var> <V|foo>: <T|int>")
-    test("var foo: Int =",     "<K|var> <V|foo>: <T|int> =")
-    test("var foo: Int = 123", "<K|var> <V|foo>: <T|int> = <L|123>")
+    test("var foo: Int",       "<K|var> <V|foo>: <T|Int>")
+    test("var foo: Int =",     "<K|var> <V|foo>: <T|Int> =")
+    test("var foo: Int = 123", "<K|var> <V|foo>: <T|Int> = <L|123>")
 
     test("def",                          "<K|def>")
     test("def foo",                      "<K|def> <V|foo>")