[SPARK-24345][SQL] Improve ParseError stop location when offending symbol is a token (apache-spark-on-k8s#375)

In the case where the offending symbol is a CommonToken, this PR increases the accuracy of the start and stop origin by leveraging the start and stop index information from CommonToken.

Author: Ruben Fiszel

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala

@@ -192,8 +192,17 @@ case object ParseErrorListener extends BaseErrorListener {
       charPositionInLine: Int,
       msg: String,
       e: RecognitionException): Unit = {
-    val position = Origin(Some(line), Some(charPositionInLine))
-    throw new ParseException(None, msg, position, position)
+    val (start, stop) = offendingSymbol match {
+      case token: CommonToken =>
+        val start = Origin(Some(line), Some(token.getCharPositionInLine))
+        val length = token.getStopIndex - token.getStartIndex + 1
+        val stop = Origin(Some(line), Some(token.getCharPositionInLine + length))
+        (start, stop)
+      case _ =>
+        val start = Origin(Some(line), Some(charPositionInLine))
+        (start, start)
+    }
+    throw new ParseException(None, msg, start, stop)
   }
 }
 

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala

@@ -22,14 +22,21 @@ import org.apache.spark.SparkFunSuite
  * Test various parser errors.
  */
 class ErrorParserSuite extends SparkFunSuite {
-  def intercept(sql: String, line: Int, startPosition: Int, messages: String*): Unit = {
+  def intercept(
+      sql: String,
+      line: Int,
+      startPosition: Int,
+      stopPosition: Int,
+      messages: String*): Unit = {
     val e = intercept[ParseException](CatalystSqlParser.parsePlan(sql))
 
     // Check position.
     assert(e.line.isDefined)
     assert(e.line.get === line)
     assert(e.startPosition.isDefined)
     assert(e.startPosition.get === startPosition)
+    assert(e.stop.startPosition.isDefined)
+    assert(e.stop.startPosition.get === stopPosition)
 
     // Check messages.
     val error = e.getMessage
@@ -39,26 +46,27 @@ class ErrorParserSuite extends SparkFunSuite {
   }
 
   test("no viable input") {
-    intercept("select ((r + 1) ", 1, 16, "no viable alternative at input", "----------------^^^")
+    intercept("select ((r + 1) ", 1, 16, 16,
+      "no viable alternative at input", "----------------^^^")
   }
 
   test("extraneous input") {
-    intercept("select 1 1", 1, 9, "extraneous input '1' expecting", "---------^^^")
-    intercept("select *\nfrom r as q t", 2, 12, "extraneous input", "------------^^^")
+    intercept("select 1 1", 1, 9, 10, "extraneous input '1' expecting", "---------^^^")
+    intercept("select *\nfrom r as q t", 2, 12, 13, "extraneous input", "------------^^^")
   }
 
   test("mismatched input") {
-    intercept("select * from r order by q from t", 1, 27,
+    intercept("select * from r order by q from t", 1, 27, 31,
       "mismatched input",
       "---------------------------^^^")
-    intercept("select *\nfrom r\norder by q\nfrom t", 4, 0, "mismatched input", "^^^")
+    intercept("select *\nfrom r\norder by q\nfrom t", 4, 0, 4, "mismatched input", "^^^")
   }
 
   test("semantic errors") {
-    intercept("select *\nfrom r\norder by q\ncluster by q", 3, 0,
+    intercept("select *\nfrom r\norder by q\ncluster by q", 3, 0, 11,
       "Combination of ORDER BY/SORT BY/DISTRIBUTE BY/CLUSTER BY is not supported",
       "^^^")
-    intercept("select * from r except all select * from t", 1, 0,
+    intercept("select * from r except all select * from t", 1, 0, 41,
       "EXCEPT ALL is not supported",
       "^^^")
   }