[SPARK-31526][SQL][TESTS] Add a new test suite for ExpressionInfo #28308
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark.sql.expressions | ||
|
|
||
| import scala.collection.parallel.immutable.ParVector | ||
|
|
||
| import org.apache.spark.SparkFunSuite | ||
| import org.apache.spark.sql.catalyst.FunctionIdentifier | ||
| import org.apache.spark.sql.catalyst.expressions.ExpressionInfo | ||
| import org.apache.spark.sql.execution.HiveResult.hiveResultString | ||
| import org.apache.spark.sql.internal.SQLConf | ||
| import org.apache.spark.sql.test.SharedSparkSession | ||
|
|
||
| class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession { | ||
|
|
||
| test("Replace _FUNC_ in ExpressionInfo") { | ||
|
|
||
| val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("upper")) | ||
| assert(info.getName === "upper") | ||
| assert(info.getClassName === "org.apache.spark.sql.catalyst.expressions.Upper") | ||
| assert(info.getUsage === "upper(str) - Returns `str` with all characters changed to uppercase.") | ||
| assert(info.getExamples.contains("> SELECT upper('SparkSql');")) | ||
| assert(info.getSince === "1.0.1") | ||
| assert(info.getNote === "") | ||
| assert(info.getExtended.contains("> SELECT upper('SparkSql');")) | ||
| } | ||
|
|
||
| test("group info in ExpressionInfo") { | ||
|
|
||
| val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("sum")) | ||
| assert(info.getGroup === "agg_funcs") | ||
|
|
||
| Seq("agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs", "window_funcs") | ||
| .foreach { groupName => | ||
| val info = new ExpressionInfo( | ||
| "testClass", null, "testName", null, "", "", "", groupName, "", "") | ||
| assert(info.getGroup === groupName) | ||
| } | ||
|
|
||
| val errMsg = intercept[IllegalArgumentException] { | ||
| val invalidGroupName = "invalid_group_funcs" | ||
| new ExpressionInfo("testClass", null, "testName", null, "", "", "", invalidGroupName, "", "") | ||
| }.getMessage | ||
| assert(errMsg.contains("'group' is malformed in the expression [testName].")) | ||
| } | ||
|
|
||
| test("error handling in ExpressionInfo") { | ||
|
Member
Author
There was a problem hiding this comment. To improve test coverage, a new test added here. |
||
| val errMsg1 = intercept[IllegalArgumentException] { | ||
| val invalidNote = " invalid note" | ||
| new ExpressionInfo("testClass", null, "testName", null, "", "", invalidNote, "", "", "") | ||
| }.getMessage | ||
| assert(errMsg1.contains("'note' is malformed in the expression [testName].")) | ||
|
|
||
| val errMsg2 = intercept[IllegalArgumentException] { | ||
| val invalidSince = "-3.0.0" | ||
| new ExpressionInfo("testClass", null, "testName", null, "", "", "", "", invalidSince, "") | ||
| }.getMessage | ||
| assert(errMsg2.contains("'since' is malformed in the expression [testName].")) | ||
|
|
||
| val errMsg3 = intercept[IllegalArgumentException] { | ||
| val invalidDeprecated = " invalid deprecated" | ||
| new ExpressionInfo("testClass", null, "testName", null, "", "", "", "", "", invalidDeprecated) | ||
| }.getMessage | ||
| assert(errMsg3.contains("'deprecated' is malformed in the expression [testName].")) | ||
| } | ||
|
|
||
| test("using _FUNC_ instead of function names in examples") { | ||
|
|
||
| val exampleRe = "(>.*;)".r | ||
| val setStmtRe = "(?i)^(>\\s+set\\s+).+".r | ||
| val ignoreSet = Set( | ||
| // Examples for CaseWhen show simpler syntax: | ||
| // `CASE WHEN ... THEN ... WHEN ... THEN ... END` | ||
| "org.apache.spark.sql.catalyst.expressions.CaseWhen", | ||
| // _FUNC_ is replaced by `locate` but `locate(... IN ...)` is not supported | ||
| "org.apache.spark.sql.catalyst.expressions.StringLocate", | ||
| // _FUNC_ is replaced by `%` which causes a parsing error on `SELECT %(2, 1.8)` | ||
| "org.apache.spark.sql.catalyst.expressions.Remainder", | ||
| // Examples demonstrate alternative names, see SPARK-20749 | ||
| "org.apache.spark.sql.catalyst.expressions.Length") | ||
| spark.sessionState.functionRegistry.listFunction().foreach { funcId => | ||
| val info = spark.sessionState.catalog.lookupFunctionInfo(funcId) | ||
| val className = info.getClassName | ||
| withClue(s"Expression class '$className'") { | ||
| val exprExamples = info.getOriginalExamples | ||
| if (!exprExamples.isEmpty && !ignoreSet.contains(className)) { | ||
| assert(exampleRe.findAllIn(exprExamples).toIterable | ||
| .filter(setStmtRe.findFirstIn(_).isEmpty) // Ignore SET commands | ||
| .forall(_.contains("_FUNC_"))) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| test("check outputs of expression examples") { | ||
|
Contributor
There was a problem hiding this comment. Hi, @maropu this test case has a minor problem when running individually for machines in different timezones: expressions such as
Contributor
There was a problem hiding this comment. @HyukjinKwon and @maropu thanks for the information, I was checking an older version of Spark 3.0 |
||
| def unindentAndTrim(s: String): String = { | ||
| s.replaceAll("\n\\s+", "\n").trim | ||
| } | ||
| val beginSqlStmtRe = " > ".r | ||
| val endSqlStmtRe = ";\n".r | ||
| def checkExampleSyntax(example: String): Unit = { | ||
| val beginStmtNum = beginSqlStmtRe.findAllIn(example).length | ||
| val endStmtNum = endSqlStmtRe.findAllIn(example).length | ||
| assert(beginStmtNum === endStmtNum, | ||
| "The number of ` > ` does not match to the number of `;`") | ||
| } | ||
| val exampleRe = """^(.+);\n(?s)(.+)$""".r | ||
| val ignoreSet = Set( | ||
| // One of examples shows getting the current timestamp | ||
| "org.apache.spark.sql.catalyst.expressions.UnixTimestamp", | ||
| // Random output without a seed | ||
| "org.apache.spark.sql.catalyst.expressions.Rand", | ||
| "org.apache.spark.sql.catalyst.expressions.Randn", | ||
| "org.apache.spark.sql.catalyst.expressions.Shuffle", | ||
| "org.apache.spark.sql.catalyst.expressions.Uuid", | ||
| // The example calls methods that return unstable results. | ||
| "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection") | ||
|
|
||
| val parFuncs = new ParVector(spark.sessionState.functionRegistry.listFunction().toVector) | ||
| parFuncs.foreach { funcId => | ||
| // Examples can change settings. We clone the session to prevent tests clashing. | ||
| val clonedSpark = spark.cloneSession() | ||
| // Coalescing partitions can change result order, so disable it. | ||
| clonedSpark.sessionState.conf.setConf(SQLConf.COALESCE_PARTITIONS_ENABLED, false) | ||
| val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId) | ||
| val className = info.getClassName | ||
| if (!ignoreSet.contains(className)) { | ||
| withClue(s"Function '${info.getName}', Expression class '$className'") { | ||
| val example = info.getExamples | ||
| checkExampleSyntax(example) | ||
| example.split(" > ").toList.foreach { | ||
|
Member
Author
There was a problem hiding this comment. minor: I modified this line a bit as my IDE suggested. |
||
| case exampleRe(sql, output) => | ||
| val df = clonedSpark.sql(sql) | ||
| val actual = unindentAndTrim( | ||
| hiveResultString(df.queryExecution.executedPlan).mkString("\n")) | ||
| val expected = unindentAndTrim(output) | ||
| assert(actual === expected) | ||
| case _ => | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why all the csv _funcs are left behind?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is just a basic set for the 3.0 release. Yea, we can improve docs by adding more groups. Actually, we need to assign all the exprs to groups for removing the SQL Built-in Function doc:
#28224 (comment) @HyukjinKwon
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I see, thanks