From d205a131f1df0a302d008743745acf1aeba36b7a Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 23 Dec 2020 08:49:39 +0900 Subject: [PATCH 1/5] Fix --- .../resources/sql-tests/inputs/transform.sql | 10 -------- .../sql-tests/results/transform.sql.out | 24 +------------------ .../BaseScriptTransformationSuite.scala | 22 +++++++++++++++++ 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/transform.sql b/sql/core/src/test/resources/sql-tests/inputs/transform.sql index 65b060eca3a62..3f39700a95913 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/transform.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/transform.sql @@ -9,16 +9,6 @@ SELECT TRANSFORM(a) USING 'cat' AS (a) FROM t; --- with non-exist command -SELECT TRANSFORM(a) -USING 'some_non_existent_command' AS (a) -FROM t; - --- with non-exist file -SELECT TRANSFORM(a) -USING 'python some_non_existent_file' AS (a) -FROM t; - -- common supported data types between no serde and serde transform SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM ( SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l) diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out index 83ab5cb729c24..3267a7625a7d9 100644 --- a/sql/core/src/test/resources/sql-tests/results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 18 +-- Number of queries: 16 -- !query @@ -26,28 +26,6 @@ struct 3 --- !query -SELECT TRANSFORM(a) -USING 'some_non_existent_command' AS (a) -FROM t --- !query schema -struct<> --- !query output -org.apache.spark.SparkException -Subprocess exited with status 127. Error: /bin/bash: some_non_existent_command: command not found - - --- !query -SELECT TRANSFORM(a) -USING 'python some_non_existent_file' AS (a) -FROM t --- !query schema -struct<> --- !query output -org.apache.spark.SparkException -Subprocess exited with status 2. Error: python: can't open file 'some_non_existent_file': [Errno 2] No such file or directory - - -- !query SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM ( SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala index 81f292809df4a..9a103c1be6f9c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala @@ -420,6 +420,28 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU 'b.cast("string").as("b"), lit(null), lit(null)).collect()) } + + test("SPARK-32106: TRANSFORM with non-existent command/file") { + val e1 = intercept[SparkException] { + sql( + """ + |SELECT TRANSFORM(a) + |USING 'some_non_existent_command' AS (a) + |FROM VALUES (1) t(a) + """.stripMargin).collect() + }.getMessage + assert(e1.contains("Subprocess exited")) + + val e2 = intercept[SparkException] { + sql( + """ + |SELECT TRANSFORM(a) + |USING 'python some_non_existent_file' AS (a) + |FROM VALUES (1) t(a) + """.stripMargin).collect() + }.getMessage + assert(e2.contains("Subprocess exited")) + } } case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode { From b38541f330754c2aabe02a1307aa684d1b7c2f9a Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 23 Dec 2020 10:40:18 +0900 Subject: [PATCH 2/5] Fix --- .../BaseScriptTransformationSuite.scala | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala index 9a103c1be6f9c..863657a7862a6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala @@ -422,25 +422,23 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU } test("SPARK-32106: TRANSFORM with non-existent command/file") { - val e1 = intercept[SparkException] { - sql( - """ - |SELECT TRANSFORM(a) - |USING 'some_non_existent_command' AS (a) - |FROM VALUES (1) t(a) - """.stripMargin).collect() - }.getMessage - assert(e1.contains("Subprocess exited")) - - val e2 = intercept[SparkException] { - sql( - """ - |SELECT TRANSFORM(a) - |USING 'python some_non_existent_file' AS (a) - |FROM VALUES (1) t(a) - """.stripMargin).collect() - }.getMessage - assert(e2.contains("Subprocess exited")) + Seq( + s""" + |SELECT TRANSFORM(a) + |USING 'some_non_existent_command' AS (a) + |FROM VALUES (1) t(a) + """.stripMargin, + s""" + |SELECT TRANSFORM(a) + |USING 'python some_non_existent_file' AS (a) + |FROM VALUES (1) t(a) + """.stripMargin).foreach { query => + intercept[SparkException] { + // Since an error message is shell-dependent, this test just checks + // if the expected exception will be thrown. + sql(query).collect() + } + } } } From ddb8061af5e6d128cac2bdd256964e6381b95005 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 23 Dec 2020 11:19:32 +0900 Subject: [PATCH 3/5] empty commit #1 From 194f4d7c6808f0383a1a3ab8f3f12214bc42eda5 Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 23 Dec 2020 11:21:46 +0900 Subject: [PATCH 4/5] empty commit #2 From b29d6a1a1f52bccb0bb5574cf93684bb9bf6befc Mon Sep 17 00:00:00 2001 From: Takeshi Yamamuro Date: Wed, 23 Dec 2020 11:22:00 +0900 Subject: [PATCH 5/5] empty commit #3