From 03627c2dc9739707b6b9e3905dd9cb7b92c93fae Mon Sep 17 00:00:00 2001 From: angerszhu Date: Wed, 6 Jan 2021 05:12:05 +0800 Subject: [PATCH 1/7] [SPARK-33934][SQL][FOLLOW-UP] Add SparkFile's root dir to env property PATH --- .../spark/sql/execution/BaseScriptTransformationExec.scala | 2 +- sql/core/src/test/resources/test_script.py | 2 +- .../spark/sql/execution/BaseScriptTransformationSuite.scala | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala index b66f94ae1107..669b90f4d06d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution -import java.io._ +import java.io.{BufferedReader, File, InputStream, InputStreamReader, OutputStream} import java.nio.charset.StandardCharsets import java.util.concurrent.TimeUnit diff --git a/sql/core/src/test/resources/test_script.py b/sql/core/src/test/resources/test_script.py index 75b4f106d3a1..a2cc08492ea6 100644 --- a/sql/core/src/test/resources/test_script.py +++ b/sql/core/src/test/resources/test_script.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/python3 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala index a25e4b8f8ea0..cef870b24998 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala @@ -521,7 +521,10 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU 'd.cast("string"), 'e.cast("string")).collect()) }.getMessage - assert(e1.contains("Permission denied")) + // Check with status exit code since in GA test, it may lose detail failed root cause. + // Different root cause's exitcode is not same. + // In this test, root cause is `Permission denied` + assert(e1.contains("Subprocess exited with status 126")) // test `/path/to/script.py' with script executable scriptFilePath.setExecutable(true) From 1fbc91c703d4fb94ed0029e7d6ca9a0a17b8bd30 Mon Sep 17 00:00:00 2001 From: AngersZhuuuu Date: Wed, 6 Jan 2021 09:27:39 +0800 Subject: [PATCH 2/7] Update sql/core/src/test/resources/test_script.py Co-authored-by: Hyukjin Kwon --- sql/core/src/test/resources/test_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/test_script.py b/sql/core/src/test/resources/test_script.py index a2cc08492ea6..4fcd483f44d4 100644 --- a/sql/core/src/test/resources/test_script.py +++ b/sql/core/src/test/resources/test_script.py @@ -1,4 +1,4 @@ -#! /usr/bin/python3 +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with From 19f124f2c5e05695e1e2afd4bfaa6e49d45f6a40 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Wed, 6 Jan 2021 11:16:51 +0800 Subject: [PATCH 3/7] Empty From 7c2ce48374ee42545842cf0369e28e500f42d605 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Wed, 6 Jan 2021 11:16:52 +0800 Subject: [PATCH 4/7] Empty From e5cf2887b99d4ece1c455a54e889ad3281fe8008 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Wed, 6 Jan 2021 11:16:52 +0800 Subject: [PATCH 5/7] Empty From 404fb631ce7fb5fd69cf560676d9ca3706517d5e Mon Sep 17 00:00:00 2001 From: angerszhu Date: Wed, 6 Jan 2021 12:16:21 +0800 Subject: [PATCH 6/7] Empty From db12e28a3cd5f3c19810f301a83c5667ff661058 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Wed, 6 Jan 2021 13:14:16 +0800 Subject: [PATCH 7/7] Empty