From d98ffeeac352080f7f1941b59e98926d637ddaec Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Wed, 21 Oct 2020 14:49:39 -0700 Subject: [PATCH 1/4] [SPARK-33214][TEST][HIVE] Stop HiveExternalCatalogVersionsSuite from using a hard-coded location to store localized Spark binaries. --- .../HiveExternalCatalogVersionsSuite.scala | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index b81b7e8ec0c0..0522f7f72242 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -42,26 +42,33 @@ import org.apache.spark.util.Utils * Test HiveExternalCatalog backward compatibility. * * Note that, this test suite will automatically download spark binary packages of different - * versions to a local directory `/tmp/spark-test`. If there is already a spark folder with - * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the - * downloading for this spark version. + * versions to a local directory. If the `spark.test.cache-dir` system property is defined, this + * directory will be used. If there is already a spark folder with expected version under this + * local directory, e.g. `//spark-2.0.3`, downloading for this spark version will be + * skipped. If the system property is not present, a temporary directory will be used and cleaned + * up after the test. */ @SlowHiveTest @ExtendedHiveTest class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { + import HiveExternalCatalogVersionsSuite._ private val isTestAtLeastJava9 = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9) private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") private val tmpDataDir = Utils.createTempDir(namePrefix = "test-data") - // For local test, you can set `sparkTestingDir` to a static value like `/tmp/test-spark`, to + // For local test, you can set `spark.test.cache-dir` to a static value like `/tmp/test-spark`, to // avoid downloading Spark of different versions in each run. - private val sparkTestingDir = new File("/tmp/test-spark") + private val sparkTestingDir = Option(System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY)) + .map(new File(_)).getOrElse(Utils.createTempDir(namePrefix = "test-spark")) private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) override def afterAll(): Unit = { try { Utils.deleteRecursively(wareHousePath) Utils.deleteRecursively(tmpDataDir) - Utils.deleteRecursively(sparkTestingDir) + // Only delete sparkTestingDir if it wasn't defined to a static location by the system prop + if (System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY).isEmpty) { + Utils.deleteRecursively(sparkTestingDir) + } } finally { super.afterAll() } @@ -307,3 +314,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils { } } } + +object HiveExternalCatalogVersionsSuite { + private val SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY = "spark.test.cache-dir" +} \ No newline at end of file From f41ae2436331cc193a2b93f400dc63508c3d36ff Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Wed, 21 Oct 2020 15:44:58 -0700 Subject: [PATCH 2/4] Add newline --- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 0522f7f72242..41710138406b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -317,4 +317,5 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils { object HiveExternalCatalogVersionsSuite { private val SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY = "spark.test.cache-dir" -} \ No newline at end of file +} + From 2b999582d3a1dbcca147fd7b36aab4d6d50ea36d Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Thu, 22 Oct 2020 11:57:56 -0700 Subject: [PATCH 3/4] Fix Scaladoc complaint --- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 41710138406b..312017d2607d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -44,7 +44,7 @@ import org.apache.spark.util.Utils * Note that, this test suite will automatically download spark binary packages of different * versions to a local directory. If the `spark.test.cache-dir` system property is defined, this * directory will be used. If there is already a spark folder with expected version under this - * local directory, e.g. `//spark-2.0.3`, downloading for this spark version will be + * local directory, e.g. `/{cache-dir}/spark-2.0.3`, downloading for this spark version will be * skipped. If the system property is not present, a temporary directory will be used and cleaned * up after the test. */ From c56cf1f0a6fde354fc80a3db1def6e563a0fbe94 Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Thu, 22 Oct 2020 13:38:21 -0700 Subject: [PATCH 4/4] Wrap null in Option. I thought this was already there, looks like I missed it. --- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 312017d2607d..38a8c492d77a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -66,7 +66,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { Utils.deleteRecursively(wareHousePath) Utils.deleteRecursively(tmpDataDir) // Only delete sparkTestingDir if it wasn't defined to a static location by the system prop - if (System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY).isEmpty) { + if (Option(System.getProperty(SPARK_TEST_CACHE_DIR_SYSTEM_PROPERTY)).isEmpty) { Utils.deleteRecursively(sparkTestingDir) } } finally {