From afe5b2efc49933f17a26e5d68965a1740f57b29f Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 14 Aug 2024 10:43:19 +0800 Subject: [PATCH 1/2] [SPARK-49232][INFRA][TESTS] Remove the logic related to `SKIP_SPARK_RELEASE_VERSIONS` --- .github/workflows/build_maven.yml | 5 ----- .../sql/hive/HiveExternalCatalogVersionsSuite.scala | 9 +-------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/build_maven.yml b/.github/workflows/build_maven.yml index c3a23c02f6a61..b5546c61eb11b 100644 --- a/.github/workflows/build_maven.yml +++ b/.github/workflows/build_maven.yml @@ -30,8 +30,3 @@ jobs: name: Run uses: ./.github/workflows/maven_test.yml if: github.repository == 'apache/spark' - with: - envs: >- - { - "SKIP_SPARK_RELEASE_VERSIONS": "3.4.2" - } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 95baffdee06cb..5d53cdb2a37ad 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -248,12 +248,6 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { } object PROCESS_TABLES extends QueryTest with SQLTestUtils { - // TODO In SPARK-46302, the env SKIP_SPARK_RELEASE_VERSIONS has been added to - // allow Maven tests to skip problematic release versions. - // Related issues will be fixed in SPARK-46400, and testing will be resumed - // after the fixed Spark 3.x version is released. - private val skipReleaseVersions = - sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet val isPythonVersionAvailable = TestUtils.isPythonVersionAvailable val releaseMirror = sys.env.getOrElse("SPARK_RELEASE_MIRROR", "https://dist.apache.org/repos/dist/release") @@ -268,8 +262,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils { .filter(_.contains("""""".r.findFirstMatchIn(_).get.group(1)) - .filter(_ < org.apache.spark.SPARK_VERSION) - .filterNot(skipReleaseVersions.contains).toImmutableArraySeq + .filter(_ < org.apache.spark.SPARK_VERSION).toImmutableArraySeq } catch { // Do not throw exception during object initialization. case NonFatal(_) => Nil From ff0137467fe73ad76371f22a94edfb957e33ec5d Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 14 Aug 2024 11:24:10 +0800 Subject: [PATCH 2/2] update --- .../sql/hive/HiveExternalCatalogVersionsSuite.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 5d53cdb2a37ad..95baffdee06cb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -248,6 +248,12 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { } object PROCESS_TABLES extends QueryTest with SQLTestUtils { + // TODO In SPARK-46302, the env SKIP_SPARK_RELEASE_VERSIONS has been added to + // allow Maven tests to skip problematic release versions. + // Related issues will be fixed in SPARK-46400, and testing will be resumed + // after the fixed Spark 3.x version is released. + private val skipReleaseVersions = + sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet val isPythonVersionAvailable = TestUtils.isPythonVersionAvailable val releaseMirror = sys.env.getOrElse("SPARK_RELEASE_MIRROR", "https://dist.apache.org/repos/dist/release") @@ -262,7 +268,8 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils { .filter(_.contains("""""".r.findFirstMatchIn(_).get.group(1)) - .filter(_ < org.apache.spark.SPARK_VERSION).toImmutableArraySeq + .filter(_ < org.apache.spark.SPARK_VERSION) + .filterNot(skipReleaseVersions.contains).toImmutableArraySeq } catch { // Do not throw exception during object initialization. case NonFatal(_) => Nil