From f0f703554bda8bae69b5893ca90c5d09bc698cdd Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Dec 2023 19:12:05 +0800 Subject: [PATCH 01/38] try fix maven test --- .github/workflows/build_and_test.yml | 6 ++++++ .github/workflows/maven_test.yml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d58356ec1c5dc..4a2f7c5019b4f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1103,3 +1103,9 @@ jobs: cd ui-test npm install --save-dev node --experimental-vm-modules node_modules/.bin/jest + + maven-test: + name: "Run Maven Test" + permissions: + packages: write + uses: ./.github/workflows/maven_test.yml diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 89d46e7961e2d..991575bb19b56 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -149,6 +149,10 @@ jobs: key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | java${{ matrix.java }}-maven- + - name: Clear Ivy Cache + run: | + rm -rf ~/.ivy2/cache/org.apache.hive/hive-exec + rm -rf ~/.ivy2/cache/org.slf4j/slf4j-api - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From 0b75a0fe03438311dac488788b25620bbd5ece7c Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Dec 2023 21:28:37 +0800 Subject: [PATCH 02/38] try --- .github/workflows/maven_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 991575bb19b56..3d5a96e4411c1 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,8 +151,11 @@ jobs: java${{ matrix.java }}-maven- - name: Clear Ivy Cache run: | + ls -al ~ rm -rf ~/.ivy2/cache/org.apache.hive/hive-exec rm -rf ~/.ivy2/cache/org.slf4j/slf4j-api + rm -rf ~/.m2/repository/org/apache/hive/hive-exec/2.3.9 + rm -rf ~/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From 08e8859ce82c0f223434a6d602cb76c949e43b34 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Dec 2023 21:33:26 +0800 Subject: [PATCH 03/38] try --- .github/workflows/maven_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 3d5a96e4411c1..5b044e970c189 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,6 +151,8 @@ jobs: java${{ matrix.java }}-maven- - name: Clear Ivy Cache run: | + ls -al /home/runner/work/spark/spark + ls -al /home/runner/work/spark ls -al ~ rm -rf ~/.ivy2/cache/org.apache.hive/hive-exec rm -rf ~/.ivy2/cache/org.slf4j/slf4j-api From 5fca2b7399c748f738428a37b14161e0c0b401ff Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Dec 2023 21:46:36 +0800 Subject: [PATCH 04/38] try --- .github/workflows/maven_test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 5b044e970c189..9e5a9aa54a34e 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -153,11 +153,12 @@ jobs: run: | ls -al /home/runner/work/spark/spark ls -al /home/runner/work/spark + pwd ls -al ~ - rm -rf ~/.ivy2/cache/org.apache.hive/hive-exec - rm -rf ~/.ivy2/cache/org.slf4j/slf4j-api - rm -rf ~/.m2/repository/org/apache/hive/hive-exec/2.3.9 - rm -rf ~/.m2/repository/org/slf4j/slf4j-api/1.7.30 + rm -rf /home/runner/.ivy2/cache/org.apache.hive/hive-exec + rm -rf /home/runner/.ivy2/cache/org.slf4j/slf4j-api + rm -rf /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + rm -rf /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From db9b07e92e72d60bf71c6c03e29f7003698e9d1e Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Dec 2023 21:47:18 +0800 Subject: [PATCH 05/38] try --- .github/workflows/maven_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 9e5a9aa54a34e..46e5841b96fa3 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -155,6 +155,7 @@ jobs: ls -al /home/runner/work/spark pwd ls -al ~ + ls -al /home/runner rm -rf /home/runner/.ivy2/cache/org.apache.hive/hive-exec rm -rf /home/runner/.ivy2/cache/org.slf4j/slf4j-api rm -rf /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 From 742e29b8a40f01202185ea04969d1ddf0b854f88 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 6 Dec 2023 22:00:59 +0800 Subject: [PATCH 06/38] try --- .../sql/hive/HiveExternalCatalogVersionsSuite.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 52f20595a10a4..c8f1e1b1e2175 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -31,6 +31,7 @@ import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkConf, TestUtils} import org.apache.spark.deploy.SparkSubmitTestUtils +import org.apache.spark.internal.Logging import org.apache.spark.internal.config.MASTER_REST_SERVER_ENABLED import org.apache.spark.internal.config.UI.UI_ENABLED import org.apache.spark.launcher.JavaModuleOptions @@ -55,7 +56,7 @@ import org.apache.spark.util.ArrayImplicits._ */ @SlowHiveTest @ExtendedHiveTest -class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils with Logging { import HiveExternalCatalogVersionsSuite._ override protected val defaultSparkSubmitTimeout: Span = 5.minutes private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") @@ -163,6 +164,16 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { override def beforeAll(): Unit = { super.beforeAll() + val hiveExecFile = new File("/home/runner/.ivy2/cache/org.apache.hive/hive-exec") + if (hiveExecFile.exists()) { + logInfo("hive exec file existed.") + Utils.deleteRecursively(hiveExecFile) + } + val slf4jApiFile = new File("/home/runner/.ivy2/cache/org.slf4j/slf4j-api") + if (slf4jApiFile.exists()) { + logInfo("slf4j api file existed.") + Utils.deleteRecursively(slf4jApiFile) + } val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit From 953bc7e779725da669849768546865bc83f261f8 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 09:28:39 +0800 Subject: [PATCH 07/38] try --- .github/workflows/maven_test.yml | 1 + .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 46e5841b96fa3..3fac65cd3974f 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -160,6 +160,7 @@ jobs: rm -rf /home/runner/.ivy2/cache/org.slf4j/slf4j-api rm -rf /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 rm -rf /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 + find . -name "hive-exec*" - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index c8f1e1b1e2175..e4b6d2a333279 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -168,11 +168,15 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils with Logging if (hiveExecFile.exists()) { logInfo("hive exec file existed.") Utils.deleteRecursively(hiveExecFile) + } else { + logInfo("hive exec file non-exist.") } val slf4jApiFile = new File("/home/runner/.ivy2/cache/org.slf4j/slf4j-api") if (slf4jApiFile.exists()) { logInfo("slf4j api file existed.") Utils.deleteRecursively(slf4jApiFile) + } else { + logInfo("slf4j api file non-exist.") } val tempPyFile = File.createTempFile("test", ".py") From e3c6e8ce176fd109eff33b32f4982509bb825b02 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 09:31:17 +0800 Subject: [PATCH 08/38] try --- .github/workflows/maven_test.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 3fac65cd3974f..090ad7b5f924b 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,15 +151,6 @@ jobs: java${{ matrix.java }}-maven- - name: Clear Ivy Cache run: | - ls -al /home/runner/work/spark/spark - ls -al /home/runner/work/spark - pwd - ls -al ~ - ls -al /home/runner - rm -rf /home/runner/.ivy2/cache/org.apache.hive/hive-exec - rm -rf /home/runner/.ivy2/cache/org.slf4j/slf4j-api - rm -rf /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - rm -rf /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 find . -name "hive-exec*" - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 From 4377bfcc1da3e85cf27a57a6779dd3634316488b Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 09:35:27 +0800 Subject: [PATCH 09/38] try --- .github/workflows/maven_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 090ad7b5f924b..1e5977e923df0 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,7 +151,9 @@ jobs: java${{ matrix.java }}-maven- - name: Clear Ivy Cache run: | - find . -name "hive-exec*" + pwd + find /home/runner -name "hive-exec*" + pwd - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: From c128af6d0b000dc0b1e1745aced9d1e54f9008cd Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 10:07:06 +0800 Subject: [PATCH 10/38] try --- .github/workflows/maven_test.yml | 6 +++++- .../hive/HiveExternalCatalogVersionsSuite.scala | 16 +--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 1e5977e923df0..c146be6eedf2c 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,9 +151,13 @@ jobs: java${{ matrix.java }}-maven- - name: Clear Ivy Cache run: | + env pwd find /home/runner -name "hive-exec*" - pwd + mkdir -p /home/runner/.ivy2/jars + mkdir -p /home/runner/.ivy2/cache + ls -al /home/runner + ls -al /home/runner/.ivy2 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index e4b6d2a333279..84acac4c5a6d3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -164,21 +164,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils with Logging override def beforeAll(): Unit = { super.beforeAll() - val hiveExecFile = new File("/home/runner/.ivy2/cache/org.apache.hive/hive-exec") - if (hiveExecFile.exists()) { - logInfo("hive exec file existed.") - Utils.deleteRecursively(hiveExecFile) - } else { - logInfo("hive exec file non-exist.") - } - val slf4jApiFile = new File("/home/runner/.ivy2/cache/org.slf4j/slf4j-api") - if (slf4jApiFile.exists()) { - logInfo("slf4j api file existed.") - Utils.deleteRecursively(slf4jApiFile) - } else { - logInfo("slf4j api file non-exist.") - } - + logger.info("Env : " + System.getenv()) val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, From a7e1b1268bd43b1ac17417a96b943892100d2107 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 10:25:38 +0800 Subject: [PATCH 11/38] try --- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 84acac4c5a6d3..a8a00a79e4702 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -164,7 +164,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils with Logging override def beforeAll(): Unit = { super.beforeAll() - logger.info("Env : " + System.getenv()) + logInfo("Env : " + System.getenv()) val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, From a0b9450ad881ed4886afeb005f5d8a5f14174485 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 11:35:28 +0800 Subject: [PATCH 12/38] try --- .github/workflows/maven_test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index c146be6eedf2c..d688eae9086a8 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -158,6 +158,10 @@ jobs: mkdir -p /home/runner/.ivy2/cache ls -al /home/runner ls -al /home/runner/.ivy2 + ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet + ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet + ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: @@ -204,6 +208,11 @@ jobs: else ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} test -fae fi + - name: List Ivy Cache + run: | + ls -al /home/runner + ls -al /home/runner/.ivy2 + find /home/runner/.ivy2 -name "hive-exec*" - name: Clean up local Maven repository run: | rm -rf ~/.m2/repository/org/apache/spark From 77e94e99725d45ad39bd17c9b6fd80f8b18ece07 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 14:00:17 +0800 Subject: [PATCH 13/38] fix maven tests --- .github/workflows/maven_test.yml | 21 +++---------------- .../HiveExternalCatalogVersionsSuite.scala | 4 +--- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index d688eae9086a8..dbf5495675265 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -149,19 +149,6 @@ jobs: key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | java${{ matrix.java }}-maven- - - name: Clear Ivy Cache - run: | - env - pwd - find /home/runner -name "hive-exec*" - mkdir -p /home/runner/.ivy2/jars - mkdir -p /home/runner/.ivy2/cache - ls -al /home/runner - ls -al /home/runner/.ivy2 - ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet - ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet - ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: @@ -193,6 +180,9 @@ jobs: export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} + # Fix some corrupted jars + ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet + ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10 export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pspark-ganglia-lgpl -Djava.version=${JAVA_VERSION/-ea} clean install @@ -208,11 +198,6 @@ jobs: else ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} test -fae fi - - name: List Ivy Cache - run: | - ls -al /home/runner - ls -al /home/runner/.ivy2 - find /home/runner/.ivy2 -name "hive-exec*" - name: Clean up local Maven repository run: | rm -rf ~/.m2/repository/org/apache/spark diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index a8a00a79e4702..7143fc1432d62 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -31,7 +31,6 @@ import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkConf, TestUtils} import org.apache.spark.deploy.SparkSubmitTestUtils -import org.apache.spark.internal.Logging import org.apache.spark.internal.config.MASTER_REST_SERVER_ENABLED import org.apache.spark.internal.config.UI.UI_ENABLED import org.apache.spark.launcher.JavaModuleOptions @@ -56,7 +55,7 @@ import org.apache.spark.util.ArrayImplicits._ */ @SlowHiveTest @ExtendedHiveTest -class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils with Logging { +class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { import HiveExternalCatalogVersionsSuite._ override protected val defaultSparkSubmitTimeout: Span = 5.minutes private val wareHousePath = Utils.createTempDir(namePrefix = "warehouse") @@ -164,7 +163,6 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils with Logging override def beforeAll(): Unit = { super.beforeAll() - logInfo("Env : " + System.getenv()) val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, From 436c24496dee176cef56814dc82697879a0bf598 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 14:01:44 +0800 Subject: [PATCH 14/38] fix maven tests --- .../apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 7143fc1432d62..52f20595a10a4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -163,6 +163,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { override def beforeAll(): Unit = { super.beforeAll() + val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, From a9e146a801bce69b4a4c526e9725f46c077d84ed Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 14:07:39 +0800 Subject: [PATCH 15/38] fix maven tests --- .github/workflows/maven_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index dbf5495675265..fad64af164cec 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -149,6 +149,10 @@ jobs: key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | java${{ matrix.java }}-maven- + - name: List Maven repository some jars - Before + run: | + ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: @@ -198,6 +202,10 @@ jobs: else ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} test -fae fi + - name: List Maven repository some jars - After + run: | + ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Clean up local Maven repository run: | rm -rf ~/.m2/repository/org/apache/spark From b7ae5cb093d0b94b61d7b8dcbae0fa675cdbc5b0 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 14:10:04 +0800 Subject: [PATCH 16/38] fix maven tests --- .github/workflows/maven_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index fad64af164cec..eba6d6515315d 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -187,6 +187,8 @@ jobs: # Fix some corrupted jars ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet + ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10 export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pspark-ganglia-lgpl -Djava.version=${JAVA_VERSION/-ea} clean install From a57c512ef34d925c911179db5ea343ab5a4d7f71 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 16:03:56 +0800 Subject: [PATCH 17/38] try --- .github/workflows/maven_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index eba6d6515315d..bcd6ed4ca48ae 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -185,8 +185,8 @@ jobs: export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} # Fix some corrupted jars - ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet - ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet + # ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet + # ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10 From 2fe75bd4427dd59497948aa0e6b0eba9909c82ec Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 16:06:21 +0800 Subject: [PATCH 18/38] try --- .github/workflows/maven_test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index bcd6ed4ca48ae..de21d5626da96 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,8 +151,9 @@ jobs: java${{ matrix.java }}-maven- - name: List Maven repository some jars - Before run: | - ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 + ls -al /home/runner + # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v3 with: @@ -187,8 +188,8 @@ jobs: # Fix some corrupted jars # ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet # ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet - ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 + # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 + # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10 export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pspark-ganglia-lgpl -Djava.version=${JAVA_VERSION/-ea} clean install From 5a971b4d38aef38550032d769fd77197a418c43f Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 17:18:53 +0800 Subject: [PATCH 19/38] try --- sql/hive/pom.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 42618fe5bc22d..eb860362d92e1 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -186,6 +186,20 @@ scalacheck_${scala.binary.version} test + + + + ${hive.group} + hive-exec + test + 2.3.9 + + + org.slf4j + slf4j-api + test + 1.7.30 + From dbcbd647e971ed5b53a8c878934c964b4a5ffc1d Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 18:36:01 +0800 Subject: [PATCH 20/38] try --- sql/hive/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index eb860362d92e1..8a8cfb0ce2064 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -193,7 +193,14 @@ hive-exec test 2.3.9 + + + org.pentaho + * + + + org.slf4j slf4j-api From b2fd3eef5f4cbd68a7dea29e6bbb64df42e3e09e Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 7 Dec 2023 20:45:35 +0800 Subject: [PATCH 21/38] try --- sql/hive/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 8a8cfb0ce2064..88e049bae9bc9 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -198,6 +198,14 @@ org.pentaho * + + org.codehaus.groovy + groovy-all + + + * + * + From 52217951ba155bab73351c95a67114d996957327 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 09:55:51 +0800 Subject: [PATCH 22/38] try --- .github/workflows/maven_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 15490b51e8757..2fe00da953441 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,6 +151,8 @@ jobs: java${{ matrix.java }}-maven- - name: List Maven repository some jars - Before run: | + find / -name "hive-exec*" + find / -name "*ivy2*" ls -al /home/runner # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 From dcbf92276d5fb2e310cee0dfb4dfeaa0dd709479 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 10:08:35 +0800 Subject: [PATCH 23/38] try --- .github/workflows/maven_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 2fe00da953441..33a68c9aa129e 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,8 +151,7 @@ jobs: java${{ matrix.java }}-maven- - name: List Maven repository some jars - Before run: | - find / -name "hive-exec*" - find / -name "*ivy2*" + find local -name "*" -type f ls -al /home/runner # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 From e4774cd735d915da0943c7258d744a3cf00c17aa Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 10:25:32 +0800 Subject: [PATCH 24/38] try --- .github/workflows/maven_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 33a68c9aa129e..c4249620c14c1 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -151,8 +151,9 @@ jobs: java${{ matrix.java }}-maven- - name: List Maven repository some jars - Before run: | - find local -name "*" -type f ls -al /home/runner + find /home/runner -name "*hive-exec*" + find /home/runner -name "*ivy*" # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} From c7561d2aa5ed6af39651ac1fcf69668ac7aeb7f3 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 10:54:32 +0800 Subject: [PATCH 25/38] try --- .github/workflows/maven_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index c4249620c14c1..18291ebd2b6fe 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -154,6 +154,8 @@ jobs: ls -al /home/runner find /home/runner -name "*hive-exec*" find /home/runner -name "*ivy*" + find /home/runner -name "slf4j-api-*" + find /home/runner -name "cache" # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} From 6f9a81cdf07fb5df8099b21b9a467e6aa9e2f1e0 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 11:35:08 +0800 Subject: [PATCH 26/38] try --- .github/workflows/maven_test.yml | 1 + sql/hive/pom.xml | 29 ------------------- .../HiveExternalCatalogVersionsSuite.scala | 9 +++++- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 18291ebd2b6fe..286c3758563c7 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -156,6 +156,7 @@ jobs: find /home/runner -name "*ivy*" find /home/runner -name "slf4j-api-*" find /home/runner -name "cache" + find /home/runner -name "jars" # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 88e049bae9bc9..42618fe5bc22d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -186,35 +186,6 @@ scalacheck_${scala.binary.version} test - - - - ${hive.group} - hive-exec - test - 2.3.9 - - - org.pentaho - * - - - org.codehaus.groovy - groovy-all - - - * - * - - - - - - org.slf4j - slf4j-api - test - 1.7.30 - diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 52f20595a10a4..df17ab2eb82ec 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -163,7 +163,14 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { override def beforeAll(): Unit = { super.beforeAll() - + // scalastyle:off println + println("-----------------") + val tmpDirName = System.getProperty("java.io.tmpdir") + val ivyCacheDir = System.getProperty("ivy.cache.dir") + val ivyHome = System.getProperty("ivy.home") + println(s"tmpDirName: $tmpDirName, ivyCacheDir: $ivyCacheDir, ivyHome: $ivyHome") + println("-----------------") + // scalastyle:on println val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, From 311a5dd156b20b177d14cd2edef032ba8bde0c4f Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 16:40:15 +0800 Subject: [PATCH 27/38] try --- .../HiveExternalCatalogVersionsSuite.scala | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index df17ab2eb82ec..9281d2e2a035e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -31,7 +31,7 @@ import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkConf, TestUtils} import org.apache.spark.deploy.SparkSubmitTestUtils -import org.apache.spark.internal.config.MASTER_REST_SERVER_ENABLED +import org.apache.spark.internal.config.{JAR_IVY_REPO_PATH, MASTER_REST_SERVER_ENABLED} import org.apache.spark.internal.config.UI.UI_ENABLED import org.apache.spark.launcher.JavaModuleOptions import org.apache.spark.sql.{QueryTest, Row, SparkSession} @@ -163,14 +163,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { override def beforeAll(): Unit = { super.beforeAll() - // scalastyle:off println - println("-----------------") - val tmpDirName = System.getProperty("java.io.tmpdir") - val ivyCacheDir = System.getProperty("ivy.cache.dir") - val ivyHome = System.getProperty("ivy.home") - println(s"tmpDirName: $tmpDirName, ivyCacheDir: $ivyCacheDir, ivyHome: $ivyHome") - println("-----------------") - // scalastyle:on println + val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, @@ -223,7 +216,8 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { if (!sparkHome.exists()) { tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) } - + val ivyHome = Utils.createTempDir(root = sparkTestingDir.getCanonicalPath, + namePrefix = s"ivy2-$version") // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get val args = Seq( @@ -233,11 +227,26 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveMetastoreVersion", "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", + "--conf", s"${JAR_IVY_REPO_PATH.key}=$ivyHome", "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", "--conf", s"spark.sql.test.version.index=$index", "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath} " + JavaModuleOptions.defaultModuleOptions(), tempPyFile.getCanonicalPath) + // scalastyle:off println + println("-----------------") + val tmpDirName = System.getProperty("java.io.tmpdir") + val ivyCacheDir = System.getProperty("ivy.cache.dir") + val defaultIvyHome = System.getProperty("ivy.home") + val ivyDefaultOvyUserDir = System.getProperty("ivy.default.ivy.user.dir") + Utils.recursiveList(new File(System.getProperty("java.io.tmpdir"))).foreach(f => { + print(f.getCanonicalPath) + }) + println(s"tmpDirName: $tmpDirName, ivyCacheDir: $ivyCacheDir, " + + s"defaultIvyHome: $defaultIvyHome, ivyHome: $ivyHome, " + + s"ivyDefaultOvyUserDir: $ivyDefaultOvyUserDir") + println("-----------------") + // scalastyle:on println runSparkSubmit(args, Some(sparkHome.getCanonicalPath), isSparkTesting = false) } From 66c78e95ac69a7a136db5d8fe2768461a6e45d34 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 19:09:28 +0800 Subject: [PATCH 28/38] try --- .../HiveExternalCatalogVersionsSuite.scala | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 9281d2e2a035e..6bd8611757504 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -163,7 +163,9 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { override def beforeAll(): Unit = { super.beforeAll() - + Utils.deleteRecursively(new File("/home/runner/work/spark/spark/sql/hive/target/tmp/hive-ivy-cache")) + Utils.deleteRecursively(new File("/home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9")) + Utils.deleteRecursively(new File("/home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30")) val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, @@ -216,8 +218,6 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { if (!sparkHome.exists()) { tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) } - val ivyHome = Utils.createTempDir(root = sparkTestingDir.getCanonicalPath, - namePrefix = s"ivy2-$version") // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get val args = Seq( @@ -227,26 +227,11 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveMetastoreVersion", "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", - "--conf", s"${JAR_IVY_REPO_PATH.key}=$ivyHome", "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", "--conf", s"spark.sql.test.version.index=$index", "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath} " + JavaModuleOptions.defaultModuleOptions(), tempPyFile.getCanonicalPath) - // scalastyle:off println - println("-----------------") - val tmpDirName = System.getProperty("java.io.tmpdir") - val ivyCacheDir = System.getProperty("ivy.cache.dir") - val defaultIvyHome = System.getProperty("ivy.home") - val ivyDefaultOvyUserDir = System.getProperty("ivy.default.ivy.user.dir") - Utils.recursiveList(new File(System.getProperty("java.io.tmpdir"))).foreach(f => { - print(f.getCanonicalPath) - }) - println(s"tmpDirName: $tmpDirName, ivyCacheDir: $ivyCacheDir, " + - s"defaultIvyHome: $defaultIvyHome, ivyHome: $ivyHome, " + - s"ivyDefaultOvyUserDir: $ivyDefaultOvyUserDir") - println("-----------------") - // scalastyle:on println runSparkSubmit(args, Some(sparkHome.getCanonicalPath), isSparkTesting = false) } From 1ae55fdf53ac80d06251ccdf025b3db8118adbcd Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 8 Dec 2023 21:20:50 +0800 Subject: [PATCH 29/38] try --- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 6bd8611757504..e7bd6cb09179f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -31,7 +31,7 @@ import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkConf, TestUtils} import org.apache.spark.deploy.SparkSubmitTestUtils -import org.apache.spark.internal.config.{JAR_IVY_REPO_PATH, MASTER_REST_SERVER_ENABLED} +import org.apache.spark.internal.config.MASTER_REST_SERVER_ENABLED import org.apache.spark.internal.config.UI.UI_ENABLED import org.apache.spark.launcher.JavaModuleOptions import org.apache.spark.sql.{QueryTest, Row, SparkSession} From 053c9a74cb65c8fbcfe591440b1801854936bce7 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Mon, 11 Dec 2023 10:15:37 +0800 Subject: [PATCH 30/38] try --- .github/workflows/maven_test.yml | 19 ------------------- .../HiveExternalCatalogVersionsSuite.scala | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 286c3758563c7..12c66f68ad1f0 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -149,16 +149,6 @@ jobs: key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | java${{ matrix.java }}-maven- - - name: List Maven repository some jars - Before - run: | - ls -al /home/runner - find /home/runner -name "*hive-exec*" - find /home/runner -name "*ivy*" - find /home/runner -name "slf4j-api-*" - find /home/runner -name "cache" - find /home/runner -name "jars" - # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v4 with: @@ -190,11 +180,6 @@ jobs: export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} - # Fix some corrupted jars - # ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.apache.hive -DartifactId=hive-exec -Dversion=2.3.9 -Dtransitive=false --quiet - # ./build/mvn dependency:get -DremoteRepositories=https://maven-central.storage-download.googleapis.com/maven2/ -DgroupId=org.slf4j -DartifactId=slf4j-api -Dversion=1.7.30 -Dtransitive=false --quiet - # ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - # ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10 export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pspark-ganglia-lgpl -Djava.version=${JAVA_VERSION/-ea} clean install @@ -210,10 +195,6 @@ jobs: else ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} test -fae fi - - name: List Maven repository some jars - After - run: | - ls -al /home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9 - ls -al /home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30 - name: Clean up local Maven repository run: | rm -rf ~/.m2/repository/org/apache/spark diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index e7bd6cb09179f..87450da6f1e44 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -161,11 +161,21 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { new String(Files.readAllBytes(contentPath), StandardCharsets.UTF_8) } + private def deleteCorruptedJars(): Unit = { + val m2Path = new File(System.getProperty("user.home"), ".m2" + File.separator + "repository") + val hive_exec_2_3_9 = new File(m2Path, "org" + File.separator + + "apache" + File.separator + "hive" + File.separator + + "hive-exec" + File.separator + "2.3.9") + val slf4j_api_1_7_30 = new File(m2Path, "org" + File.separator + + "slf4j" + File.separator + "slf4j-api" + File.separator + "1.7.30") + Utils.deleteRecursively(hive_exec_2_3_9) + Utils.deleteRecursively(slf4j_api_1_7_30) + } + override def beforeAll(): Unit = { super.beforeAll() - Utils.deleteRecursively(new File("/home/runner/work/spark/spark/sql/hive/target/tmp/hive-ivy-cache")) - Utils.deleteRecursively(new File("/home/runner/.m2/repository/org/apache/hive/hive-exec/2.3.9")) - Utils.deleteRecursively(new File("/home/runner/.m2/repository/org/slf4j/slf4j-api/1.7.30")) + deleteCorruptedJars() + val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit Files.write(tempPyFile.toPath, @@ -218,6 +228,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { if (!sparkHome.exists()) { tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) } + // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get val args = Seq( From c79916e3a91751cfa1a0fdaf058ff13a662132fc Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 15 Dec 2023 07:49:06 +0800 Subject: [PATCH 31/38] [SPARK-46302][BUILD] Fix maven daily testing --- .github/workflows/maven_test.yml | 3 +- .../HiveExternalCatalogVersionsSuite.scala | 83 +++++++++---------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 12c66f68ad1f0..5d5a47341f1e0 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -40,7 +40,7 @@ on: description: Additional environment variables to set when running the tests. Should be in JSON format. required: false type: string - default: '{}' + default: '{"SKIP_SPARK_RELEASE_VERSIONS": "spark-3.3.3,spark-3.4.2,spark-3.5.0,master"}' jobs: # Build: build Spark and run the tests for specified modules using maven build: @@ -115,6 +115,7 @@ jobs: HIVE_PROFILE: ${{ matrix.hive }} SPARK_LOCAL_IP: localhost GITHUB_PREV_SHA: ${{ github.event.before }} + SKIP_SPARK_RELEASE_VERSIONS: ${{ fromJSON(inputs.envs).SKIP_SPARK_RELEASE_VERSIONS }} steps: - name: Checkout Spark repository uses: actions/checkout@v4 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 87450da6f1e44..60bd7a6fd97f9 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -66,6 +66,13 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { .map(new File(_)).getOrElse(Utils.createTempDir(namePrefix = "test-spark")) private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) val hiveVersion = HiveUtils.builtinHiveVersion + private val skipReleaseVersions = + sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet + // scalastyle:off println + println("-----------------") + println(s"Skip release versions: $skipReleaseVersions") + println("-----------------") + // scalastyle:on println override def afterAll(): Unit = { try { @@ -161,20 +168,8 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { new String(Files.readAllBytes(contentPath), StandardCharsets.UTF_8) } - private def deleteCorruptedJars(): Unit = { - val m2Path = new File(System.getProperty("user.home"), ".m2" + File.separator + "repository") - val hive_exec_2_3_9 = new File(m2Path, "org" + File.separator + - "apache" + File.separator + "hive" + File.separator + - "hive-exec" + File.separator + "2.3.9") - val slf4j_api_1_7_30 = new File(m2Path, "org" + File.separator + - "slf4j" + File.separator + "slf4j-api" + File.separator + "1.7.30") - Utils.deleteRecursively(hive_exec_2_3_9) - Utils.deleteRecursively(slf4j_api_1_7_30) - } - override def beforeAll(): Unit = { super.beforeAll() - deleteCorruptedJars() val tempPyFile = File.createTempFile("test", ".py") // scalastyle:off line.size.limit @@ -224,45 +219,49 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { } PROCESS_TABLES.testingVersions.zipWithIndex.foreach { case (version, index) => - val sparkHome = new File(sparkTestingDir, s"spark-$version") - if (!sparkHome.exists()) { - tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) + if (!skipReleaseVersions.contains(version)) { + val sparkHome = new File(sparkTestingDir, s"spark-$version") + if (!sparkHome.exists()) { + tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) + } + + // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. + val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get + val args = Seq( + "--name", "prepare testing tables", + "--master", "local[2]", + "--conf", s"${UI_ENABLED.key}=false", + "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", + "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveMetastoreVersion", + "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", + "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", + "--conf", s"spark.sql.test.version.index=$index", + "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath} " + + JavaModuleOptions.defaultModuleOptions(), + tempPyFile.getCanonicalPath) + runSparkSubmit(args, Some(sparkHome.getCanonicalPath), isSparkTesting = false) } + } + + tempPyFile.delete() + } - // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. - val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get + test("backward compatibility") { + if (!skipReleaseVersions.contains("master")) { + assume(PROCESS_TABLES.isPythonVersionAvailable) val args = Seq( - "--name", "prepare testing tables", + "--class", PROCESS_TABLES.getClass.getName.stripSuffix("$"), + "--name", "HiveExternalCatalog backward compatibility test", "--master", "local[2]", "--conf", s"${UI_ENABLED.key}=false", "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", - "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveMetastoreVersion", + "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion", "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", - "--conf", s"spark.sql.test.version.index=$index", - "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath} " + - JavaModuleOptions.defaultModuleOptions(), - tempPyFile.getCanonicalPath) - runSparkSubmit(args, Some(sparkHome.getCanonicalPath), isSparkTesting = false) + "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}", + unusedJar.toString) + if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args) } - - tempPyFile.delete() - } - - test("backward compatibility") { - assume(PROCESS_TABLES.isPythonVersionAvailable) - val args = Seq( - "--class", PROCESS_TABLES.getClass.getName.stripSuffix("$"), - "--name", "HiveExternalCatalog backward compatibility test", - "--master", "local[2]", - "--conf", s"${UI_ENABLED.key}=false", - "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", - "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion", - "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", - "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", - "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}", - unusedJar.toString) - if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args) } } From 2035b9126f9389e2d952b229272b2c6f65f10e56 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 15 Dec 2023 09:12:05 +0800 Subject: [PATCH 32/38] [SPARK-46302][BUILD] Fix maven daily testing --- .github/workflows/maven_test.yml | 2 +- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 5d5a47341f1e0..b933b150d0570 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -40,7 +40,7 @@ on: description: Additional environment variables to set when running the tests. Should be in JSON format. required: false type: string - default: '{"SKIP_SPARK_RELEASE_VERSIONS": "spark-3.3.3,spark-3.4.2,spark-3.5.0,master"}' + default: '{"SKIP_SPARK_RELEASE_VERSIONS": "3.3.3,3.4.2,3.5.0,master"}' jobs: # Build: build Spark and run the tests for specified modules using maven build: diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 60bd7a6fd97f9..7235011902c08 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -219,6 +219,11 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { } PROCESS_TABLES.testingVersions.zipWithIndex.foreach { case (version, index) => + // scalastyle:off println + println("-----------------") + println(s"Release versions: $version") + println("-----------------") + // scalastyle:on println if (!skipReleaseVersions.contains(version)) { val sparkHome = new File(sparkTestingDir, s"spark-$version") if (!sparkHome.exists()) { From feb39321c5ad589d1f70f4a09bce463d35ee3e93 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 19 Dec 2023 14:14:57 +0800 Subject: [PATCH 33/38] [SPARK-46302][BUILD] Fix maven daily testing --- .../HiveExternalCatalogVersionsSuite.scala | 81 ++++++++----------- 1 file changed, 34 insertions(+), 47 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 7235011902c08..f774fd0e66bb3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -66,13 +66,6 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { .map(new File(_)).getOrElse(Utils.createTempDir(namePrefix = "test-spark")) private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) val hiveVersion = HiveUtils.builtinHiveVersion - private val skipReleaseVersions = - sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet - // scalastyle:off println - println("-----------------") - println(s"Skip release versions: $skipReleaseVersions") - println("-----------------") - // scalastyle:on println override def afterAll(): Unit = { try { @@ -219,58 +212,51 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { } PROCESS_TABLES.testingVersions.zipWithIndex.foreach { case (version, index) => - // scalastyle:off println - println("-----------------") - println(s"Release versions: $version") - println("-----------------") - // scalastyle:on println - if (!skipReleaseVersions.contains(version)) { - val sparkHome = new File(sparkTestingDir, s"spark-$version") - if (!sparkHome.exists()) { - tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) - } - - // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. - val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get - val args = Seq( - "--name", "prepare testing tables", - "--master", "local[2]", - "--conf", s"${UI_ENABLED.key}=false", - "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", - "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveMetastoreVersion", - "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", - "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", - "--conf", s"spark.sql.test.version.index=$index", - "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath} " + - JavaModuleOptions.defaultModuleOptions(), - tempPyFile.getCanonicalPath) - runSparkSubmit(args, Some(sparkHome.getCanonicalPath), isSparkTesting = false) + val sparkHome = new File(sparkTestingDir, s"spark-$version") + if (!sparkHome.exists()) { + tryDownloadSpark(version, sparkTestingDir.getCanonicalPath) } - } - - tempPyFile.delete() - } - test("backward compatibility") { - if (!skipReleaseVersions.contains("master")) { - assume(PROCESS_TABLES.isPythonVersionAvailable) + // Extract major.minor for testing Spark 3.1.x and 3.0.x with metastore 2.3.9 and Java 11. + val hiveMetastoreVersion = """^\d+\.\d+""".r.findFirstIn(hiveVersion).get val args = Seq( - "--class", PROCESS_TABLES.getClass.getName.stripSuffix("$"), - "--name", "HiveExternalCatalog backward compatibility test", + "--name", "prepare testing tables", "--master", "local[2]", "--conf", s"${UI_ENABLED.key}=false", "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", - "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion", + "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveMetastoreVersion", "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", - "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}", - unusedJar.toString) - if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args) + "--conf", s"spark.sql.test.version.index=$index", + "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath} " + + JavaModuleOptions.defaultModuleOptions(), + tempPyFile.getCanonicalPath) + runSparkSubmit(args, Some(sparkHome.getCanonicalPath), isSparkTesting = false) } + + tempPyFile.delete() + } + + test("backward compatibility") { + assume(PROCESS_TABLES.isPythonVersionAvailable) + val args = Seq( + "--class", PROCESS_TABLES.getClass.getName.stripSuffix("$"), + "--name", "HiveExternalCatalog backward compatibility test", + "--master", "local[2]", + "--conf", s"${UI_ENABLED.key}=false", + "--conf", s"${MASTER_REST_SERVER_ENABLED.key}=false", + "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=$hiveVersion", + "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", + "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", + "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}", + unusedJar.toString) + if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args) } } object PROCESS_TABLES extends QueryTest with SQLTestUtils { + private val skipReleaseVersions = + sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet val isPythonVersionAvailable = TestUtils.isPythonVersionAvailable val releaseMirror = sys.env.getOrElse("SPARK_RELEASE_MIRROR", "https://dist.apache.org/repos/dist/release") @@ -285,7 +271,8 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils { .filter(_.contains("""""".r.findFirstMatchIn(_).get.group(1)) - .filter(_ < org.apache.spark.SPARK_VERSION).toImmutableArraySeq + .filter(_ < org.apache.spark.SPARK_VERSION) + .filterNot(skipReleaseVersions.contains).toImmutableArraySeq } catch { // Do not throw exception during object initialization. case NonFatal(_) => Nil From ab066136dd78167b523529f1031c5dc1ea5bf515 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 19 Dec 2023 14:59:58 +0800 Subject: [PATCH 34/38] [SPARK-46302][BUILD] Fix maven daily testing --- .github/workflows/build_maven.yml | 5 +++++ .github/workflows/maven_test.yml | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_maven.yml b/.github/workflows/build_maven.yml index 7a01c136fce92..fc754a837085e 100644 --- a/.github/workflows/build_maven.yml +++ b/.github/workflows/build_maven.yml @@ -30,3 +30,8 @@ jobs: name: Run uses: ./.github/workflows/maven_test.yml if: github.repository == 'apache/spark' + with: + envs: >- + { + "SKIP_SPARK_RELEASE_VERSIONS": "3.3.3,3.4.2,3.5.0" + } diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index b933b150d0570..12c66f68ad1f0 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -40,7 +40,7 @@ on: description: Additional environment variables to set when running the tests. Should be in JSON format. required: false type: string - default: '{"SKIP_SPARK_RELEASE_VERSIONS": "3.3.3,3.4.2,3.5.0,master"}' + default: '{}' jobs: # Build: build Spark and run the tests for specified modules using maven build: @@ -115,7 +115,6 @@ jobs: HIVE_PROFILE: ${{ matrix.hive }} SPARK_LOCAL_IP: localhost GITHUB_PREV_SHA: ${{ github.event.before }} - SKIP_SPARK_RELEASE_VERSIONS: ${{ fromJSON(inputs.envs).SKIP_SPARK_RELEASE_VERSIONS }} steps: - name: Checkout Spark repository uses: actions/checkout@v4 From aa377912bfaa65648e2cd055476dce52b82176dd Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 19 Dec 2023 15:04:38 +0800 Subject: [PATCH 35/38] [SPARK-46302][BUILD] Fix maven daily testing --- .github/workflows/build_maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_maven.yml b/.github/workflows/build_maven.yml index fc754a837085e..d43366fa86e77 100644 --- a/.github/workflows/build_maven.yml +++ b/.github/workflows/build_maven.yml @@ -33,5 +33,5 @@ jobs: with: envs: >- { - "SKIP_SPARK_RELEASE_VERSIONS": "3.3.3,3.4.2,3.5.0" + "SKIP_SPARK_RELEASE_VERSIONS": "3.3.4,3.4.2,3.5.0" } From 9eb03f80db5355ae495e464b84dd084f29d9bfb8 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Tue, 19 Dec 2023 18:58:45 +0800 Subject: [PATCH 36/38] [SPARK-46302][TESTS] Skip maven daily testing as ivy uses some corrupted cache jar files --- .github/workflows/build_and_test.yml | 5 +++++ .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 3 +++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 270ec8cb6975f..a6f05ca33b4b6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1104,3 +1104,8 @@ jobs: permissions: packages: write uses: ./.github/workflows/maven_test.yml + with: + envs: >- + { + "SKIP_SPARK_RELEASE_VERSIONS": "3.3.4,3.4.2,3.5.0" + } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 49d50ef622e62..75ad76d42c80a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -244,11 +244,14 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}", unusedJar.toString) + // TODO After SPARK-46400, we need to run the following logic if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args) } } object PROCESS_TABLES extends QueryTest with SQLTestUtils { + // TODO After SPARK-46400 and applying it to the already released Spark version, + // we can remove the following logic private val skipReleaseVersions = sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet val isPythonVersionAvailable = TestUtils.isPythonVersionAvailable From a70bf610e821a95f7ab84fb590debc1ded92bfd2 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 20 Dec 2023 08:31:12 +0800 Subject: [PATCH 37/38] [SPARK-46302][TESTS] Skip maven daily testing as ivy uses some corrupted cache jar files --- .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 75ad76d42c80a..50cf4017bd1e0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -244,14 +244,15 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { "--conf", s"${WAREHOUSE_PATH.key}=${wareHousePath.getCanonicalPath}", "--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}", unusedJar.toString) - // TODO After SPARK-46400, we need to run the following logic if (PROCESS_TABLES.testingVersions.nonEmpty) runSparkSubmit(args) } } object PROCESS_TABLES extends QueryTest with SQLTestUtils { - // TODO After SPARK-46400 and applying it to the already released Spark version, - // we can remove the following logic + // TODO In SPARK-46302, the env SKIP_SPARK_RELEASE_VERSIONS has been added to + // allow Maven tests to skip problematic release versions. + // Related issues will be fixed in SPARK-46400, and testing will be resumed + // after the fixed Spark 3.x version is released. private val skipReleaseVersions = sys.env.getOrElse("SKIP_SPARK_RELEASE_VERSIONS", "").split(",").toSet val isPythonVersionAvailable = TestUtils.isPythonVersionAvailable From 941ab8e3118d765033713ffa846ab4134570ca1e Mon Sep 17 00:00:00 2001 From: panbingkun Date: Wed, 20 Dec 2023 13:48:19 +0800 Subject: [PATCH 38/38] [SPARK-46302][TESTS] Skip maven daily testing as ivy uses some corrupted cache jar files --- .github/workflows/build_and_test.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a6f05ca33b4b6..bdcb1dd1ea5ce 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1098,14 +1098,3 @@ jobs: cd ui-test npm install --save-dev node --experimental-vm-modules node_modules/.bin/jest - - maven-test: - name: "Run Maven Test" - permissions: - packages: write - uses: ./.github/workflows/maven_test.yml - with: - envs: >- - { - "SKIP_SPARK_RELEASE_VERSIONS": "3.3.4,3.4.2,3.5.0" - }