Merge branch 'upmaster' into SPARK-33673

Author: LuciferYang

.github/workflows/build_and_test.yml

@@ -335,7 +335,7 @@ jobs:
         r-version: 4.0
     - name: Install R linter dependencies and SparkR
       run: |
-        sudo apt-get install -y libcurl4-openssl-dev
+        sudo apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev
         sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
         sudo Rscript -e "devtools::install_github('jimhester/[email protected]')"
         ./R/install-dev.sh
@@ -414,7 +414,7 @@ jobs:
     - name: Build with SBT
       run: |
         ./dev/change-scala-version.sh 2.13
-        ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pscala-2.13 compile test:compile
+        ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pdocker-integration-tests -Pkubernetes-integration-tests -Pscala-2.13 compile test:compile
 
   hadoop-2:
     name: Hadoop 2 build with SBT

.github/workflows/publish_snapshot.yml

@@ -7,9 +7,17 @@ on:
 jobs:
   publish-snapshot:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+          - branch-3.1
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@master
+      with:
+        ref: ${{ matrix.branch }}
     - name: Cache Maven local repository
       uses: actions/cache@v2
       with:
@@ -27,4 +35,5 @@ jobs:
         ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
         GPG_KEY: "not_used"
         GPG_PASSPHRASE: "not_used"
+        GIT_REF: ${{ matrix.branch }}
       run: ./dev/create-release/release-build.sh publish-snapshot

LICENSE-binary

@@ -521,7 +521,6 @@ Common Development and Distribution License (CDDL) 1.1
 ------------------------------------------------------
 
 javax.el:javax.el-api	https://javaee.github.io/uel-ri/
-javax.servlet:javax.servlet-api   https://javaee.github.io/servlet-spec/
 javax.servlet.jsp:jsp-api
 javax.transaction:jta http://www.oracle.com/technetwork/java/index.html
 javax.xml.bind:jaxb-api    https://github.com/javaee/jaxb-v2
@@ -553,6 +552,7 @@ Eclipse Public License (EPL) 2.0
 --------------------------------
 
 jakarta.annotation:jakarta-annotation-api https://projects.eclipse.org/projects/ee4j.ca
+jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet
 jakarta.ws.rs:jakarta.ws.rs-api https://github.com/eclipse-ee4j/jaxrs-api
 org.glassfish.hk2.external:jakarta.inject
 

core/pom.xml

@@ -161,9 +161,9 @@
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>javax.servlet</groupId>
-      <artifactId>javax.servlet-api</artifactId>
-      <version>${javaxservlet.version}</version>
+      <groupId>jakarta.servlet</groupId>
+      <artifactId>jakarta.servlet-api</artifactId>
+      <version>${jakartaservlet.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>

core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala

@@ -44,7 +44,7 @@ object PythonRunner {
       .orElse(sparkConf.get(PYSPARK_PYTHON))
       .orElse(sys.env.get("PYSPARK_DRIVER_PYTHON"))
       .orElse(sys.env.get("PYSPARK_PYTHON"))
-      .getOrElse("python")
+      .getOrElse("python3")
 
     // Format python file paths before adding them to the PYTHONPATH
     val formattedPythonFile = formatPath(pythonFile)

core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala

@@ -284,9 +284,9 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
 
     val pendingOrSkippedTableId =
       if (isComplete) {
-        "pending"
-      } else {
         "skipped"
+      } else {
+        "pending"
       }
 
     val activeStagesTable =

dev/deps/spark-deps-hadoop-2.7-hive-2.3

@@ -4,8 +4,9 @@ JTransforms/3.1//JTransforms-3.1.jar
 RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar
 ST4/4.0.4//ST4-4.0.4.jar
 activation/1.1.1//activation-1.1.1.jar
-aircompressor/0.10//aircompressor-0.10.jar
+aircompressor/0.16//aircompressor-0.16.jar
 algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar
+annotations/17.0.0//annotations-17.0.0.jar
 antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
 antlr4-runtime/4.8-1//antlr4-runtime-4.8-1.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
@@ -32,7 +33,7 @@ chill-java/0.9.5//chill-java-0.9.5.jar
 chill_2.12/0.9.5//chill_2.12-0.9.5.jar
 commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.10//commons-codec-1.10.jar
+commons-codec/1.15//commons-codec-1.15.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.20//commons-compress-1.20.jar
@@ -103,29 +104,29 @@ httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.12//httpcore-4.4.12.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.4.0//ivy-2.4.0.jar
-jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar
+jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.10.0//jackson-core-2.10.0.jar
-jackson-databind/2.10.0//jackson-databind-2.10.0.jar
-jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-core/2.11.4//jackson-core-2.11.4.jar
+jackson-databind/2.11.4//jackson-databind-2.11.4.jar
+jackson-dataformat-yaml/2.11.4//jackson-dataformat-yaml-2.11.4.jar
 jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
-jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar
-jackson-module-scala_2.12/2.10.0//jackson-module-scala_2.12-2.10.0.jar
+jackson-module-jaxb-annotations/2.11.4//jackson-module-jaxb-annotations-2.11.4.jar
+jackson-module-paranamer/2.11.4//jackson-module-paranamer-2.11.4.jar
+jackson-module-scala_2.12/2.11.4//jackson-module-scala_2.12-2.11.4.jar
 jackson-xc/1.9.13//jackson-xc-1.9.13.jar
 jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
+jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
 janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.2//jaxb-api-2.2.2.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
@@ -195,9 +196,9 @@ objenesis/2.6//objenesis-2.6.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.12//orc-core-1.5.12.jar
-orc-mapreduce/1.5.12//orc-mapreduce-1.5.12.jar
-orc-shims/1.5.12//orc-shims-1.5.12.jar
+orc-core/1.6.6//orc-core-1.6.6.jar
+orc-mapreduce/1.6.6//orc-mapreduce-1.6.6.jar
+orc-shims/1.6.6//orc-shims-1.6.6.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
@@ -220,8 +221,8 @@ shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.9.0//shims-0.9.0.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
-snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.8//snappy-java-1.1.8.jar
+snakeyaml/1.26//snakeyaml-1.26.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -5,8 +5,9 @@ RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar
 ST4/4.0.4//ST4-4.0.4.jar
 accessors-smart/1.2//accessors-smart-1.2.jar
 activation/1.1.1//activation-1.1.1.jar
-aircompressor/0.10//aircompressor-0.10.jar
+aircompressor/0.16//aircompressor-0.16.jar
 algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar
+annotations/17.0.0//annotations-17.0.0.jar
 antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
 antlr4-runtime/4.8-1//antlr4-runtime-4.8-1.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
@@ -29,7 +30,7 @@ chill-java/0.9.5//chill-java-0.9.5.jar
 chill_2.12/0.9.5//chill_2.12-0.9.5.jar
 commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.10//commons-codec-1.10.jar
+commons-codec/1.15//commons-codec-1.15.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.20//commons-compress-1.20.jar
@@ -102,21 +103,22 @@ httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.12//httpcore-4.4.12.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.4.0//ivy-2.4.0.jar
-jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar
+jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.10.0//jackson-core-2.10.0.jar
-jackson-databind/2.10.0//jackson-databind-2.10.0.jar
-jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-core/2.11.4//jackson-core-2.11.4.jar
+jackson-databind/2.11.4//jackson-databind-2.11.4.jar
+jackson-dataformat-yaml/2.11.4//jackson-dataformat-yaml-2.11.4.jar
 jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
 jackson-jaxrs-base/2.9.5//jackson-jaxrs-base-2.9.5.jar
 jackson-jaxrs-json-provider/2.9.5//jackson-jaxrs-json-provider-2.9.5.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
-jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar
-jackson-module-scala_2.12/2.10.0//jackson-module-scala_2.12-2.10.0.jar
+jackson-module-jaxb-annotations/2.11.4//jackson-module-jaxb-annotations-2.11.4.jar
+jackson-module-paranamer/2.11.4//jackson-module-paranamer-2.11.4.jar
+jackson-module-scala_2.12/2.11.4//jackson-module-scala_2.12-2.11.4.jar
 jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
+jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
@@ -209,9 +211,9 @@ okhttp/2.7.5//okhttp-2.7.5.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.12//orc-core-1.5.12.jar
-orc-mapreduce/1.5.12//orc-mapreduce-1.5.12.jar
-orc-shims/1.5.12//orc-shims-1.5.12.jar
+orc-core/1.6.6//orc-core-1.6.6.jar
+orc-mapreduce/1.6.6//orc-mapreduce-1.6.6.jar
+orc-shims/1.6.6//orc-shims-1.6.6.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
@@ -235,8 +237,8 @@ shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.9.0//shims-0.9.0.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
-snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.8//snappy-java-1.1.8.jar
+snakeyaml/1.26//snakeyaml-1.26.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar

dev/sparktestsupport/modules.py

@@ -31,9 +31,10 @@ class Module(object):
     files have changed.
     """
 
-    def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(), environ={},
-                 sbt_test_goals=(), python_test_goals=(), excluded_python_implementations=(),
-                 test_tags=(), should_run_r_tests=False, should_run_build_tests=False):
+    def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(),
+                 environ=None, sbt_test_goals=(), python_test_goals=(),
+                 excluded_python_implementations=(), test_tags=(), should_run_r_tests=False,
+                 should_run_build_tests=False):
         """
         Define a new module.
 
@@ -62,7 +63,7 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=
         self.source_file_prefixes = source_file_regexes
         self.sbt_test_goals = sbt_test_goals
         self.build_profile_flags = build_profile_flags
-        self.environ = environ
+        self.environ = environ or {}
         self.python_test_goals = python_test_goals
         self.excluded_python_implementations = excluded_python_implementations
         self.test_tags = test_tags

dev/tox.ini

@@ -16,9 +16,9 @@
 [pycodestyle]
 ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
 max-line-length=100
-exclude=python/pyspark/cloudpickle/*.py,shared.py,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
+exclude=*/target/*,python/pyspark/cloudpickle/*.py,shared.py,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
 
 [flake8]
-select = E901,E999,F821,F822,F823,F401,F405
-exclude = python/pyspark/cloudpickle/*.py,shared.py*,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*,python/out,python/pyspark/sql/pandas/functions.pyi,python/pyspark/sql/column.pyi,python/pyspark/worker.pyi,python/pyspark/java_gateway.pyi
+select = E901,E999,F821,F822,F823,F401,F405,B006
+exclude = */target/*,python/pyspark/cloudpickle/*.py,shared.py*,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*,python/out,python/pyspark/sql/pandas/functions.pyi,python/pyspark/sql/column.pyi,python/pyspark/worker.pyi,python/pyspark/java_gateway.pyi
 max-line-length = 100