From 799364ebbe45aaff02fc744c8565db7bb5fb862f Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 25 Feb 2021 22:59:42 +0800 Subject: [PATCH 1/6] Upgrade Parquet to 1.12.0 --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 12 ++++++------ dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 12 ++++++------ pom.xml | 15 +++++++++++++-- project/SparkBuild.scala | 3 +++ 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 228bb94d22920..e207cb90a250c 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -203,12 +203,12 @@ orc-shims/1.6.7//orc-shims-1.6.7.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar -parquet-column/1.11.1//parquet-column-1.11.1.jar -parquet-common/1.11.1//parquet-common-1.11.1.jar -parquet-encoding/1.11.1//parquet-encoding-1.11.1.jar -parquet-format-structures/1.11.1//parquet-format-structures-1.11.1.jar -parquet-hadoop/1.11.1//parquet-hadoop-1.11.1.jar -parquet-jackson/1.11.1//parquet-jackson-1.11.1.jar +parquet-column/1.12.0//parquet-column-1.12.0.jar +parquet-common/1.12.0//parquet-common-1.12.0.jar +parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar +parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar +parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar +parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9.1//py4j-0.10.9.1.jar pyrolite/4.30//pyrolite-4.30.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 71ef4c1ef998e..4d9458ba222d3 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -173,12 +173,12 @@ orc-shims/1.6.7//orc-shims-1.6.7.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar -parquet-column/1.11.1//parquet-column-1.11.1.jar -parquet-common/1.11.1//parquet-common-1.11.1.jar -parquet-encoding/1.11.1//parquet-encoding-1.11.1.jar -parquet-format-structures/1.11.1//parquet-format-structures-1.11.1.jar -parquet-hadoop/1.11.1//parquet-hadoop-1.11.1.jar -parquet-jackson/1.11.1//parquet-jackson-1.11.1.jar +parquet-column/1.12.0//parquet-column-1.12.0.jar +parquet-common/1.12.0//parquet-common-1.12.0.jar +parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar +parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar +parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar +parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9.1//py4j-0.10.9.1.jar pyrolite/4.30//pyrolite-4.30.jar diff --git a/pom.xml b/pom.xml index d497ed176bd35..0d03533902c65 100644 --- a/pom.xml +++ b/pom.xml @@ -136,7 +136,7 @@ 2.6.0 10.14.2.0 - 1.11.1 + 1.12.0 1.6.7 9.4.36.v20210114 4.0.3 @@ -291,6 +291,17 @@ false + + staging + apache-staging-releases + https://repository.apache.org/content/repositories/staging/ + + true + + + true + + @@ -2061,7 +2072,7 @@ ${hive.group} hive-service-rpc - + org.apache.parquet parquet-hadoop-bundle diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 470d2c77ea0aa..05a27b7cb42ac 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -271,6 +271,9 @@ object SparkBuild extends PomBuild { DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) + ) ++ Seq( + "apache-staging-releases" at "https://repository.apache.org/content/repositories/staging/", + Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value, From 741eb211d81554dc372f61cdf96e41bb893baebb Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 26 Feb 2021 22:36:15 +0800 Subject: [PATCH 2/6] git commit --allow-empty -m "Trigger GithubAction" From e7b14c8b6d2de28eef9c27d6944aaaefa85cdc16 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 27 Feb 2021 09:02:30 +0800 Subject: [PATCH 3/6] Update StatisticsSuite.scala --- .../test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala index b4de82f5d9e51..8afdd50b0c96b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala @@ -1502,7 +1502,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto Seq(tbl, ext_tbl).foreach { tblName => sql(s"INSERT INTO $tblName VALUES (1, 'a', '2019-12-13')") - val expectedSize = 651 + val expectedSize = 657 // analyze table sql(s"ANALYZE TABLE $tblName COMPUTE STATISTICS NOSCAN") var tableStats = getTableStats(tblName) From cab8d7ebdd1d3a0e158f335c646ac36d4d9f0679 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 19 Mar 2021 10:39:01 +0800 Subject: [PATCH 4/6] Avro to 1.10.2 and jackson to 2.12.2 --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 21 ++++++++++----------- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 21 ++++++++++----------- pom.xml | 4 ++-- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 59e40a98d6611..08100a7c20a36 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -22,9 +22,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar arrow-vector/2.0.0//arrow-vector-2.0.0.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar automaton/1.11-8//automaton-1.11-8.jar -avro-ipc/1.10.1//avro-ipc-1.10.1.jar -avro-mapred/1.10.1//avro-mapred-1.10.1.jar -avro/1.10.1//avro-1.10.1.jar +avro-ipc/1.10.2//avro-ipc-1.10.2.jar +avro-mapred/1.10.2//avro-mapred-1.10.2.jar +avro/1.10.2//avro-1.10.2.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar breeze_2.12/1.0//breeze_2.12-1.0.jar @@ -104,17 +104,16 @@ httpclient/4.5.13//httpclient-4.5.13.jar httpcore/4.4.12//httpcore-4.4.12.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.4.0//ivy-2.4.0.jar -jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar +jackson-annotations/2.12.2//jackson-annotations-2.12.2.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar -jackson-core/2.11.4//jackson-core-2.11.4.jar -jackson-databind/2.11.4//jackson-databind-2.11.4.jar -jackson-dataformat-yaml/2.11.4//jackson-dataformat-yaml-2.11.4.jar +jackson-core/2.12.2//jackson-core-2.12.2.jar +jackson-databind/2.12.2//jackson-databind-2.12.2.jar +jackson-dataformat-yaml/2.12.2//jackson-dataformat-yaml-2.12.2.jar jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations/2.11.4//jackson-module-jaxb-annotations-2.11.4.jar -jackson-module-paranamer/2.11.4//jackson-module-paranamer-2.11.4.jar -jackson-module-scala_2.12/2.11.4//jackson-module-scala_2.12-2.11.4.jar +jackson-module-jaxb-annotations/2.12.2//jackson-module-jaxb-annotations-2.12.2.jar +jackson-module-scala_2.12/2.12.2//jackson-module-scala_2.12-2.12.2.jar jackson-xc/1.9.13//jackson-xc-1.9.13.jar jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar @@ -222,7 +221,7 @@ shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.9.0//shims-0.9.0.jar slf4j-api/1.7.30//slf4j-api-1.7.30.jar slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar -snakeyaml/1.26//snakeyaml-1.26.jar +snakeyaml/1.27//snakeyaml-1.27.jar snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 45ef2259bd6d0..b9466d261de43 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -17,9 +17,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar arrow-vector/2.0.0//arrow-vector-2.0.0.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar automaton/1.11-8//automaton-1.11-8.jar -avro-ipc/1.10.1//avro-ipc-1.10.1.jar -avro-mapred/1.10.1//avro-mapred-1.10.1.jar -avro/1.10.1//avro-1.10.1.jar +avro-ipc/1.10.2//avro-ipc-1.10.2.jar +avro-mapred/1.10.2//avro-mapred-1.10.2.jar +avro/1.10.2//avro-1.10.2.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar breeze_2.12/1.0//breeze_2.12-1.0.jar @@ -82,16 +82,15 @@ httpclient/4.5.13//httpclient-4.5.13.jar httpcore/4.4.12//httpcore-4.4.12.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.4.0//ivy-2.4.0.jar -jackson-annotations/2.11.4//jackson-annotations-2.11.4.jar +jackson-annotations/2.12.2//jackson-annotations-2.12.2.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar -jackson-core/2.11.4//jackson-core-2.11.4.jar -jackson-databind/2.11.4//jackson-databind-2.11.4.jar -jackson-dataformat-yaml/2.11.4//jackson-dataformat-yaml-2.11.4.jar +jackson-core/2.12.2//jackson-core-2.12.2.jar +jackson-databind/2.12.2//jackson-databind-2.12.2.jar +jackson-dataformat-yaml/2.12.2//jackson-dataformat-yaml-2.12.2.jar jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations/2.11.4//jackson-module-jaxb-annotations-2.11.4.jar -jackson-module-paranamer/2.11.4//jackson-module-paranamer-2.11.4.jar -jackson-module-scala_2.12/2.11.4//jackson-module-scala_2.12-2.11.4.jar +jackson-module-jaxb-annotations/2.12.2//jackson-module-jaxb-annotations-2.12.2.jar +jackson-module-scala_2.12/2.12.2//jackson-module-scala_2.12-2.12.2.jar jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar @@ -193,7 +192,7 @@ shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.9.0//shims-0.9.0.jar slf4j-api/1.7.30//slf4j-api-1.7.30.jar slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar -snakeyaml/1.26//snakeyaml-1.26.jar +snakeyaml/1.27//snakeyaml-1.27.jar snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar diff --git a/pom.xml b/pom.xml index 80a4ea6a702f7..396c1728f04c3 100644 --- a/pom.xml +++ b/pom.xml @@ -148,7 +148,7 @@ the link to metrics.dropwizard.io in docs/monitoring.md. --> 4.1.1 - 1.10.1 + 1.10.2 1.14.0 1.11.844 @@ -169,7 +169,7 @@ true 1.9.13 - 2.11.4 + 2.12.2 1.1.8.2 1.1.2 1.15 From 0ff51144350241f7bb62ef97c3391f80741f3b8c Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 20 Mar 2021 22:49:38 +0800 Subject: [PATCH 5/6] fix --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 774a62dad2d19..195aee1670548 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -661,7 +661,7 @@ object DependencyOverrides { dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0", dependencyOverrides += "jline" % "jline" % "2.14.6", - dependencyOverrides += "org.apache.avro" % "avro" % "1.10.1") + dependencyOverrides += "org.apache.avro" % "avro" % "1.10.2") } /** From 8b58e29adcbad1bf7cdeb416cd1e22928d49c892 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 26 Mar 2021 08:49:44 +0800 Subject: [PATCH 6/6] fix --- pom.xml | 11 ----------- project/SparkBuild.scala | 3 --- 2 files changed, 14 deletions(-) diff --git a/pom.xml b/pom.xml index 568e2acf2b973..2aca8c7656f76 100644 --- a/pom.xml +++ b/pom.xml @@ -297,17 +297,6 @@ false - - staging - apache-staging-releases - https://repository.apache.org/content/repositories/staging/ - - true - - - true - - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 195aee1670548..a9d38d232f2af 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -274,9 +274,6 @@ object SparkBuild extends PomBuild { DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) - ) ++ Seq( - "apache-staging-releases" at "https://repository.apache.org/content/repositories/staging/", - Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,