From 839c27e0d7660c787fcd98155606db07ad7c2e09 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 10 Aug 2019 18:41:12 +0800 Subject: [PATCH 1/4] Upgrade Scala to 2.12.9 --- dev/deps/spark-deps-hadoop-2.7 | 6 +++--- dev/deps/spark-deps-hadoop-3.2 | 6 +++--- docs/_config.yml | 2 +- external/docker/spark-test/base/Dockerfile | 2 +- pom.xml | 2 +- .../org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 82389ea6aa58a..ff186c50ae3e4 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -172,10 +172,10 @@ parquet-jackson-1.10.1.jar protobuf-java-2.5.0.jar py4j-0.10.8.1.jar pyrolite-4.30.jar -scala-compiler-2.12.8.jar -scala-library-2.12.8.jar +scala-compiler-2.12.9.jar +scala-library-2.12.9.jar scala-parser-combinators_2.12-1.1.0.jar -scala-reflect-2.12.8.jar +scala-reflect-2.12.9.jar scala-xml_2.12-1.2.0.jar shapeless_2.12-2.3.2.jar shims-0.7.45.jar diff --git a/dev/deps/spark-deps-hadoop-3.2 b/dev/deps/spark-deps-hadoop-3.2 index a02735d2be893..488be8793d673 100644 --- a/dev/deps/spark-deps-hadoop-3.2 +++ b/dev/deps/spark-deps-hadoop-3.2 @@ -191,10 +191,10 @@ protobuf-java-2.5.0.jar py4j-0.10.8.1.jar pyrolite-4.30.jar re2j-1.1.jar -scala-compiler-2.12.8.jar -scala-library-2.12.8.jar +scala-compiler-2.12.9.jar +scala-library-2.12.9.jar scala-parser-combinators_2.12-1.1.0.jar -scala-reflect-2.12.8.jar +scala-reflect-2.12.9.jar scala-xml_2.12-1.2.0.jar shapeless_2.12-2.3.2.jar shims-0.7.45.jar diff --git a/docs/_config.yml b/docs/_config.yml index 146c90fcff6e5..650b71bb0d4ac 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -17,7 +17,7 @@ include: SPARK_VERSION: 3.0.0-SNAPSHOT SPARK_VERSION_SHORT: 3.0.0 SCALA_BINARY_VERSION: "2.12" -SCALA_VERSION: "2.12.8" +SCALA_VERSION: "2.12.9" MESOS_VERSION: 1.0.0 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK SPARK_GITHUB_URL: https://github.com/apache/spark diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile index c1fd630d0b665..fc8cba64f3942 100644 --- a/external/docker/spark-test/base/Dockerfile +++ b/external/docker/spark-test/base/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && \ apt-get install -y less openjdk-8-jre-headless iproute2 vim-tiny sudo openssh-server && \ rm -rf /var/lib/apt/lists/* -ENV SCALA_VERSION 2.12.8 +ENV SCALA_VERSION 2.12.9 ENV CDH_VERSION cdh4 ENV SCALA_HOME /opt/scala-$SCALA_VERSION ENV SPARK_HOME /opt/spark diff --git a/pom.xml b/pom.xml index cee85515cb993..3f94dc4df983c 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ 3.4.1 3.2.2 - 2.12.8 + 2.12.9 2.12 --diff --test diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 2a4e84241a941..51a1097549bdd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -72,7 +72,7 @@ object ScalaReflection extends ScalaReflection { /** * Synchronize to prevent concurrent usage of `<:<` operator. * This operator is not thread safe in any current version of scala; i.e. - * (2.11.12, 2.12.8, 2.13.0-M5). + * (2.11.12, 2.12.9, 2.13.0-M5). * * See https://github.com/scala/bug/issues/10766 */ From e5811fdb89e63f889cac3c3663e397efb84e901e Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Wed, 11 Sep 2019 12:58:46 +0800 Subject: [PATCH 2/4] Upgrade to 2.12.10 --- dev/deps/spark-deps-hadoop-2.7 | 6 +++--- dev/deps/spark-deps-hadoop-3.2 | 6 +++--- docs/_config.yml | 2 +- external/docker/spark-test/base/Dockerfile | 2 +- pom.xml | 2 +- .../org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index ff186c50ae3e4..cbf64e69db2d4 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -172,10 +172,10 @@ parquet-jackson-1.10.1.jar protobuf-java-2.5.0.jar py4j-0.10.8.1.jar pyrolite-4.30.jar -scala-compiler-2.12.9.jar -scala-library-2.12.9.jar +scala-compiler-2.12.10.jar +scala-library-2.12.10.jar scala-parser-combinators_2.12-1.1.0.jar -scala-reflect-2.12.9.jar +scala-reflect-2.12.10.jar scala-xml_2.12-1.2.0.jar shapeless_2.12-2.3.2.jar shims-0.7.45.jar diff --git a/dev/deps/spark-deps-hadoop-3.2 b/dev/deps/spark-deps-hadoop-3.2 index 488be8793d673..f78714d3da947 100644 --- a/dev/deps/spark-deps-hadoop-3.2 +++ b/dev/deps/spark-deps-hadoop-3.2 @@ -191,10 +191,10 @@ protobuf-java-2.5.0.jar py4j-0.10.8.1.jar pyrolite-4.30.jar re2j-1.1.jar -scala-compiler-2.12.9.jar -scala-library-2.12.9.jar +scala-compiler-2.12.10.jar +scala-library-2.12.10.jar scala-parser-combinators_2.12-1.1.0.jar -scala-reflect-2.12.9.jar +scala-reflect-2.12.10.jar scala-xml_2.12-1.2.0.jar shapeless_2.12-2.3.2.jar shims-0.7.45.jar diff --git a/docs/_config.yml b/docs/_config.yml index 650b71bb0d4ac..57b8d716ee55c 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -17,7 +17,7 @@ include: SPARK_VERSION: 3.0.0-SNAPSHOT SPARK_VERSION_SHORT: 3.0.0 SCALA_BINARY_VERSION: "2.12" -SCALA_VERSION: "2.12.9" +SCALA_VERSION: "2.12.10" MESOS_VERSION: 1.0.0 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK SPARK_GITHUB_URL: https://github.com/apache/spark diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile index fc8cba64f3942..5bec5d3f16548 100644 --- a/external/docker/spark-test/base/Dockerfile +++ b/external/docker/spark-test/base/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && \ apt-get install -y less openjdk-8-jre-headless iproute2 vim-tiny sudo openssh-server && \ rm -rf /var/lib/apt/lists/* -ENV SCALA_VERSION 2.12.9 +ENV SCALA_VERSION 2.12.10 ENV CDH_VERSION cdh4 ENV SCALA_HOME /opt/scala-$SCALA_VERSION ENV SPARK_HOME /opt/spark diff --git a/pom.xml b/pom.xml index 3f94dc4df983c..cf1c9018bc93b 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ 3.4.1 3.2.2 - 2.12.9 + 2.12.10 2.12 --diff --test diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 51a1097549bdd..fbb2ba3e270e0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -72,7 +72,7 @@ object ScalaReflection extends ScalaReflection { /** * Synchronize to prevent concurrent usage of `<:<` operator. * This operator is not thread safe in any current version of scala; i.e. - * (2.11.12, 2.12.9, 2.13.0-M5). + * (2.11.12, 2.12.10, 2.13.0-M5). * * See https://github.com/scala/bug/issues/10766 */ From 075134df44836a84c59cedeec62c4fba85cd3b3c Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Tue, 17 Sep 2019 21:15:09 +0800 Subject: [PATCH 3/4] Update genjavadoc to 0.14 --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f1d2e3788918f..30bd589544662 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -223,7 +223,7 @@ object SparkBuild extends PomBuild { .map(file), incOptions := incOptions.value.withNameHashing(true), publishMavenStyle := true, - unidocGenjavadocVersion := "0.13", + unidocGenjavadocVersion := "0.14", // Override SBT's default resolvers: resolvers := Seq( From 57de6c4f8f6401c3643847288072ab028d603103 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Wed, 18 Sep 2019 08:13:53 +0800 Subject: [PATCH 4/4] Apply SPARK-29124 --- .../scala/org/apache/spark/util/random/XORShiftRandom.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala b/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala index af09e50a157ae..11aa9da83b9ed 100644 --- a/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala +++ b/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala @@ -60,7 +60,7 @@ private[spark] object XORShiftRandom { /** Hash seeds to have 0/1 bits throughout. */ private[random] def hashSeed(seed: Long): Long = { val bytes = ByteBuffer.allocate(java.lang.Long.BYTES).putLong(seed).array() - val lowBits = MurmurHash3.bytesHash(bytes) + val lowBits = MurmurHash3.bytesHash(bytes, MurmurHash3.arraySeed) val highBits = MurmurHash3.bytesHash(bytes, lowBits) (highBits.toLong << 32) | (lowBits.toLong & 0xFFFFFFFFL) }