From 61750b8c324d5264c67bd4ecf876cf6bc6bc6c56 Mon Sep 17 00:00:00 2001 From: Xingbo Jiang Date: Thu, 24 Oct 2019 13:25:15 +0200 Subject: [PATCH 1/3] Prepare Spark release 3.0.0-preview --- R/pkg/R/sparkR.R | 11 ----------- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- docs/_config.yml | 4 ++-- examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10-token-provider/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graph/api/pom.xml | 2 +- graph/cypher/pom.xml | 2 +- graph/graph/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- python/pyspark/version.py | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- .../kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 42 files changed, 42 insertions(+), 53 deletions(-) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index cdb59093781f..99a95ce771dc 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -334,17 +334,6 @@ sparkR.session <- function( assign(".sparkRsession", sparkSession, envir = .sparkREnv) } - # Check if version number of SparkSession matches version number of SparkR package - jvmVersion <- callJMethod(sparkSession, "version") - # Remove -SNAPSHOT from jvm versions - jvmVersionStrip <- gsub("-SNAPSHOT", "", jvmVersion) - rPackageVersion <- paste0(packageVersion("SparkR")) - - if (jvmVersionStrip != rPackageVersion) { - warning(paste("Version mismatch between Spark JVM and SparkR package. JVM version was", - jvmVersion, ", while R package version was", rPackageVersion)) - } - sparkSession } diff --git a/assembly/pom.xml b/assembly/pom.xml index ef916fb99a04..69efac5f43ca 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index f042a12fda3d..b19380d12312 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 2ee17800c10e..4000f9296dc8 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index a6d99813a850..f144b70b70c8 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 76a402bb2bd3..b355b29abc80 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 3c3c0d2d96a1..5175e00739d1 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 883b73a69c9d..ca6de221c037 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 93a4f67fd23f..ad2ba6659e9c 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 38eb8adac500..19e67359a1c6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/docs/_config.yml b/docs/_config.yml index 57b8d716ee55..b52081970165 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -14,8 +14,8 @@ include: # These allow the documentation to be updated with newer releases # of Spark, Scala, and Mesos. -SPARK_VERSION: 3.0.0-SNAPSHOT -SPARK_VERSION_SHORT: 3.0.0 +SPARK_VERSION: 3.0.0-preview +SPARK_VERSION_SHORT: 3.0.0-preview SCALA_BINARY_VERSION: "2.12" SCALA_VERSION: "2.12.10" MESOS_VERSION: 1.0.0 diff --git a/examples/pom.xml b/examples/pom.xml index a099f1e042e9..f37f059a728c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index ba6f20bfdbf5..60c5f80492d5 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index aff79b8b8e64..0f0176af7190 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index f2dcf5d217a8..8442c46c3cb2 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 693820da6af6..cceace392d5a 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml index 01ca96b4f8f4..744423a88c1d 100644 --- a/external/kafka-0-10-token-provider/pom.xml +++ b/external/kafka-0-10-token-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index d11569d709b2..0e1d95cf822f 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 0ce922349ea6..ae75d1b9f808 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 7d69764b77de..e284c52e8404 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index a23d255f9187..190fffd909b1 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/graph/api/pom.xml b/graph/api/pom.xml index a18c5290195f..953d0351b5b4 100644 --- a/graph/api/pom.xml +++ b/graph/api/pom.xml @@ -23,7 +23,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/graph/cypher/pom.xml b/graph/cypher/pom.xml index 12b3832d9777..865a1bcf9d5a 100644 --- a/graph/cypher/pom.xml +++ b/graph/cypher/pom.xml @@ -23,7 +23,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/graph/graph/pom.xml b/graph/graph/pom.xml index 7438ee5e54fe..8ed0b92a419d 100644 --- a/graph/graph/pom.xml +++ b/graph/graph/pom.xml @@ -23,7 +23,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 444568a03d6c..6f46cccc1187 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 02734e82ed26..7c9357f49218 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index e75e8345cd51..86f9c1a0f200 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 2eab868ac0dc..497b70ddfed9 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 11769ef548d7..75ab818515a3 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/pom.xml b/pom.xml index b5c34790f1d3..a10850fc7938 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview pom Spark Project Parent POM http://spark.apache.org/ diff --git a/python/pyspark/version.py b/python/pyspark/version.py index ba2a40cec01e..57f09c306a44 100644 --- a/python/pyspark/version.py +++ b/python/pyspark/version.py @@ -16,4 +16,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.0.0.dev0" +__version__ = "3.0.0.p0" diff --git a/repl/pom.xml b/repl/pom.xml index c7de67e41ca9..284dc923313a 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 487876b12ce9..d4a073376d11 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index c79caff164ce..038613f810f0 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 107ba365aa77..2038ddc2317e 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 0e5df14e060d..15671c17a752 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index bcebb225dfac..ad7fe28e8d18 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 3a8b7d22397f..3565056b6206 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 9dd927084298..64b78265e485 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index f627227aa038..1f3fbef6100d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 87af6388e111..7394391d014d 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index e380e869f55c..e2608976938d 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.0.0-SNAPSHOT + 3.0.0-preview ../pom.xml From 3149e7d3d8678c776ad42919f9ad9b417b5b9816 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 28 Oct 2019 11:52:55 +0900 Subject: [PATCH 2/3] Use JVM version check in SparkR --- R/pkg/R/sparkR.R | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 99a95ce771dc..dc1d1b238923 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -334,6 +334,17 @@ sparkR.session <- function( assign(".sparkRsession", sparkSession, envir = .sparkREnv) } + # Check if version number of SparkSession matches version number of SparkR package + jvmVersion <- callJMethod(sparkSession, "version") + # Remove -SNAPSHOT from jvm versions + jvmVersionStrip <- gsub("-preview", "", jvmVersion) + rPackageVersion <- paste0(packageVersion("SparkR")) + + if (jvmVersionStrip != rPackageVersion) { + warning(paste("Version mismatch between Spark JVM and SparkR package. JVM version was", + jvmVersion, ", while R package version was", rPackageVersion)) + } + sparkSession } From e1808a56e5e8c55b599d9e80873ad17ba9503abc Mon Sep 17 00:00:00 2001 From: Xingbo Jiang Date: Mon, 28 Oct 2019 17:13:18 -0700 Subject: [PATCH 3/3] update python version --- python/pyspark/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/version.py b/python/pyspark/version.py index 57f09c306a44..ba2a40cec01e 100644 --- a/python/pyspark/version.py +++ b/python/pyspark/version.py @@ -16,4 +16,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.0.0.p0" +__version__ = "3.0.0.dev0"