From a92c80d0d96db064da436ecfcc24bbcbc9bc015d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 14:22:41 +0800 Subject: [PATCH 01/10] move unused-import check to sbt only --- pom.xml | 5 +---- project/SparkBuild.scala | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 3ae2e7420e154..85cf5a00b0b24 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,6 @@ 3.2.2 2.12.10 2.12 - -Ywarn-unused-import 2.0.0 --test @@ -2538,7 +2537,6 @@ -deprecation -feature -explaintypes - ${scalac.arg.unused-imports} -target:jvm-1.8 @@ -3262,13 +3260,12 @@ - + scala-2.13 2.13.3 2.13 - -Wconf:cat=unused-imports:e diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 55c87fcb3aaa2..846ebaf25d86f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -221,6 +221,7 @@ object SparkBuild extends PomBuild { Seq( "-Xfatal-warnings", "-deprecation", + "Ywarn-unused-import", "-P:silencer:globalFilters=.*deprecated.*" //regex to catch deprecation warnings and supress them ) } else { @@ -230,6 +231,7 @@ object SparkBuild extends PomBuild { // see `scalac -Wconf:help` for details "-Wconf:cat=deprecation:wv,any:e", // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately + "-Wconf:cat=unused-imports:e", "-Wconf:cat=lint-multiarg-infix:wv", "-Wconf:cat=other-nullary-override:wv", "-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv", From b9026f2567d8ca94ef4bc145e726554521a1099f Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 14:33:15 +0800 Subject: [PATCH 02/10] add a unused import to test --- core/src/main/scala/org/apache/spark/SparkContext.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 0440a9de6ab31..c7e234a7941ee 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -27,6 +27,7 @@ import scala.collection.JavaConverters._ import scala.collection.Map import scala.collection.immutable import scala.collection.mutable.HashMap +import scala.collection.mutable.HashSet import scala.language.implicitConversions import scala.reflect.{classTag, ClassTag} import scala.util.control.NonFatal From 4f4cf1acd3280596178022999a616650a0d191e1 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 14:55:24 +0800 Subject: [PATCH 03/10] fix --- project/SparkBuild.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 846ebaf25d86f..626567e44fe36 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -221,7 +221,7 @@ object SparkBuild extends PomBuild { Seq( "-Xfatal-warnings", "-deprecation", - "Ywarn-unused-import", + "-Ywarn-unused-import", "-P:silencer:globalFilters=.*deprecated.*" //regex to catch deprecation warnings and supress them ) } else { @@ -231,8 +231,8 @@ object SparkBuild extends PomBuild { // see `scalac -Wconf:help` for details "-Wconf:cat=deprecation:wv,any:e", // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately - "-Wconf:cat=unused-imports:e", "-Wconf:cat=lint-multiarg-infix:wv", + "-Wconf:cat=unused-imports:e", "-Wconf:cat=other-nullary-override:wv", "-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv", "-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv", From 3f721974de35761a4663ee1f481ac5d9f18223a3 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 16:17:01 +0800 Subject: [PATCH 04/10] change scala 213 conf --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 626567e44fe36..f8370404e921d 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -232,7 +232,7 @@ object SparkBuild extends PomBuild { "-Wconf:cat=deprecation:wv,any:e", // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately "-Wconf:cat=lint-multiarg-infix:wv", - "-Wconf:cat=unused-imports:e", + "-Wunused:imports", "-Wconf:cat=other-nullary-override:wv", "-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv", "-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv", From cffdbcdae58cd21dfaaa3490122dff93e8985765 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 16:23:45 +0800 Subject: [PATCH 05/10] revert test --- core/src/main/scala/org/apache/spark/SparkContext.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index c7e234a7941ee..0440a9de6ab31 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -27,7 +27,6 @@ import scala.collection.JavaConverters._ import scala.collection.Map import scala.collection.immutable import scala.collection.mutable.HashMap -import scala.collection.mutable.HashSet import scala.language.implicitConversions import scala.reflect.{classTag, ClassTag} import scala.util.control.NonFatal From 70e84c6cf26f000b65b2c542af98857f1c0daac7 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 16:44:37 +0800 Subject: [PATCH 06/10] by scala 2.13 --- .../scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala | 1 - .../org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala | 1 - 2 files changed, 2 deletions(-) diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala index 8da46db98be81..bbc4bca501641 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala @@ -17,7 +17,6 @@ package org.apache.spark.graphx.impl -import scala.language.higherKinds import scala.reflect.ClassTag import org.apache.spark.graphx._ diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala index a8ed59b09bbb7..cf4c8ca2a9c42 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala @@ -17,7 +17,6 @@ package org.apache.spark.graphx.impl -import scala.language.higherKinds import scala.language.implicitConversions import scala.reflect.ClassTag From 454c52514e4d4404992f6fef1cf9f30168fce127 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 16:59:43 +0800 Subject: [PATCH 07/10] revert --- .../scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala | 1 + .../org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala index bbc4bca501641..8da46db98be81 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala @@ -17,6 +17,7 @@ package org.apache.spark.graphx.impl +import scala.language.higherKinds import scala.reflect.ClassTag import org.apache.spark.graphx._ diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala index cf4c8ca2a9c42..a8ed59b09bbb7 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala @@ -17,6 +17,7 @@ package org.apache.spark.graphx.impl +import scala.language.higherKinds import scala.language.implicitConversions import scala.reflect.ClassTag From 6fbbd2ca65534a7df5499fe696803557afeccd81 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 17:21:53 +0800 Subject: [PATCH 08/10] comments Scala 2.13 conifig --- project/SparkBuild.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f8370404e921d..253e092638163 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -231,8 +231,9 @@ object SparkBuild extends PomBuild { // see `scalac -Wconf:help` for details "-Wconf:cat=deprecation:wv,any:e", // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately + // TODO Enable this option when Scala 2.12 is no longer supported. + // "-Wunused:imports", "-Wconf:cat=lint-multiarg-infix:wv", - "-Wunused:imports", "-Wconf:cat=other-nullary-override:wv", "-Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv", "-Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv", From 911acfeff75ed072941703dfe96959a7b878f0e1 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 18:55:58 +0800 Subject: [PATCH 09/10] add TODO jira number --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 253e092638163..0dc52f8dca82c 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -231,7 +231,7 @@ object SparkBuild extends PomBuild { // see `scalac -Wconf:help` for details "-Wconf:cat=deprecation:wv,any:e", // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately - // TODO Enable this option when Scala 2.12 is no longer supported. + // TODO(SPARK-33499) Enable this option when Scala 2.12 is no longer supported. // "-Wunused:imports", "-Wconf:cat=lint-multiarg-infix:wv", "-Wconf:cat=other-nullary-override:wv", From 0c48e8dd055ccebc01a7402e671e0f7fa62b9b2b Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2020 19:02:18 +0800 Subject: [PATCH 10/10] add : --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 0dc52f8dca82c..05413b7091ad9 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -231,7 +231,7 @@ object SparkBuild extends PomBuild { // see `scalac -Wconf:help` for details "-Wconf:cat=deprecation:wv,any:e", // 2.13-specific warning hits to be muted (as narrowly as possible) and addressed separately - // TODO(SPARK-33499) Enable this option when Scala 2.12 is no longer supported. + // TODO(SPARK-33499): Enable this option when Scala 2.12 is no longer supported. // "-Wunused:imports", "-Wconf:cat=lint-multiarg-infix:wv", "-Wconf:cat=other-nullary-override:wv",