From 46dd2ff094f48117a10a13bf72d235bf5cab19a4 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Wed, 29 Jul 2020 14:43:32 +0300
Subject: [PATCH 01/15] [WIP}[SPARK-21708][Build] Migrate build to sbt 1.x

Migrate sbt-launcher URL to download one for sbt 1.x.
Update plugin version where required by sbt update.
Change sbt version to be used to latest at the moment.
Adjust build settings according to plugin and sbt changes.
---
 build/sbt-launch-lib.bash |  2 +-
 project/SparkBuild.scala  | 68 +++++++++++++++++++++++++--------------
 project/build.properties  |  2 +-
 project/plugins.sbt       | 22 ++++---------
 4 files changed, 52 insertions(+), 42 deletions(-)

diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash
index 162bfbf2257c7..423ba3b766e61 100755
--- a/build/sbt-launch-lib.bash
+++ b/build/sbt-launch-lib.bash
@@ -39,7 +39,7 @@ dlog () {
 
 acquire_sbt_jar () {
   SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties`
-  URL1=https://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+  URL1=https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar
   JAR=build/sbt-launch-${SBT_VERSION}.jar
 
   sbt_jar=$JAR
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6328daec027ef..519a726f812f8 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -28,13 +28,14 @@ import scala.collection.mutable.Stack
 import sbt._
 import sbt.Classpaths.publishTask
 import sbt.Keys._
-import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
 import com.etsy.sbt.checkstyle.CheckstylePlugin.autoImport._
 import com.simplytyped.Antlr4Plugin._
 import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
 import com.typesafe.tools.mima.plugin.MimaKeys
 import org.scalastyle.sbt.ScalastylePlugin.autoImport._
 import org.scalastyle.sbt.Tasks
+import sbtassembly.AssemblyPlugin.autoImport._
+import sbtunidoc.GenJavadocPlugin.autoImport._
 
 import spray.revolver.RevolverPlugin._
 
@@ -83,6 +84,7 @@ object BuildCommons {
 object SparkBuild extends PomBuild {
 
   import BuildCommons._
+  import sbtunidoc.ScalaUnidocPlugin
   import scala.collection.mutable.Map
 
   val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty
@@ -204,7 +206,6 @@ object SparkBuild extends PomBuild {
     javaHome := sys.env.get("JAVA_HOME")
       .orElse(sys.props.get("java.home").map { p => new File(p).getParentFile().getAbsolutePath() })
       .map(file),
-    incOptions := incOptions.value.withNameHashing(true),
     publishMavenStyle := true,
     unidocGenjavadocVersion := "0.16",
 
@@ -219,8 +220,10 @@ object SparkBuild extends PomBuild {
     ),
     externalResolvers := resolvers.value,
     otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,
-    publishLocalConfiguration in MavenCompile :=
-      new PublishConfiguration(None, "dotM2", packagedArtifacts.value, Seq(), ivyLoggingLevel.value),
+    publishLocalConfiguration in MavenCompile := PublishConfiguration()
+        .withResolverName("dotM2")
+        .withArtifacts(packagedArtifacts.value.toVector)
+        .withLogging(ivyLoggingLevel.value),
     publishMavenStyle in MavenCompile := true,
     publishLocal in MavenCompile := publishTask(publishLocalConfiguration in MavenCompile, deliverLocal).value,
     publishLocalBoth := Seq(publishLocal in MavenCompile, publishLocal).dependOn.value,
@@ -273,14 +276,15 @@ object SparkBuild extends PomBuild {
       val out = streams.value
 
       def logProblem(l: (=> String) => Unit, f: File, p: xsbti.Problem) = {
-        l(f.toString + ":" + p.position.line.fold("")(_ + ":") + " " + p.message)
+        val jmap = new java.util.function.Function[Integer, String]() {override def apply(i: Integer): String = {i.toString}}
+        l(f.toString + ":" + p.position.line.map[String](jmap.apply).map(_ + ":").orElse("") + " " + p.message)
         l(p.position.lineContent)
         l("")
       }
 
       var failed = 0
-      analysis.infos.allInfos.foreach { case (k, i) =>
-        i.reportedProblems foreach { p =>
+      analysis.asInstanceOf[sbt.internal.inc.Analysis].infos.allInfos.foreach { case (k, i) =>
+        i.getReportedProblems foreach { p =>
           val deprecation = p.message.contains("deprecated")
 
           if (!deprecation) {
@@ -485,12 +489,12 @@ object SparkParallelTestGrouping {
     testGrouping in Test := {
       val tests: Seq[TestDefinition] = (definedTests in Test).value
       val defaultForkOptions = ForkOptions(
-        bootJars = Nil,
         javaHome = javaHome.value,
-        connectInput = connectInput.value,
         outputStrategy = outputStrategy.value,
-        runJVMOptions = (javaOptions in Test).value,
+        bootJars = Vector.empty[java.io.File],
         workingDirectory = Some(baseDirectory.value),
+        runJVMOptions = (javaOptions in Test).value.toVector,
+        connectInput = connectInput.value,
         envVars = (envVars in Test).value
       )
       tests.groupBy(test => testNameToTestGroup(test.name)).map { case (groupName, groupTests) =>
@@ -498,7 +502,7 @@ object SparkParallelTestGrouping {
           if (groupName == DEFAULT_TEST_GROUP) {
             defaultForkOptions
           } else {
-            defaultForkOptions.copy(runJVMOptions = defaultForkOptions.runJVMOptions ++
+            defaultForkOptions.withRunJVMOptions(defaultForkOptions.runJVMOptions ++
               Seq(s"-Djava.io.tmpdir=${baseDirectory.value}/target/tmp/$groupName"))
           }
         }
@@ -512,6 +516,7 @@ object SparkParallelTestGrouping {
 }
 
 object Core {
+  import scala.sys.process.Process
   lazy val settings = Seq(
     resourceGenerators in Compile += Def.task {
       val buildScript = baseDirectory.value + "/../build/spark-build-info"
@@ -557,6 +562,7 @@ object DockerIntegrationTests {
  */
 object KubernetesIntegrationTests {
   import BuildCommons._
+  import scala.sys.process.Process
 
   val dockerBuild = TaskKey[Unit]("docker-imgs", "Build the docker images for ITs.")
   val runITs = TaskKey[Unit]("run-its", "Only run ITs, skip image build.")
@@ -634,7 +640,9 @@ object ExcludedDependencies {
  */
 object OldDeps {
 
-  lazy val project = Project("oldDeps", file("dev"), settings = oldDepsSettings)
+  lazy val project = Project("oldDeps", file("dev"))
+    .settings(oldDepsSettings)
+    .disablePlugins(com.typesafe.sbt.pom.PomReaderPlugin)
 
   lazy val allPreviousArtifactKeys = Def.settingDyn[Seq[Set[ModuleID]]] {
     SparkBuild.mimaProjects
@@ -650,7 +658,10 @@ object OldDeps {
 }
 
 object Catalyst {
-  lazy val settings = antlr4Settings ++ Seq(
+  import com.simplytyped.Antlr4Plugin
+  import com.simplytyped.Antlr4Plugin.autoImport._
+
+  lazy val settings = Antlr4Plugin.projectSettings ++ Seq(
     antlr4Version in Antlr4 := SbtPomKeys.effectivePom.value.getProperties.get("antlr4.version").asInstanceOf[String],
     antlr4PackageName in Antlr4 := Some("org.apache.spark.sql.catalyst.parser"),
     antlr4GenListener in Antlr4 := true,
@@ -721,27 +732,27 @@ object Hive {
 
 object Assembly {
   import sbtassembly.AssemblyUtils._
-  import sbtassembly.Plugin._
-  import AssemblyKeys._
+  import sbtassembly.AssemblyPlugin.autoImport._
 
   val hadoopVersion = taskKey[String]("The version of hadoop that spark is compiled against.")
 
-  lazy val settings = assemblySettings ++ Seq(
+  lazy val settings = baseAssemblySettings ++ Seq(
     test in assembly := {},
     hadoopVersion := {
       sys.props.get("hadoop.version")
         .getOrElse(SbtPomKeys.effectivePom.value.getProperties.get("hadoop.version").asInstanceOf[String])
     },
-    jarName in assembly := {
+    assemblyJarName in assembly := {
+      lazy val hdpVersion = hadoopVersion.value
       if (moduleName.value.contains("streaming-kafka-0-10-assembly")
         || moduleName.value.contains("streaming-kinesis-asl-assembly")) {
         s"${moduleName.value}-${version.value}.jar"
       } else {
-        s"${moduleName.value}-${version.value}-hadoop${hadoopVersion.value}.jar"
+        s"${moduleName.value}-${version.value}-hadoop${hdpVersion}.jar"
       }
     },
-    jarName in (Test, assembly) := s"${moduleName.value}-test-${version.value}.jar",
-    mergeStrategy in assembly := {
+    assemblyJarName in (Test, assembly) := s"${moduleName.value}-test-${version.value}.jar",
+    assemblyMergeStrategy in assembly := {
       case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf")
                                                                => MergeStrategy.discard
       case m if m.toLowerCase(Locale.ROOT).matches("meta-inf.*\\.sf$")
@@ -756,8 +767,7 @@ object Assembly {
 }
 
 object PySparkAssembly {
-  import sbtassembly.Plugin._
-  import AssemblyKeys._
+  import sbtassembly.AssemblyPlugin.autoImport._
   import java.util.zip.{ZipOutputStream, ZipEntry}
 
   lazy val settings = Seq(
@@ -807,8 +817,13 @@ object PySparkAssembly {
 object Unidoc {
 
   import BuildCommons._
-  import sbtunidoc.Plugin._
-  import UnidocKeys._
+  import sbtunidoc.BaseUnidocPlugin
+  import sbtunidoc.JavaUnidocPlugin
+  import sbtunidoc.ScalaUnidocPlugin
+  import sbtunidoc.BaseUnidocPlugin.autoImport._
+  import sbtunidoc.GenJavadocPlugin.autoImport._
+  import sbtunidoc.JavaUnidocPlugin.autoImport._
+  import sbtunidoc.ScalaUnidocPlugin.autoImport._
 
   private def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = {
     packages
@@ -848,7 +863,10 @@ object Unidoc {
 
   val unidocSourceBase = settingKey[String]("Base URL of source links in Scaladoc.")
 
-  lazy val settings = scalaJavaUnidocSettings ++ Seq (
+  lazy val settings = BaseUnidocPlugin.projectSettings ++
+                      ScalaUnidocPlugin.projectSettings ++
+                      JavaUnidocPlugin.projectSettings ++
+                      Seq (
     publish := {},
 
     unidocProjectFilter in(ScalaUnidoc, unidoc) :=
diff --git a/project/build.properties b/project/build.properties
index 23aa187fb35a7..b1e5e313d853f 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-sbt.version=0.13.18
+sbt.version=1.3.13
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 5f21d8126e48a..e5dfa3cb90d92 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -23,8 +23,7 @@ libraryDependencies += "com.puppycrawl.tools" % "checkstyle" % "8.25"
 // checkstyle uses guava 23.0.
 libraryDependencies += "com.google.guava" % "guava" % "23.0"
 
-// need to make changes to uptake sbt 1.0 support in "com.eed3si9n" % "sbt-assembly" % "1.14.5"
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
 
 addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4")
 
@@ -38,13 +37,14 @@ resolvers += Resolver.url("bintray",
 addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0")
 
 // sbt 1.0.0 support: https://github.com/AlpineNow/junit_xml_listener/issues/6
-addSbtPlugin("com.alpinenow" % "junit_xml_listener" % "0.5.1")
+// addSbtPlugin("com.alpinenow" % "junit_xml_listener" % "0.5.1")
 
-// need to make changes to uptake sbt 1.0 support in "com.eed3si9n" % "sbt-unidoc" % "0.4.1"
-addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.3")
+addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3")
 
 // need to make changes to uptake sbt 1.0 support in "com.cavorite" % "sbt-avro-1-7" % "1.1.2"
 addSbtPlugin("com.cavorite" % "sbt-avro" % "0.3.2")
+addSbtPlugin("com.cavorite" % "sbt-avro" % "2.1.1")
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.10.0"
 
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1")
 
@@ -52,14 +52,6 @@ libraryDependencies += "org.ow2.asm"  % "asm" % "7.2"
 
 libraryDependencies += "org.ow2.asm"  % "asm-commons" % "7.2"
 
-// sbt 1.0.0 support: https://github.com/ihji/sbt-antlr4/issues/14
-addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.7.13")
+addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.2")
 
-// Spark uses a custom fork of the sbt-pom-reader plugin which contains a patch to fix issues
-// related to test-jar dependencies (https://github.com/sbt/sbt-pom-reader/pull/14). The source for
-// this fork is published at https://github.com/JoshRosen/sbt-pom-reader/tree/v1.0.0-spark
-// and corresponds to commit b160317fcb0b9d1009635a7c5aa05d0f3be61936 in that repository.
-// In the long run, we should try to merge our patch upstream and switch to an upstream version of
-// the plugin; this is tracked at SPARK-14401.
-
-addSbtPlugin("org.spark-project" % "sbt-pom-reader" % "1.0.0-spark")
+addSbtPlugin("com.typesafe.sbt" % "sbt-pom-reader" % "2.1.0")

From 8fd7d64ef4b3b0c9617fab7be7e7f902cce81a81 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Fri, 21 Aug 2020 22:20:06 +0300
Subject: [PATCH 02/15] Aligh avro compiler version with project's one

---
 project/plugins.sbt | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index e5dfa3cb90d92..a1fa41020c719 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -41,10 +41,8 @@ addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0")
 
 addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3")
 
-// need to make changes to uptake sbt 1.0 support in "com.cavorite" % "sbt-avro-1-7" % "1.1.2"
-addSbtPlugin("com.cavorite" % "sbt-avro" % "0.3.2")
 addSbtPlugin("com.cavorite" % "sbt-avro" % "2.1.1")
-libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.10.0"
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.8.2"
 
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1")
 

From 846c9bd2e750c7a588c2fdd21495f00eb636f975 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Fri, 21 Aug 2020 22:20:31 +0300
Subject: [PATCH 03/15] Use upstream sbt-pom-reader plugin with classifer fix

---
 project/plugins.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index a1fa41020c719..f5ff93641103b 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -52,4 +52,4 @@ libraryDependencies += "org.ow2.asm"  % "asm-commons" % "7.2"
 
 addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.2")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-pom-reader" % "2.1.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-pom-reader" % "2.2.0")

From 9b41ffd01ad54fa27010762b25fd27383aef3207 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Mon, 24 Aug 2020 13:43:05 +0300
Subject: [PATCH 04/15] Remove obsoleted junit_xml_listener plugin

---
 project/plugins.sbt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index f5ff93641103b..3f0c36a80e56a 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -36,9 +36,6 @@ resolvers += Resolver.url("bintray",
   new java.net.URL("https://dl.bintray.com/typesafe/sbt-plugins"))(Resolver.defaultIvyPatterns)
 addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0")
 
-// sbt 1.0.0 support: https://github.com/AlpineNow/junit_xml_listener/issues/6
-// addSbtPlugin("com.alpinenow" % "junit_xml_listener" % "0.5.1")
-
 addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3")
 
 addSbtPlugin("com.cavorite" % "sbt-avro" % "2.1.1")

From c5ba1e6248dd9aa085b4ff79909bae0ebbc8c50e Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Tue, 25 Aug 2020 03:50:12 +0300
Subject: [PATCH 05/15] Reuse avro auto-generated classes in sql/core module
 tests

---
 project/SparkBuild.scala | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 519a726f812f8..19d773ff2421f 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -671,6 +671,9 @@ object Catalyst {
 }
 
 object SQL {
+
+  import sbtavro.SbtAvro.autoImport._
+
   lazy val settings = Seq(
     initialCommands in console :=
       """
@@ -692,8 +695,10 @@ object SQL {
         |import sqlContext.implicits._
         |import sqlContext._
       """.stripMargin,
-    cleanupCommands in console := "sc.stop()"
+    cleanupCommands in console := "sc.stop()",
+    Test / avroGenerate := (Compile / avroGenerate).value
   )
+
 }
 
 object Hive {

From 1d312024450f9f5802e9bc41b6be9d249501ca22 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Sun, 6 Sep 2020 22:30:36 +0300
Subject: [PATCH 06/15] Update sbt-mima to latest version

---
 project/plugins.sbt | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index 3f0c36a80e56a..864109c8bab56 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -31,10 +31,7 @@ addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2")
 
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 
-// SPARK-29560 Only sbt-mima-plugin needs this repo
-resolvers += Resolver.url("bintray",
-  new java.net.URL("https://dl.bintray.com/typesafe/sbt-plugins"))(Resolver.defaultIvyPatterns)
-addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0")
+addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.7.0")
 
 addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3")
 

From 45dd45c571859fb43c0a9706df6eb79cd0661d96 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Sun, 6 Sep 2020 22:31:50 +0300
Subject: [PATCH 07/15] Check for binary compatibility against Spark 3.0.0

---
 project/MimaBuild.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index 10c02103aeddb..cc51ffb6e377f 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -88,7 +88,7 @@ object MimaBuild {
 
   def mimaSettings(sparkHome: File, projectRef: ProjectRef) = {
     val organization = "org.apache.spark"
-    val previousSparkVersion = "2.4.0"
+    val previousSparkVersion = "3.0.0"
     val project = projectRef.project
     val fullId = "spark-" + project + "_2.12"
     mimaDefaultSettings ++

From e9fabac4a941dce8e469a95ad0c43b70b7080159 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Wed, 9 Sep 2020 04:34:04 +0300
Subject: [PATCH 08/15] Fix exception in mima-ignores generator

---
 .../main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
index f9bc499961ad7..a6fee8616df11 100644
--- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
+++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
@@ -24,6 +24,7 @@ import scala.reflect.runtime.universe.runtimeMirror
 import scala.util.Try
 
 import org.clapper.classutil.ClassFinder
+import org.objectweb.asm.Opcodes
 
 /**
  * A tool for generating classes to be excluded during binary checking with MIMA. It is expected
@@ -146,7 +147,7 @@ object GenerateMIMAIgnore {
    * and subpackages both from directories and jars present on the classpath.
    */
   private def getClasses(packageName: String): Set[String] = {
-    val finder = ClassFinder()
+    val finder = ClassFinder(maybeOverrideAsmVersion = Some(Opcodes.ASM7))
     finder
       .getClasses
       .map(_.name)

From 754c637429118d7e5b5bd7f6c1f9c1afbdb00356 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Sat, 12 Sep 2020 02:19:31 +0300
Subject: [PATCH 09/15] Set correct usage of updated Mima plugin

Update sbt-mima plugin to latest released.
Modify settings to be applied in modules to be checked.
Modify build file to ignore modules and general and check modules that
have previous binary artifacts.
Exclude incompatibilities detected on plugin update - to be verified
afterwards.
---
 project/MimaBuild.scala    | 15 ++++++++-------
 project/MimaExcludes.scala | 30 ++++++++++++++++++++++++++++++
 project/SparkBuild.scala   |  5 ++++-
 project/plugins.sbt        |  2 +-
 4 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index cc51ffb6e377f..badcdf34a2ad0 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -22,9 +22,7 @@ import com.typesafe.tools.mima.core._
 import com.typesafe.tools.mima.core.MissingClassProblem
 import com.typesafe.tools.mima.core.MissingTypesProblem
 import com.typesafe.tools.mima.core.ProblemFilters._
-import com.typesafe.tools.mima.plugin.MimaKeys.{mimaBinaryIssueFilters, mimaPreviousArtifacts}
-import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
-
+import com.typesafe.tools.mima.plugin.MimaKeys.{mimaBinaryIssueFilters, mimaPreviousArtifacts, mimaFailOnNoPrevious}
 
 object MimaBuild {
 
@@ -86,14 +84,17 @@ object MimaBuild {
     ignoredMembers.flatMap(excludeMember) ++ MimaExcludes.excludes(currentSparkVersion)
   }
 
-  def mimaSettings(sparkHome: File, projectRef: ProjectRef) = {
+  def mimaSettings(sparkHome: File, projectRef: ProjectRef): Seq[Setting[_]] = {
     val organization = "org.apache.spark"
     val previousSparkVersion = "3.0.0"
     val project = projectRef.project
     val fullId = "spark-" + project + "_2.12"
-    mimaDefaultSettings ++
-    Seq(mimaPreviousArtifacts := Set(organization % fullId % previousSparkVersion),
-      mimaBinaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value))
+
+    Seq(
+      mimaFailOnNoPrevious := true,
+      mimaPreviousArtifacts := Set(organization % fullId % previousSparkVersion),
+      mimaBinaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value)
+    )
   }
 
 }
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index d32d31daae8e7..98769d951b6ac 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -36,6 +36,36 @@ object MimaExcludes {
 
   // Exclude rules for 3.1.x
   lazy val v31excludes = v30excludes ++ Seq(
+    // mima plugin update caused new incompatibilities to be detected
+    // core module
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.shuffle.sort.io.LocalDiskShuffleMapOutputWriter.commitAllPartitions"),
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.shuffle.api.ShuffleMapOutputWriter.commitAllPartitions"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.shuffle.api.ShuffleMapOutputWriter.commitAllPartitions"),
+    // mllib module
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.totalIterations"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.classification.LogisticRegressionTrainingSummary.$init$"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.labels"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.truePositiveRateByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.falsePositiveRateByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.precisionByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.recallByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.fMeasureByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.fMeasureByLabel"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.accuracy"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedTruePositiveRate"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFalsePositiveRate"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedRecall"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedPrecision"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFMeasure"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.LogisticRegressionSummary.weightedFMeasure"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.roc"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.areaUnderROC"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.pr"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.fMeasureByThreshold"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.precisionByThreshold"),
+    ProblemFilters.exclude[NewMixinForwarderProblem]("org.apache.spark.ml.classification.BinaryLogisticRegressionSummary.recallByThreshold"),
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.classification.FMClassifier.trainImpl"),
+    ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.FMRegressor.trainImpl"),
     // [SPARK-31077] Remove ChiSqSelector dependency on mllib.ChiSqSelectorModel
     // private constructor
     ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.feature.ChiSqSelectorModel.this"),
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 19d773ff2421f..6bf36e7a0ea87 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -306,7 +306,10 @@ object SparkBuild extends PomBuild {
         sys.error(s"$failed fatal warnings")
       }
       analysis
-    }
+    },
+    // disable Mima check for all modules,
+    // to be enabled in specific ones that have previous artifacts
+    MimaKeys.mimaFailOnNoPrevious := false
   )
 
   def enable(settings: Seq[Setting[_]])(projectRef: ProjectRef) = {
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 864109c8bab56..da466da9945c1 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -31,7 +31,7 @@ addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2")
 
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 
-addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.7.0")
+addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.8.0")
 
 addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3")
 

From da9def589c29c164be2a60733eb4bf591bc12b77 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Sun, 13 Sep 2020 17:13:12 +0300
Subject: [PATCH 10/15] Add .sbtopts file

Increase stack size for SBT at github actions environment.
Set max memory similar to Maven ones.
---
 .sbtopts | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 .sbtopts

diff --git a/.sbtopts b/.sbtopts
new file mode 100644
index 0000000000000..9afbdca6db1c7
--- /dev/null
+++ b/.sbtopts
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+-J-Xmx4G
+-J-Xss4m

From c570004dace8e095f4eecd98ece4ce3c65f25177 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Thu, 24 Sep 2020 01:48:46 +0300
Subject: [PATCH 11/15] Fix scope warnings

Ignore warning on independent evaluation for `streams.in(config).value`.
Use correct initialization for spark-shell settings.
---
 project/SparkBuild.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6bf36e7a0ea87..853aadd18fa69 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -159,7 +159,7 @@ object SparkBuild extends PomBuild {
         val scalaSourceV = Seq(file(scalaSource.in(config).value.getAbsolutePath))
         val configV = (baseDirectory in ThisBuild).value / scalaStyleOnCompileConfig
         val configUrlV = scalastyleConfigUrl.in(config).value
-        val streamsV = streams.in(config).value
+        val streamsV = (streams.in(config).value: @sbtUnchecked)
         val failOnErrorV = true
         val failOnWarningV = false
         val scalastyleTargetV = scalastyleTarget.in(config).value
@@ -418,7 +418,7 @@ object SparkBuild extends PomBuild {
     }
   ))(assembly)
 
-  enable(Seq(sparkShell := sparkShell in LocalProject("assembly")))(spark)
+  enable(Seq(sparkShell := (sparkShell in LocalProject("assembly")).value))(spark)
 
   // TODO: move this to its upstream project.
   override def projectDefinitions(baseDirectory: File): Seq[Project] = {

From c1d9148730608cee404aa86733879f43eccbfca8 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Thu, 24 Sep 2020 03:49:51 +0300
Subject: [PATCH 12/15] Adapt local publishing task

Rename task due prohibition of hyphenated names in SBT 1.x
Fix warning
---
 project/SparkBuild.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 853aadd18fa69..d4cd35620a86c 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -108,7 +108,7 @@ object SparkBuild extends PomBuild {
   override val userPropertiesMap = System.getProperties.asScala.toMap
 
   lazy val MavenCompile = config("m2r") extend(Compile)
-  lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
+  lazy val publishLocalBoth = TaskKey[Unit]("localPublish", "publish local for m2 and ivy", KeyRanks.ATask)
 
   lazy val sparkGenjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq(
     libraryDependencies += compilerPlugin(
@@ -225,7 +225,7 @@ object SparkBuild extends PomBuild {
         .withArtifacts(packagedArtifacts.value.toVector)
         .withLogging(ivyLoggingLevel.value),
     publishMavenStyle in MavenCompile := true,
-    publishLocal in MavenCompile := publishTask(publishLocalConfiguration in MavenCompile, deliverLocal).value,
+    publishLocal in MavenCompile := publishTask(publishLocalConfiguration in MavenCompile).value,
     publishLocalBoth := Seq(publishLocal in MavenCompile, publishLocal).dependOn.value,
 
     javacOptions in (Compile, doc) ++= {

From 942f26a251f1c7a38401e11b588af42ec0c91192 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Sun, 27 Sep 2020 23:21:07 +0300
Subject: [PATCH 13/15] Optimize Scalaunidoc settings other build file

---
 project/SparkBuild.scala | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index d4cd35620a86c..98fa20ae290cc 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -35,7 +35,6 @@ import com.typesafe.tools.mima.plugin.MimaKeys
 import org.scalastyle.sbt.ScalastylePlugin.autoImport._
 import org.scalastyle.sbt.Tasks
 import sbtassembly.AssemblyPlugin.autoImport._
-import sbtunidoc.GenJavadocPlugin.autoImport._
 
 import spray.revolver.RevolverPlugin._
 
@@ -84,7 +83,8 @@ object BuildCommons {
 object SparkBuild extends PomBuild {
 
   import BuildCommons._
-  import sbtunidoc.ScalaUnidocPlugin
+  import sbtunidoc.GenJavadocPlugin
+  import sbtunidoc.GenJavadocPlugin.autoImport._
   import scala.collection.mutable.Map
 
   val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty
@@ -110,11 +110,8 @@ object SparkBuild extends PomBuild {
   lazy val MavenCompile = config("m2r") extend(Compile)
   lazy val publishLocalBoth = TaskKey[Unit]("localPublish", "publish local for m2 and ivy", KeyRanks.ATask)
 
-  lazy val sparkGenjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq(
-    libraryDependencies += compilerPlugin(
-      "com.typesafe.genjavadoc" %% "genjavadoc-plugin" % unidocGenjavadocVersion.value cross CrossVersion.full),
+  lazy val sparkGenjavadocSettings: Seq[sbt.Def.Setting[_]] = GenJavadocPlugin.projectSettings ++ Seq(
     scalacOptions ++= Seq(
-      "-P:genjavadoc:out=" + (target.value / "java"),
       "-P:genjavadoc:strictVisibility=true" // hide package private types
     )
   )

From ccf82b0155b95283ed0afd6c962769e023bd727b Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Sun, 27 Sep 2020 23:21:44 +0300
Subject: [PATCH 14/15] Add exclusion for problematic class

---
 project/SparkBuild.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 98fa20ae290cc..794c1610fa63c 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -858,6 +858,7 @@ object Unidoc {
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalog/v2/utils")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hive")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/v2/avro")))
+      .map(_.filterNot(_.getCanonicalPath.contains("SSLOptions")))
   }
 
   private def ignoreClasspaths(classpaths: Seq[Classpath]): Seq[Classpath] = {

From 106492c62248717244e5c04e5122f44b3495143b Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Mon, 5 Oct 2020 18:30:12 +0300
Subject: [PATCH 15/15] Propagate externally set Maven profiles

---
 project/SparkBuild.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 794c1610fa63c..6929342d2f539 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -251,6 +251,8 @@ object SparkBuild extends PomBuild {
       "-sourcepath", (baseDirectory in ThisBuild).value.getAbsolutePath  // Required for relative source links in scaladoc
     ),
 
+    SbtPomKeys.profiles := profiles,
+
     // Remove certain packages from Scaladoc
     scalacOptions in (Compile, doc) := Seq(
       "-groups",