From d9830effa96fb0c831f5586327f645a01d075af0 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 31 Aug 2022 17:44:52 +0800 Subject: [PATCH 01/17] update version --- project/MimaBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala index 2bd05e60c02d0..ec9ce94a6c495 100644 --- a/project/MimaBuild.scala +++ b/project/MimaBuild.scala @@ -86,7 +86,7 @@ object MimaBuild { def mimaSettings(sparkHome: File, projectRef: ProjectRef): Seq[Setting[_]] = { val organization = "org.apache.spark" - val previousSparkVersion = "3.2.0" + val previousSparkVersion = "3.3.0" val project = projectRef.project val id = "spark-" + project From f8e3d79f733f5fb60c641ce8febd3409dfe48e02 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 31 Aug 2022 19:09:19 +0800 Subject: [PATCH 02/17] sql module --- project/MimaExcludes.scala | 47 +++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 3f3d85754775e..635f7f256c544 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -62,7 +62,52 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.copy"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.copy$default$2"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.this"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.apply") + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.apply"), + + ProblemFilters.exclude[MissingClassProblem]("org.apache.parquet.io.ColumnIOUtil"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.indexExists"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex") ) // Exclude rules for 3.3.x from 3.2.0 From 0d568b90eba1bfb059349eaaf76cad8bcfb4efbb Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 31 Aug 2022 19:18:55 +0800 Subject: [PATCH 03/17] catalyst module --- project/MimaExcludes.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 635f7f256c544..9b383c466f1b5 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -107,7 +107,12 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex") + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex"), + + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), ) // Exclude rules for 3.3.x from 3.2.0 From b47bc42b1551cb5ac52ff478f078076918eb871b Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 31 Aug 2022 19:19:56 +0800 Subject: [PATCH 04/17] catalyst module --- project/MimaExcludes.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 9b383c466f1b5..0cbac44217d40 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -112,7 +112,7 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), - ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3") ) // Exclude rules for 3.3.x from 3.2.0 From bd0cca18a140d5023b210bb8fce796aa60e2f5de Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 31 Aug 2022 19:21:27 +0800 Subject: [PATCH 05/17] catalyst module --- project/MimaExcludes.scala | 44 +++++++++++++++----------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 0cbac44217d40..3af87db79f609 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -64,55 +64,45 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.this"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.apply"), - ProblemFilters.exclude[MissingClassProblem]("org.apache.parquet.io.ColumnIOUtil"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.indexExists"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.indexExists"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex"), - - ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), - ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3") + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.parquet.io.ColumnIOUtil") ) // Exclude rules for 3.3.x from 3.2.0 From b43c8e33ad80ffc35c7cf4895c9bbe4478e8118e Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 31 Aug 2022 20:23:53 +0800 Subject: [PATCH 06/17] classify by Jira number and add comments --- project/MimaExcludes.scala | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 3af87db79f609..dc62677cc49a5 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -34,7 +34,7 @@ import com.typesafe.tools.mima.core.ProblemFilters._ */ object MimaExcludes { - // Exclude rules for 3.4.x + // Exclude rules for 3.4.x from 3.3.0 lazy val v34excludes = v33excludes ++ Seq( ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALS.checkedCast"), ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALSModel.checkedCast"), @@ -64,44 +64,51 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.this"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.apply"), + // [SPARK-38929][SQL] Improve error messages for cast failures in ANSI + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), + + // [SPARK-39704][SQL] Implement createIndex & dropIndex & indexExists in JDBC (H2 dialect) ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.dropIndex"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.indexExists"), + + // [SPARK-39759][SQL] Implement listIndexes in JDBC (H2 dialect) + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.listIndexes"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.listIndexes"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), - ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), + + // [SPARK-36511][MINOR][SQL] Remove ColumnIOUtil ProblemFilters.exclude[MissingClassProblem]("org.apache.parquet.io.ColumnIOUtil") ) From f2ed5d8118062bb382a1bf8631db3fedad5dfc92 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 08:29:04 +0800 Subject: [PATCH 07/17] remove outdated --- project/MimaExcludes.scala | 88 +------------------------------------- 1 file changed, 1 insertion(+), 87 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index dc62677cc49a5..0340fc34c830e 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -113,7 +113,7 @@ object MimaExcludes { ) // Exclude rules for 3.3.x from 3.2.0 - lazy val v33excludes = v32excludes ++ Seq( + lazy val v33excludes = Seq( // [SPARK-35672][CORE][YARN] Pass user classpath entries to executors using config instead of command line // The followings are necessary for Scala 2.13. ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.CoarseGrainedExecutorBackend#Arguments.*"), @@ -149,95 +149,9 @@ object MimaExcludes { ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI") ) - // Exclude rules for 3.2.x from 3.1.1 - lazy val v32excludes = Seq( - // Spark Internals - ProblemFilters.exclude[Problem]("org.apache.spark.rpc.*"), - ProblemFilters.exclude[Problem]("org.spark-project.jetty.*"), - ProblemFilters.exclude[Problem]("org.spark_project.jetty.*"), - ProblemFilters.exclude[Problem]("org.sparkproject.jetty.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.internal.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.unused.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.unsafe.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.memory.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.util.collection.unsafe.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.sql.catalyst.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), - // DSv2 catalog and expression APIs are unstable yet. We should enable this back. - ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"), - // Avro source implementation is internal. - ProblemFilters.exclude[Problem]("org.apache.spark.sql.v2.avro.*"), - - // [SPARK-34848][CORE] Add duration to TaskMetricDistributions - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.this"), - - // [SPARK-34488][CORE] Support task Metrics Distributions and executor Metrics Distributions - // in the REST API call for a specified stage - ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.status.api.v1.StageData.this"), - - // [SPARK-36173][CORE] Support getting CPU number in TaskContext - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.cpus"), - - // [SPARK-38679][CORE] Expose the number of partitions in a stage to TaskContext - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.numPartitions"), - - // [SPARK-35896] Include more granular metrics for stateful operators in StreamingQueryProgress - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StateOperatorProgress.this"), - - (problem: Problem) => problem match { - case MissingClassProblem(cls) => !cls.fullName.startsWith("org.sparkproject.jpmml") && - !cls.fullName.startsWith("org.sparkproject.dmg.pmml") - case _ => true - }, - - // [SPARK-33808][SQL] DataSource V2: Build logical writes in the optimizer - ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.connector.write.V1WriteBuilder"), - - // [SPARK-33955] Add latest offsets to source progress - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.SourceProgress.this"), - - // [SPARK-34862][SQL] Support nested column in ORC vectorized reader - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getBoolean"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getByte"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getShort"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getInt"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getLong"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getFloat"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getDouble"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getDecimal"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getUTF8String"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getBinary"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getArray"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getMap"), - ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.vectorized.ColumnVector.getChild"), - - // [SPARK-35135][CORE] Turn WritablePartitionedIterator from trait into a default implementation class - ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.util.collection.WritablePartitionedIterator"), - - // [SPARK-35757][CORE] Add bitwise AND operation and functionality for intersecting bloom filters - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.util.sketch.BloomFilter.intersectInPlace"), - - // [SPARK-35276][CORE] Calculate checksum for shuffle data and write as checksum file - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.shuffle.sort.io.LocalDiskShuffleMapOutputWriter.commitAllPartitions"), - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.shuffle.sort.io.LocalDiskSingleSpillMapOutputWriter.transferMapSpillFile"), - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.shuffle.api.ShuffleMapOutputWriter.commitAllPartitions"), - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.shuffle.api.SingleSpillShuffleMapOutputWriter.transferMapSpillFile"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.shuffle.api.SingleSpillShuffleMapOutputWriter.transferMapSpillFile"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.shuffle.api.ShuffleMapOutputWriter.commitAllPartitions"), - - // [SPARK-39506] In terms of 3 layer namespace effort, add currentCatalog, setCurrentCatalog and listCatalogs API to Catalog interface - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.currentCatalog"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs") - ) - def excludes(version: String) = version match { case v if v.startsWith("3.4") => v34excludes case v if v.startsWith("3.3") => v33excludes - case v if v.startsWith("3.2") => v32excludes case _ => Seq() } } From 5be1c1f178bfb17c3f1c7125a06e2e7804a5aa75 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 08:45:48 +0800 Subject: [PATCH 08/17] keep sparkInternalexcludes --- project/MimaExcludes.scala | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 0340fc34c830e..3575e4c3c510b 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -113,7 +113,7 @@ object MimaExcludes { ) // Exclude rules for 3.3.x from 3.2.0 - lazy val v33excludes = Seq( + lazy val v33excludes = sparkInternalexcludes ++ Seq( // [SPARK-35672][CORE][YARN] Pass user classpath entries to executors using config instead of command line // The followings are necessary for Scala 2.13. ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.CoarseGrainedExecutorBackend#Arguments.*"), @@ -149,6 +149,22 @@ object MimaExcludes { ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI") ) + lazy val sparkInternalexcludes = Seq( + // Spark Internals + ProblemFilters.exclude[Problem]("org.apache.spark.rpc.*"), + ProblemFilters.exclude[Problem]("org.spark-project.jetty.*"), + ProblemFilters.exclude[Problem]("org.spark_project.jetty.*"), + ProblemFilters.exclude[Problem]("org.sparkproject.jetty.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.internal.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.unused.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.unsafe.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.memory.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.util.collection.unsafe.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.sql.catalyst.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), + ) + def excludes(version: String) = version match { case v if v.startsWith("3.4") => v34excludes case v if v.startsWith("3.3") => v33excludes From 9182c4bfa9a91febc685556c5857286cfc903410 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 08:49:19 +0800 Subject: [PATCH 09/17] keep sparkInternalexcludes --- project/MimaExcludes.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 3575e4c3c510b..80b687d13c4e9 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -149,6 +149,7 @@ object MimaExcludes { ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI") ) + // Exclude rules for 3.2.x from 3.1.1 lazy val sparkInternalexcludes = Seq( // Spark Internals ProblemFilters.exclude[Problem]("org.apache.spark.rpc.*"), @@ -163,6 +164,18 @@ object MimaExcludes { ProblemFilters.exclude[Problem]("org.apache.spark.sql.catalyst.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), + // DSv2 catalog and expression APIs are unstable yet. We should enable this back. + ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"), + // Avro source implementation is internal. + ProblemFilters.exclude[Problem]("org.apache.spark.sql.v2.avro.*"), + + (problem: Problem) => problem match { + case MissingClassProblem(cls) => !cls.fullName.startsWith("org.sparkproject.jpmml") && + !cls.fullName.startsWith("org.sparkproject.dmg.pmml") + case _ => true + } ) def excludes(version: String) = version match { From d843a217a50bea9e7c88fe3597dd74da4ff0e068 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 09:06:15 +0800 Subject: [PATCH 10/17] revert --- project/MimaExcludes.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 80b687d13c4e9..25076f4da48af 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -149,7 +149,6 @@ object MimaExcludes { ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI") ) - // Exclude rules for 3.2.x from 3.1.1 lazy val sparkInternalexcludes = Seq( // Spark Internals ProblemFilters.exclude[Problem]("org.apache.spark.rpc.*"), @@ -175,7 +174,12 @@ object MimaExcludes { case MissingClassProblem(cls) => !cls.fullName.startsWith("org.sparkproject.jpmml") && !cls.fullName.startsWith("org.sparkproject.dmg.pmml") case _ => true - } + }, + + // [SPARK-39506] In terms of 3 layer namespace effort, add currentCatalog, setCurrentCatalog and listCatalogs API to Catalog interface + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.currentCatalog"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs") ) def excludes(version: String) = version match { From d71251859a36d564d5bf9633a9b3f36e565129ec Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 09:15:15 +0800 Subject: [PATCH 11/17] correct SPARK-38679 and SPARK-39506 --- project/MimaExcludes.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 25076f4da48af..60b6a4b9cfdcf 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -64,6 +64,14 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.this"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.deploy.DeployMessages#RequestExecutors.apply"), + // [SPARK-38679][CORE] Expose the number of partitions in a stage to TaskContext + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.numPartitions"), + + // [SPARK-39506] In terms of 3 layer namespace effort, add currentCatalog, setCurrentCatalog and listCatalogs API to Catalog interface + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.currentCatalog"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs"), + // [SPARK-38929][SQL] Improve error messages for cast failures in ANSI ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), @@ -174,12 +182,7 @@ object MimaExcludes { case MissingClassProblem(cls) => !cls.fullName.startsWith("org.sparkproject.jpmml") && !cls.fullName.startsWith("org.sparkproject.dmg.pmml") case _ => true - }, - - // [SPARK-39506] In terms of 3 layer namespace effort, add currentCatalog, setCurrentCatalog and listCatalogs API to Catalog interface - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.currentCatalog"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs") + } ) def excludes(version: String) = version match { From 31ad5591d245954cdc8b27f8b589baf8531dbf14 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 10:50:47 +0800 Subject: [PATCH 12/17] rename to defaultExcludes --- project/MimaExcludes.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 60b6a4b9cfdcf..84ccede0e353b 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -121,7 +121,7 @@ object MimaExcludes { ) // Exclude rules for 3.3.x from 3.2.0 - lazy val v33excludes = sparkInternalexcludes ++ Seq( + lazy val v33excludes = defaultExcludes ++ Seq( // [SPARK-35672][CORE][YARN] Pass user classpath entries to executors using config instead of command line // The followings are necessary for Scala 2.13. ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.CoarseGrainedExecutorBackend#Arguments.*"), @@ -157,7 +157,8 @@ object MimaExcludes { ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI") ) - lazy val sparkInternalexcludes = Seq( + // Defulat exclude rules + lazy val defaultExcludes = Seq( // Spark Internals ProblemFilters.exclude[Problem]("org.apache.spark.rpc.*"), ProblemFilters.exclude[Problem]("org.spark-project.jetty.*"), From c2d9f3724914eb1706db010ad7daa91643f8ef28 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 11:37:21 +0800 Subject: [PATCH 13/17] add jdbc to default excludes --- project/MimaExcludes.scala | 42 ++------------------------------------ 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 84ccede0e353b..0b1827bfd2148 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -76,46 +76,6 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), - // [SPARK-39704][SQL] Implement createIndex & dropIndex & indexExists in JDBC (H2 dialect) - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.indexExists"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.createIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.dropIndex"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.indexExists"), - - // [SPARK-39759][SQL] Implement listIndexes in JDBC (H2 dialect) - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DB2Dialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.DerbyDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.MySQLDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.NoopDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.OracleDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.PostgresDialect.listIndexes"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.TeradataDialect.listIndexes"), - // [SPARK-36511][MINOR][SQL] Remove ColumnIOUtil ProblemFilters.exclude[MissingClassProblem]("org.apache.parquet.io.ColumnIOUtil") ) @@ -173,6 +133,8 @@ object MimaExcludes { ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), + // SPARK-40283: add jdbc as default excludes + ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.*"), // DSv2 catalog and expression APIs are unstable yet. We should enable this back. ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"), From e20d2d7809ca207186e7b57c30bab76ac00c1b87 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 12:23:09 +0800 Subject: [PATCH 14/17] remove 33 --- project/MimaExcludes.scala | 48 +++----------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 0b1827bfd2148..877ed8b94961a 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -35,7 +35,7 @@ import com.typesafe.tools.mima.core.ProblemFilters._ object MimaExcludes { // Exclude rules for 3.4.x from 3.3.0 - lazy val v34excludes = v33excludes ++ Seq( + lazy val v34excludes = defaultExcludes ++ Seq( ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALS.checkedCast"), ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALSModel.checkedCast"), @@ -67,11 +67,6 @@ object MimaExcludes { // [SPARK-38679][CORE] Expose the number of partitions in a stage to TaskContext ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.numPartitions"), - // [SPARK-39506] In terms of 3 layer namespace effort, add currentCatalog, setCurrentCatalog and listCatalogs API to Catalog interface - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.currentCatalog"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs"), - // [SPARK-38929][SQL] Improve error messages for cast failures in ANSI ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), @@ -80,43 +75,6 @@ object MimaExcludes { ProblemFilters.exclude[MissingClassProblem]("org.apache.parquet.io.ColumnIOUtil") ) - // Exclude rules for 3.3.x from 3.2.0 - lazy val v33excludes = defaultExcludes ++ Seq( - // [SPARK-35672][CORE][YARN] Pass user classpath entries to executors using config instead of command line - // The followings are necessary for Scala 2.13. - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.CoarseGrainedExecutorBackend#Arguments.*"), - ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.executor.CoarseGrainedExecutorBackend#Arguments.*"), - ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.executor.CoarseGrainedExecutorBackend$Arguments$"), - - // [SPARK-37391][SQL] JdbcConnectionProvider tells if it modifies security context - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.jdbc.JdbcConnectionProvider.modifiesSecurityContext"), - - // [SPARK-37780][SQL] QueryExecutionListener support SQLConf as constructor parameter - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.util.ExecutionListenerManager.this"), - // [SPARK-37786][SQL] StreamingQueryListener support use SQLConf.get to get corresponding SessionState's SQLConf - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryManager.this"), - // [SPARK-38432][SQL] Reactor framework so as JDBC dialect could compile filter by self way - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.Filter.toV2"), - - // [SPARK-37831][CORE] Add task partition id in TaskInfo and Task Metrics - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskData.this"), - - // [SPARK-37600][BUILD] Upgrade to Hadoop 3.3.2 - ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Compressor"), - ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Factory"), - ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4SafeDecompressor"), - - // [SPARK-37377][SQL] Initial implementation of Storage-Partitioned Join - ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.connector.read.partitioning.ClusteredDistribution"), - ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.connector.read.partitioning.Distribution"), - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.connector.read.partitioning.Partitioning.*"), - ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.connector.read.partitioning.Partitioning.*"), - - // [SPARK-38908][SQL] Provide query context in runtime error of Casting from String to - // Number/Date/Timestamp/Boolean - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI") - ) - // Defulat exclude rules lazy val defaultExcludes = Seq( // Spark Internals @@ -133,8 +91,9 @@ object MimaExcludes { ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), - // SPARK-40283: add jdbc as default excludes + // SPARK-40283: add jdbc and catalog as default excludes ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.*"), + ProblemFilters.exclude[Problem]("org.apache.spark.sql.catalog.*"), // DSv2 catalog and expression APIs are unstable yet. We should enable this back. ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"), @@ -150,7 +109,6 @@ object MimaExcludes { def excludes(version: String) = version match { case v if v.startsWith("3.4") => v34excludes - case v if v.startsWith("3.3") => v33excludes case _ => Seq() } } From c9c4b4cd76cf1c9d7b5b086e262111b52e7fc718 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 12:24:44 +0800 Subject: [PATCH 15/17] revert --- project/MimaExcludes.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 877ed8b94961a..45661fb940633 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -67,6 +67,11 @@ object MimaExcludes { // [SPARK-38679][CORE] Expose the number of partitions in a stage to TaskContext ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.numPartitions"), + // [SPARK-39506] In terms of 3 layer namespace effort, add currentCatalog, setCurrentCatalog and listCatalogs API to Catalog interface + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.currentCatalog"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs"), + // [SPARK-38929][SQL] Improve error messages for cast failures in ANSI ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), @@ -91,9 +96,8 @@ object MimaExcludes { ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), - // SPARK-40283: add jdbc and catalog as default excludes + // SPARK-40283: add jdbc as default excludes ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.*"), - ProblemFilters.exclude[Problem]("org.apache.spark.sql.catalog.*"), // DSv2 catalog and expression APIs are unstable yet. We should enable this back. ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"), From ad543c057bed529e36172e1e6e463b9c2c71369d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 1 Sep 2022 15:11:33 +0800 Subject: [PATCH 16/17] add moduleSymbol.isPrivate --- project/MimaExcludes.scala | 4 ++-- .../scala/org/apache/spark/tools/GenerateMIMAIgnore.scala | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 45661fb940633..ca960fd946b55 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -96,8 +96,8 @@ object MimaExcludes { ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), - // SPARK-40283: add jdbc as default excludes - ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.*"), + // SPARK-40283: add jdbc.JdbcDialect as default excludes + ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.JdbcDialect.*"), // DSv2 catalog and expression APIs are unstable yet. We should enable this back. ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"), diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala index ef28095850bad..a46a7fbeec497 100644 --- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala +++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala @@ -65,7 +65,8 @@ object GenerateMIMAIgnore { val directlyPrivateSpark = isPackagePrivate(classSymbol) || isPackagePrivateModule(moduleSymbol) || - classSymbol.isPrivate + classSymbol.isPrivate || + moduleSymbol.isPrivate /* Inner classes defined within a private[spark] class or object are effectively invisible, so we account for them as package private. */ lazy val indirectlyPrivateSpark = { From 34fb8c4d2e56cb3244ce8c1ffa7cb927fbe4c204 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sat, 3 Sep 2022 00:58:07 +0800 Subject: [PATCH 17/17] explicitly track JdbcDialect --- project/MimaExcludes.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index ca960fd946b55..939fa9a9f45d2 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -72,6 +72,14 @@ object MimaExcludes { ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.setCurrentCatalog"), ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.listCatalogs"), + // [SPARK-39704][SQL] Implement createIndex & dropIndex & indexExists in JDBC (H2 dialect) + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.createIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.dropIndex"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.indexExists"), + + // [SPARK-39759][SQL] Implement listIndexes in JDBC (H2 dialect) + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.jdbc.JdbcDialect.listIndexes"), + // [SPARK-38929][SQL] Improve error messages for cast failures in ANSI ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"), @@ -96,8 +104,6 @@ object MimaExcludes { ProblemFilters.exclude[Problem]("org.apache.spark.sql.execution.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.internal.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.errors.*"), - // SPARK-40283: add jdbc.JdbcDialect as default excludes - ProblemFilters.exclude[Problem]("org.apache.spark.sql.jdbc.JdbcDialect.*"), // DSv2 catalog and expression APIs are unstable yet. We should enable this back. ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.catalog.*"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.connector.expressions.*"),