From 626874f35891c8de51d5f7b53a03d11acb3ad7a1 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 22 Apr 2024 18:49:03 +0800 Subject: [PATCH 1/2] [SPARK-47938][SQL] MsSQLServer: Cannot find data type BYTE error --- .../spark/sql/jdbc/MsSqlServerIntegrationSuite.scala | 9 +++++++++ .../org/apache/spark/sql/jdbc/MsSqlServerDialect.scala | 1 + 2 files changed, 10 insertions(+) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala index 8bceb9506e850..a16ddf6dc545a 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala @@ -437,4 +437,13 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite { .load() assert(df.collect().toSet === expectedResult) } + + + test("SPARK-47938: Fix 'Cannot find data type BYTE' in SQL Server") { + spark.sql("select cast(1 as byte) as c0") + .write + .jdbc(jdbcUrl, "test_byte", new Properties) + val df = spark.read.jdbc(jdbcUrl, "test_byte", new Properties) + checkAnswer(df, Row(1.toShort)) + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala index 862e99adc3b0d..1d05c0d7c24ed 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala @@ -136,6 +136,7 @@ private case class MsSqlServerDialect() extends JdbcDialect { case BinaryType => Some(JdbcType("VARBINARY(MAX)", java.sql.Types.VARBINARY)) case ShortType if !SQLConf.get.legacyMsSqlServerNumericMappingEnabled => Some(JdbcType("SMALLINT", java.sql.Types.SMALLINT)) + case ByteType => Some(JdbcType("SMALLINT", java.sql.Types.TINYINT)) case _ => None } From cfe29b9216faadabc24ebd023c12431a61f3b4e9 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 22 Apr 2024 22:30:33 -0700 Subject: [PATCH 2/2] Remove empty line --- .../org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala index a16ddf6dc545a..273e8c35dd073 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala @@ -438,7 +438,6 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite { assert(df.collect().toSet === expectedResult) } - test("SPARK-47938: Fix 'Cannot find data type BYTE' in SQL Server") { spark.sql("select cast(1 as byte) as c0") .write