Skip to content

Commit ce6661f

Browse files
heyihongheyihong
committed
[SPARK-53438] Fix Product type handling in convertToCatalyst
1 parent a74d50b commit ce6661f

File tree

6 files changed

+138
-4
lines changed

6 files changed

+138
-4
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,7 @@ object CatalystTypeConverters {
582582
case seq: scala.collection.mutable.ArraySeq[_] => convertToCatalyst(seq.array)
583583
case seq: Seq[Any] => new GenericArrayData(seq.map(convertToCatalyst).toArray)
584584
case r: Row => InternalRow(r.toSeq.map(convertToCatalyst): _*)
585+
case p: Product => InternalRow(p.productIterator.toSeq.map(convertToCatalyst): _*)
585586
case arr: Array[Byte] => arr
586587
case arr: Array[Char] => StringConverter.toCatalyst(arr)
587588
case arr: Array[_] => new GenericArrayData(arr.map(convertToCatalyst))

‎sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3414,6 +3414,7 @@ class PlanGenerationTestSuite
34143414
fn.typedlit[collection.immutable.Map[Int, Option[Int]]](
34153415
collection.immutable.Map(1 -> None)),
34163416
fn.typedLit(Seq(Seq(1, 2, 3), Seq(4, 5, 6), Seq(7, 8, 9))),
3417+
fn.typedLit(Seq((1, "2", Seq("3", "4")), (5, "6", Seq.empty[String]))),
34173418
fn.typedLit(
34183419
Seq(
34193420
mutable.LinkedHashMap("a" -> 1, "b" -> 2),

‎sql/connect/common/src/test/resources/query-tests/explain-results/function_typedLit.explain‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
Project [id#0L, id#0L, 1 AS 1#0, null AS NULL#0, true AS true#0, 68 AS 68#0, 9872 AS 9872#0, -8726532 AS -8726532#0, 7834609328726532 AS 7834609328726532#0L, 2.718281828459045 AS 2.718281828459045#0, -0.8 AS -0.8#0, 89.97620 AS 89.97620#0, 89889.7667231 AS 89889.7667231#0, connect! AS connect!#0, T AS T#0, ABCDEFGHIJ AS ABCDEFGHIJ#0, 0x78797A7B7C7D7E7F808182838485868788898A8B8C8D8E AS X'78797A7B7C7D7E7F808182838485868788898A8B8C8D8E'#0, 0x0806 AS X'0806'#0, [8,6] AS ARRAY(8, 6)#0, null AS NULL#0, 2020-10-10 AS DATE '2020-10-10'#0, 8.997620 AS 8.997620#0, 2023-02-23 04:31:59.808 AS TIMESTAMP '2023-02-23 04:31:59.808'#0, 1969-12-31 16:00:12.345 AS TIMESTAMP '1969-12-31 16:00:12.345'#0, 2023-02-23 20:36:00 AS TIMESTAMP_NTZ '2023-02-23 20:36:00'#0, 2023-02-23 AS DATE '2023-02-23'#0, INTERVAL '0 00:03:20' DAY TO SECOND AS INTERVAL '0 00:03:20' DAY TO SECOND#0, INTERVAL '0-0' YEAR TO MONTH AS INTERVAL '0-0' YEAR TO MONTH#0, 23:59:59.999999999 AS TIME '23:59:59.999999999'#0, 2 months 20 days 0.0001 seconds AS INTERVAL '2 months 20 days 0.0001 seconds'#0, 1 AS 1#0, [1,2,3] AS ARRAY(1, 2, 3)#0, [1,2,3] AS ARRAY(1, 2, 3)#0, map(keys: [a,b], values: [1,2]) AS MAP('a', 1, 'b', 2)#0, [a,2,1.0] AS NAMED_STRUCT('_1', 'a', '_2', 2, '_3', 1.0D)#0, null AS NULL#0, [1] AS ARRAY(1)#0, map(keys: [1], values: [0]) AS MAP(1, 0)#0, map(keys: [1], values: [0]) AS MAP(1, 0)#0, map(keys: [1], values: [0]) AS MAP(1, 0)#0, [[1,2,3],[4,5,6],[7,8,9]] AS ARRAY(ARRAY(1, 2, 3), ARRAY(4, 5, 6), ARRAY(7, 8, 9))#0, [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4],keys: [a,b], values: [5,6]] AS ARRAY(MAP('a', 1, 'b', 2), MAP('a', 3, 'b', 4), MAP('a', 5, 'b', 6))#0, map(keys: [1,2], values: [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4]]) AS MAP(1, MAP('a', 1, 'b', 2), 2, MAP('a', 3, 'b', 4))#0, [[1,2,3],keys: [a,b], values: [1,2],[a,keys: [1,2], values: [a,b]]] AS NAMED_STRUCT('_1', ARRAY(1, 2, 3), '_2', MAP('a', 1, 'b', 2), '_3', NAMED_STRUCT('_1', 'a', '_2', MAP(1, 'a', 2, 'b')))#0]
1+
Project [id#0L, id#0L, 1 AS 1#0, null AS NULL#0, true AS true#0, 68 AS 68#0, 9872 AS 9872#0, -8726532 AS -8726532#0, 7834609328726532 AS 7834609328726532#0L, 2.718281828459045 AS 2.718281828459045#0, -0.8 AS -0.8#0, 89.97620 AS 89.97620#0, 89889.7667231 AS 89889.7667231#0, connect! AS connect!#0, T AS T#0, ABCDEFGHIJ AS ABCDEFGHIJ#0, 0x78797A7B7C7D7E7F808182838485868788898A8B8C8D8E AS X'78797A7B7C7D7E7F808182838485868788898A8B8C8D8E'#0, 0x0806 AS X'0806'#0, [8,6] AS ARRAY(8, 6)#0, null AS NULL#0, 2020-10-10 AS DATE '2020-10-10'#0, 8.997620 AS 8.997620#0, 2023-02-23 04:31:59.808 AS TIMESTAMP '2023-02-23 04:31:59.808'#0, 1969-12-31 16:00:12.345 AS TIMESTAMP '1969-12-31 16:00:12.345'#0, 2023-02-23 20:36:00 AS TIMESTAMP_NTZ '2023-02-23 20:36:00'#0, 2023-02-23 AS DATE '2023-02-23'#0, INTERVAL '0 00:03:20' DAY TO SECOND AS INTERVAL '0 00:03:20' DAY TO SECOND#0, INTERVAL '0-0' YEAR TO MONTH AS INTERVAL '0-0' YEAR TO MONTH#0, 23:59:59.999999999 AS TIME '23:59:59.999999999'#0, 2 months 20 days 0.0001 seconds AS INTERVAL '2 months 20 days 0.0001 seconds'#0, 1 AS 1#0, [1,2,3] AS ARRAY(1, 2, 3)#0, [1,2,3] AS ARRAY(1, 2, 3)#0, map(keys: [a,b], values: [1,2]) AS MAP('a', 1, 'b', 2)#0, [a,2,1.0] AS NAMED_STRUCT('_1', 'a', '_2', 2, '_3', 1.0D)#0, null AS NULL#0, [1] AS ARRAY(1)#0, map(keys: [1], values: [0]) AS MAP(1, 0)#0, map(keys: [1], values: [0]) AS MAP(1, 0)#0, map(keys: [1], values: [0]) AS MAP(1, 0)#0, [[1,2,3],[4,5,6],[7,8,9]] AS ARRAY(ARRAY(1, 2, 3), ARRAY(4, 5, 6), ARRAY(7, 8, 9))#0, [[1,2,[3,4]],[5,6,[]]] AS ARRAY(NAMED_STRUCT('_1', 1, '_2', '2', '_3', ARRAY('3', '4')), NAMED_STRUCT('_1', 5, '_2', '6', '_3', ARRAY()))#0, [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4],keys: [a,b], values: [5,6]] AS ARRAY(MAP('a', 1, 'b', 2), MAP('a', 3, 'b', 4), MAP('a', 5, 'b', 6))#0, map(keys: [1,2], values: [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4]]) AS MAP(1, MAP('a', 1, 'b', 2), 2, MAP('a', 3, 'b', 4))#0, [[1,2,3],keys: [a,b], values: [1,2],[a,keys: [1,2], values: [a,b]]] AS NAMED_STRUCT('_1', ARRAY(1, 2, 3), '_2', MAP('a', 1, 'b', 2), '_3', NAMED_STRUCT('_1', 'a', '_2', MAP(1, 'a', 2, 'b')))#0]
22
+- LocalRelation <empty>, [id#0L, a#0, b#0]

‎sql/connect/common/src/test/resources/query-tests/queries/function_typedLit.json‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1120,6 +1120,139 @@
11201120
}
11211121
}
11221122
}
1123+
}, {
1124+
"literal": {
1125+
"array": {
1126+
"elements": [{
1127+
"struct": {
1128+
"elements": [{
1129+
"integer": 1
1130+
}, {
1131+
"string": "2"
1132+
}, {
1133+
"array": {
1134+
"elements": [{
1135+
"string": "3"
1136+
}, {
1137+
"string": "4"
1138+
}],
1139+
"dataType": {
1140+
"elementType": {
1141+
"string": {
1142+
"collation": "UTF8_BINARY"
1143+
}
1144+
},
1145+
"containsNull": true
1146+
}
1147+
}
1148+
}],
1149+
"dataTypeStruct": {
1150+
"fields": [{
1151+
"name": "_1"
1152+
}, {
1153+
"name": "_2",
1154+
"dataType": {
1155+
"string": {
1156+
"collation": "UTF8_BINARY"
1157+
}
1158+
},
1159+
"nullable": true
1160+
}, {
1161+
"name": "_3",
1162+
"nullable": true
1163+
}]
1164+
}
1165+
}
1166+
}, {
1167+
"struct": {
1168+
"elements": [{
1169+
"integer": 5
1170+
}, {
1171+
"string": "6"
1172+
}, {
1173+
"array": {
1174+
"dataType": {
1175+
"elementType": {
1176+
"string": {
1177+
"collation": "UTF8_BINARY"
1178+
}
1179+
},
1180+
"containsNull": true
1181+
}
1182+
}
1183+
}],
1184+
"dataTypeStruct": {
1185+
"fields": [{
1186+
"name": "_1"
1187+
}, {
1188+
"name": "_2",
1189+
"dataType": {
1190+
"string": {
1191+
"collation": "UTF8_BINARY"
1192+
}
1193+
},
1194+
"nullable": true
1195+
}, {
1196+
"name": "_3",
1197+
"nullable": true
1198+
}]
1199+
}
1200+
}
1201+
}],
1202+
"dataType": {
1203+
"elementType": {
1204+
"struct": {
1205+
"fields": [{
1206+
"name": "_1",
1207+
"dataType": {
1208+
"integer": {
1209+
}
1210+
}
1211+
}, {
1212+
"name": "_2",
1213+
"dataType": {
1214+
"string": {
1215+
"collation": "UTF8_BINARY"
1216+
}
1217+
},
1218+
"nullable": true
1219+
}, {
1220+
"name": "_3",
1221+
"dataType": {
1222+
"array": {
1223+
"elementType": {
1224+
"string": {
1225+
"collation": "UTF8_BINARY"
1226+
}
1227+
},
1228+
"containsNull": true
1229+
}
1230+
},
1231+
"nullable": true
1232+
}]
1233+
}
1234+
},
1235+
"containsNull": true
1236+
}
1237+
}
1238+
},
1239+
"common": {
1240+
"origin": {
1241+
"jvmOrigin": {
1242+
"stackTrace": [{
1243+
"classLoaderName": "app",
1244+
"declaringClass": "org.apache.spark.sql.functions$",
1245+
"methodName": "typedLit",
1246+
"fileName": "functions.scala"
1247+
}, {
1248+
"classLoaderName": "app",
1249+
"declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite",
1250+
"methodName": "~~trimmed~anonfun~~",
1251+
"fileName": "PlanGenerationTestSuite.scala"
1252+
}]
1253+
}
1254+
}
1255+
}
11231256
}, {
11241257
"literal": {
11251258
"array": {

‎sql/connect/common/src/test/resources/query-tests/queries/function_typedLit.proto.bin‎

440 Bytes
Binary file not shown.

‎sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/LiteralExpressionProtoConverter.scala‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.apache.spark.sql.connect.planner
1919

2020
import org.apache.spark.connect.proto
21-
import org.apache.spark.sql.catalyst.{expressions, CatalystTypeConverters}
21+
import org.apache.spark.sql.catalyst.expressions
2222
import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, InvalidPlanInput, LiteralValueProtoConverter}
2323
import org.apache.spark.sql.types._
2424
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -126,8 +126,7 @@ object LiteralExpressionProtoConverter {
126126
proto.DataType.newBuilder
127127
.setStruct(LiteralValueProtoConverter.getProtoStructType(lit.getStruct))
128128
.build())
129-
val convert = CatalystTypeConverters.createToCatalystConverter(dataType)
130-
expressions.Literal(convert(structData), dataType)
129+
expressions.Literal.create(structData, dataType)
131130

132131
case _ =>
133132
throw InvalidPlanInput(

0 commit comments

Comments
 (0)