From acb4b067ed97834f782bd60cb726f68a78f90e26 Mon Sep 17 00:00:00 2001 From: zommerfelds Date: Thu, 2 Jun 2016 16:14:04 +0200 Subject: [PATCH 1/3] [SPARK-15642][SQL] Added test case --- .../org/apache/spark/sql/ColumnExpressionSuite.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala index e89fa32b15143..a1715b017fb06 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala @@ -120,6 +120,15 @@ class ColumnExpressionSuite extends QueryTest with SharedSQLContext { assert(newCol.expr.asInstanceOf[NamedExpression].metadata.getString("key") === "value") } + test("extracting from struct preserves metadata") { + val metadata = new MetadataBuilder().putString("key", "value").build() + + val df = spark.createDataFrame(sparkContext.parallelize(Seq(Row(Row("foo")))), + StructType(Seq(StructField("struct", StructType( + Seq(StructField("str", StringType, false, metadata))))))) + assert(df.select(df("struct")("str")).schema.fields(0).metadata === metadata) + } + test("single explode") { val df = Seq((1, Seq(1, 2, 3))).toDF("a", "intList") checkAnswer( From e745c48b80f1b5fbfbc953958652d6549ac90f5c Mon Sep 17 00:00:00 2001 From: zommerfelds Date: Thu, 2 Jun 2016 16:31:16 +0200 Subject: [PATCH 2/3] [SPARK-15642][SQL] Moved test to more appropriate file --- .../org/apache/spark/sql/ColumnExpressionSuite.scala | 9 --------- .../apache/spark/sql/DataFrameComplexTypeSuite.scala | 10 ++++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala index a1715b017fb06..e89fa32b15143 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala @@ -120,15 +120,6 @@ class ColumnExpressionSuite extends QueryTest with SharedSQLContext { assert(newCol.expr.asInstanceOf[NamedExpression].metadata.getString("key") === "value") } - test("extracting from struct preserves metadata") { - val metadata = new MetadataBuilder().putString("key", "value").build() - - val df = spark.createDataFrame(sparkContext.parallelize(Seq(Row(Row("foo")))), - StructType(Seq(StructField("struct", StructType( - Seq(StructField("str", StringType, false, metadata))))))) - assert(df.select(df("struct")("str")).schema.fields(0).metadata === metadata) - } - test("single explode") { val df = Seq((1, Seq(1, 2, 3))).toDF("a", "intList") checkAnswer( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala index 72f676e6225ee..53553cb20fc20 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSQLContext +import org.apache.spark.sql.types.{MetadataBuilder, StringType, StructField, StructType} /** * A test suite to test DataFrame/SQL functionalities with complex types (i.e. array, struct, map). @@ -58,4 +59,13 @@ class DataFrameComplexTypeSuite extends QueryTest with SharedSQLContext { val nullIntRow = df.selectExpr("i[1]").collect()(0) assert(nullIntRow == org.apache.spark.sql.Row(null)) } + + test("SPARK-15642 extracting from struct preserves metadata") { + val metadata = new MetadataBuilder().putString("key", "value").build() + + val df = spark.createDataFrame(sparkContext.parallelize(Seq(Row(Row("foo")))), + StructType(Seq(StructField("struct", StructType( + Seq(StructField("str", StringType, false, metadata))))))) + assert(df.select(df("struct")("str")).schema.fields(0).metadata === metadata) + } } From 1152690aed6f32c0b492f6b92fd86503c2f40e77 Mon Sep 17 00:00:00 2001 From: zommerfelds Date: Sat, 11 Feb 2017 11:56:07 -0500 Subject: [PATCH 3/3] fixed bad conflict resolution --- .../scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala index 31b848715dbad..3d48f9d095add 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala @@ -68,6 +68,7 @@ class DataFrameComplexTypeSuite extends QueryTest with SharedSQLContext { StructType(Seq(StructField("struct", StructType( Seq(StructField("str", StringType, false, metadata))))))) assert(df.select(df("struct")("str")).schema.fields(0).metadata === metadata) + } test("SPARK-15285 Generated SpecificSafeProjection.apply method grows beyond 64KB") { val ds100_5 = Seq(S100_5()).toDS()