Skip to content

[SPARK-6776] [SQL] [WIP] More Parquet compatibility tests #7472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

[SPARK-6776] [SQL] [WIP] More Parquet compatibility tests #7472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3cb48ab
Tests for Avro enum values
liancheng Jul 17, 2015
e152ed6
Makes implicit conversions more explicit
liancheng Jul 17, 2015
3d69df3
Tests for nullable Avro complex types
liancheng Jul 17, 2015
4380470
Tests for Hive complex types
liancheng Jul 17, 2015
87b59ab
Tests for Avro array of struct
liancheng Jul 17, 2015
b1be549
Updates .gitignore and adds an Avro generated Java source file
liancheng Jul 18, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,3 +74,4 @@ metastore/
warehouse/
TempStatsStore/
sql/hive-thriftserver/test_warehouses
sql/core/src/test/gen-java/org/apache/spark/sql/parquet/test/thrift
15 changes: 15 additions & 0 deletions sql/core/src/test/avro/parquet-compat.avdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@
// This is a test protocol for testing parquet-avro compatibility.
@namespace("org.apache.spark.sql.parquet.test.avro")
protocol CompatibilityTest {
enum Suit {
SPADES,
HEARTS,
DIAMONDS,
CLUBS
}

record Nested {
array<int> nested_ints_column;
string nested_string_column;
Expand All @@ -31,6 +38,7 @@ protocol CompatibilityTest {
double double_column;
bytes binary_column;
string string_column;
Suit enum_column;

union { null, boolean } maybe_bool_column;
union { null, int } maybe_int_column;
Expand All @@ -39,9 +47,16 @@ protocol CompatibilityTest {
union { null, double } maybe_double_column;
union { null, bytes } maybe_binary_column;
union { null, string } maybe_string_column;
union { null, Suit } maybe_enum_column;

array<string> strings_column;
array<Nested> structs_column;
map<int> string_to_int_column;
map<array<Nested>> complex_column;

union { null, array<string> } maybe_strings_column;
union { null, array<Nested> } maybe_structs_column;
union { null, map<int> } maybe_string_to_int_column;
union { null, map<array<Nested>> } maybe_complex_column;
}
}
43 changes: 43 additions & 0 deletions sql/core/src/test/avro/parquet-compat.avpr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
"protocol" : "CompatibilityTest",
"namespace" : "org.apache.spark.sql.parquet.test.avro",
"types" : [ {
"type" : "enum",
"name" : "Suit",
"symbols" : [ "SPADES", "HEARTS", "DIAMONDS", "CLUBS" ]
}, {
"type" : "record",
"name" : "Nested",
"fields" : [ {
Expand Down Expand Up @@ -38,6 +42,9 @@
}, {
"name" : "string_column",
"type" : "string"
}, {
"name" : "enum_column",
"type" : "Suit"
}, {
"name" : "maybe_bool_column",
"type" : [ "null", "boolean" ]
Expand All @@ -59,12 +66,21 @@
}, {
"name" : "maybe_string_column",
"type" : [ "null", "string" ]
}, {
"name" : "maybe_enum_column",
"type" : [ "null", "Suit" ]
}, {
"name" : "strings_column",
"type" : {
"type" : "array",
"items" : "string"
}
}, {
"name" : "structs_column",
"type" : {
"type" : "array",
"items" : "Nested"
}
}, {
"name" : "string_to_int_column",
"type" : {
Expand All @@ -80,6 +96,33 @@
"items" : "Nested"
}
}
}, {
"name" : "maybe_strings_column",
"type" : [ "null", {
"type" : "array",
"items" : "string"
} ]
}, {
"name" : "maybe_structs_column",
"type" : [ "null", {
"type" : "array",
"items" : "Nested"
} ]
}, {
"name" : "maybe_string_to_int_column",
"type" : [ "null", {
"type" : "map",
"values" : "int"
} ]
}, {
"name" : "maybe_complex_column",
"type" : [ "null", {
"type" : "map",
"values" : {
"type" : "array",
"items" : "Nested"
}
} ]
} ]
} ],
"messages" : { }
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/gen-java/org/apache/spark/sql/parquet/test/avro/CompatibilityTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
@SuppressWarnings("all")
@org.apache.avro.specific.AvroGenerated
public interface CompatibilityTest {
public static final org.apache.avro.Protocol PROTOCOL = org.apache.avro.Protocol.parse("{\"protocol\":\"CompatibilityTest\",\"namespace\":\"org.apache.spark.sql.parquet.test.avro\",\"types\":[{\"type\":\"record\",\"name\":\"Nested\",\"fields\":[{\"name\":\"nested_ints_column\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"nested_string_column\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]},{\"type\":\"record\",\"name\":\"ParquetAvroCompat\",\"fields\":[{\"name\":\"bool_column\",\"type\":\"boolean\"},{\"name\":\"int_column\",\"type\":\"int\"},{\"name\":\"long_column\",\"type\":\"long\"},{\"name\":\"float_column\",\"type\":\"float\"},{\"name\":\"double_column\",\"type\":\"double\"},{\"name\":\"binary_column\",\"type\":\"bytes\"},{\"name\":\"string_column\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"maybe_bool_column\",\"type\":[\"null\",\"boolean\"]},{\"name\":\"maybe_int_column\",\"type\":[\"null\",\"int\"]},{\"name\":\"maybe_long_column\",\"type\":[\"null\",\"long\"]},{\"name\":\"maybe_float_column\",\"type\":[\"null\",\"float\"]},{\"name\":\"maybe_double_column\",\"type\":[\"null\",\"double\"]},{\"name\":\"maybe_binary_column\",\"type\":[\"null\",\"bytes\"]},{\"name\":\"maybe_string_column\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"strings_column\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}},{\"name\":\"string_to_int_column\",\"type\":{\"type\":\"map\",\"values\":\"int\",\"avro.java.string\":\"String\"}},{\"name\":\"complex_column\",\"type\":{\"type\":\"map\",\"values\":{\"type\":\"array\",\"items\":\"Nested\"},\"avro.java.string\":\"String\"}}]}],\"messages\":{}}");
public static final org.apache.avro.Protocol PROTOCOL = org.apache.avro.Protocol.parse("{\"protocol\":\"CompatibilityTest\",\"namespace\":\"org.apache.spark.sql.parquet.test.avro\",\"types\":[{\"type\":\"enum\",\"name\":\"Suit\",\"symbols\":[\"SPADES\",\"HEARTS\",\"DIAMONDS\",\"CLUBS\"]},{\"type\":\"record\",\"name\":\"Nested\",\"fields\":[{\"name\":\"nested_ints_column\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"nested_string_column\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]},{\"type\":\"record\",\"name\":\"ParquetAvroCompat\",\"fields\":[{\"name\":\"bool_column\",\"type\":\"boolean\"},{\"name\":\"int_column\",\"type\":\"int\"},{\"name\":\"long_column\",\"type\":\"long\"},{\"name\":\"float_column\",\"type\":\"float\"},{\"name\":\"double_column\",\"type\":\"double\"},{\"name\":\"binary_column\",\"type\":\"bytes\"},{\"name\":\"string_column\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"enum_column\",\"type\":\"Suit\"},{\"name\":\"maybe_bool_column\",\"type\":[\"null\",\"boolean\"]},{\"name\":\"maybe_int_column\",\"type\":[\"null\",\"int\"]},{\"name\":\"maybe_long_column\",\"type\":[\"null\",\"long\"]},{\"name\":\"maybe_float_column\",\"type\":[\"null\",\"float\"]},{\"name\":\"maybe_double_column\",\"type\":[\"null\",\"double\"]},{\"name\":\"maybe_binary_column\",\"type\":[\"null\",\"bytes\"]},{\"name\":\"maybe_string_column\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"maybe_enum_column\",\"type\":[\"null\",\"Suit\"]},{\"name\":\"strings_column\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}},{\"name\":\"structs_column\",\"type\":{\"type\":\"array\",\"items\":\"Nested\"}},{\"name\":\"string_to_int_column\",\"type\":{\"type\":\"map\",\"values\":\"int\",\"avro.java.string\":\"String\"}},{\"name\":\"complex_column\",\"type\":{\"type\":\"map\",\"values\":{\"type\":\"array\",\"items\":\"Nested\"},\"avro.java.string\":\"String\"}},{\"name\":\"maybe_strings_column\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]},{\"name\":\"maybe_structs_column\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"Nested\"}]},{\"name\":\"maybe_string_to_int_column\",\"type\":[\"null\",{\"type\":\"map\",\"values\":\"int\",\"avro.java.string\":\"String\"}]},{\"name\":\"maybe_complex_column\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"array\",\"items\":\"Nested\"},\"avro.java.string\":\"String\"}]}]}],\"messages\":{}}");

@SuppressWarnings("all")
public interface Callback extends CompatibilityTest {
Expand Down
Loading