Skip to content

Commit 49b5dd1

Browse files
belieferdongjoon-hyun
authored andcommitted
[SPARK-37492][SQL] Optimize Orc test code with withAllNativeOrcReaders
### What changes were proposed in this pull request? This PR used to optimize some Orc test code in OrcSourceSuite.scala ### Why are the changes needed? Improve code. ### Does this PR introduce _any_ user-facing change? 'No'. Just improve code for UT. ### How was this patch tested? Jenkins test. Closes #34723 from beliefer/optimize-orc. Authored-by: Jiaan Geng <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 98b0c80 commit 49b5dd1

File tree

1 file changed

+22
-29
lines changed

1 file changed

+22
-29
lines changed

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -485,12 +485,10 @@ abstract class OrcSuite
485485
}
486486

487487
test("SPARK-31238: compatibility with Spark 2.4 in reading dates") {
488-
Seq(false, true).foreach { vectorized =>
489-
withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
490-
checkAnswer(
491-
readResourceOrcFile("test-data/before_1582_date_v2_4.snappy.orc"),
492-
Row(java.sql.Date.valueOf("1200-01-01")))
493-
}
488+
withAllNativeOrcReaders {
489+
checkAnswer(
490+
readResourceOrcFile("test-data/before_1582_date_v2_4.snappy.orc"),
491+
Row(java.sql.Date.valueOf("1200-01-01")))
494492
}
495493
}
496494

@@ -502,23 +500,19 @@ abstract class OrcSuite
502500
.write
503501
.orc(path)
504502

505-
Seq(false, true).foreach { vectorized =>
506-
withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
507-
checkAnswer(
508-
spark.read.orc(path),
509-
Seq(Row(Date.valueOf("1001-01-01")), Row(Date.valueOf("1582-10-15"))))
510-
}
503+
withAllNativeOrcReaders {
504+
checkAnswer(
505+
spark.read.orc(path),
506+
Seq(Row(Date.valueOf("1001-01-01")), Row(Date.valueOf("1582-10-15"))))
511507
}
512508
}
513509
}
514510

515511
test("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") {
516-
Seq(false, true).foreach { vectorized =>
517-
withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
518-
checkAnswer(
519-
readResourceOrcFile("test-data/before_1582_ts_v2_4.snappy.orc"),
520-
Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
521-
}
512+
withAllNativeOrcReaders {
513+
checkAnswer(
514+
readResourceOrcFile("test-data/before_1582_ts_v2_4.snappy.orc"),
515+
Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
522516
}
523517
}
524518

@@ -530,14 +524,12 @@ abstract class OrcSuite
530524
.write
531525
.orc(path)
532526

533-
Seq(false, true).foreach { vectorized =>
534-
withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
535-
checkAnswer(
536-
spark.read.orc(path),
537-
Seq(
538-
Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")),
539-
Row(java.sql.Timestamp.valueOf("1582-10-15 11:12:13.654321"))))
540-
}
527+
withAllNativeOrcReaders {
528+
checkAnswer(
529+
spark.read.orc(path),
530+
Seq(
531+
Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")),
532+
Row(java.sql.Timestamp.valueOf("1582-10-15 11:12:13.654321"))))
541533
}
542534
}
543535
}
@@ -809,11 +801,12 @@ abstract class OrcSourceSuite extends OrcSuite with SharedSparkSession {
809801
}
810802
}
811803

812-
Seq(true, false).foreach { vecReaderEnabled =>
804+
withAllNativeOrcReaders {
813805
Seq(true, false).foreach { vecReaderNestedColEnabled =>
806+
val vecReaderEnabled = SQLConf.get.orcVectorizedReaderEnabled
814807
test("SPARK-36931: Support reading and writing ANSI intervals (" +
815-
s"${SQLConf.ORC_VECTORIZED_READER_ENABLED.key}=$vecReaderEnabled, " +
816-
s"${SQLConf.ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key}=$vecReaderNestedColEnabled)") {
808+
s"${SQLConf.ORC_VECTORIZED_READER_ENABLED.key}=$vecReaderEnabled, " +
809+
s"${SQLConf.ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key}=$vecReaderNestedColEnabled)") {
817810

818811
withSQLConf(
819812
SQLConf.ORC_VECTORIZED_READER_ENABLED.key ->

0 commit comments

Comments
 (0)