Skip to content

Commit cf331a4

Browse files
committed
Correct existing tests
1 parent a55ad54 commit cf331a4

File tree

1 file changed

+9
-3
lines changed

1 file changed

+9
-3
lines changed

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources.parquet
1919

2020
import org.apache.parquet.filter2.predicate.Operators._
2121
import org.apache.parquet.filter2.predicate.{FilterPredicate, Operators}
22+
import org.apache.spark.sql.execution.PhysicalRDD
2223

2324
import org.apache.spark.sql.{Column, DataFrame, QueryTest, Row, SQLConf}
2425
import org.apache.spark.sql.catalyst.dsl.expressions._
@@ -75,7 +76,10 @@ class ParquetFilterSuite extends QueryTest with ParquetTest with SharedSQLContex
7576
assert(f.getClass === filterClass)
7677
}
7778
}
78-
checker(stripSparkFilter(query), expected)
79+
// Check if SparkPlan Filter is removed and this plan only has PhysicalRDD.
80+
val executedPlan = query.queryExecution.executedPlan
81+
assert(executedPlan.isInstanceOf[PhysicalRDD])
82+
checker(query, expected)
7983
}
8084
}
8185
}
@@ -325,7 +329,6 @@ class ParquetFilterSuite extends QueryTest with ParquetTest with SharedSQLContex
325329

326330
test("SPARK-11103: Filter applied on merged Parquet schema with new column fails") {
327331
import testImplicits._
328-
329332
withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
330333
SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true") {
331334
withTempPath { dir =>
@@ -353,11 +356,14 @@ class ParquetFilterSuite extends QueryTest with ParquetTest with SharedSQLContex
353356
(1 to 3).map(i => (i, i.toString)).toDF("a", "b").write.parquet(path)
354357
val df = sqlContext.read.parquet(path).filter("a = 2")
355358

359+
// Check if SparkPlan Filter is removed and this plan only has PhysicalRDD.
360+
val executedPlan = df.queryExecution.executedPlan
361+
assert(executedPlan.isInstanceOf[PhysicalRDD])
356362
// The result should be single row.
357363
// When a filter is pushed to Parquet, Parquet can apply it to every row.
358364
// So, we can check the number of rows returned from the Parquet
359365
// to make sure our filter pushdown work.
360-
assert(stripSparkFilter(df).count == 1)
366+
assert(df.count == 1)
361367
}
362368
}
363369
}

0 commit comments

Comments
 (0)