Skip to content

Commit 0130897

Browse files
committed
Fixes Parquet comparison filter generation
1 parent 36b0956 commit 0130897

File tree

2 files changed

+16
-4
lines changed

2 files changed

+16
-4
lines changed

sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetFilters.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,22 +135,22 @@ private[sql] object ParquetFilters {
135135
case LessThan(NamedExpression(name, _), Literal(value, dataType)) =>
136136
makeLt.lift(dataType).map(_(name, value))
137137
case LessThan(Literal(value, dataType), NamedExpression(name, _)) =>
138-
makeLt.lift(dataType).map(_(name, value))
138+
makeGt.lift(dataType).map(_(name, value))
139139

140140
case LessThanOrEqual(NamedExpression(name, _), Literal(value, dataType)) =>
141141
makeLtEq.lift(dataType).map(_(name, value))
142142
case LessThanOrEqual(Literal(value, dataType), NamedExpression(name, _)) =>
143-
makeLtEq.lift(dataType).map(_(name, value))
143+
makeGtEq.lift(dataType).map(_(name, value))
144144

145145
case GreaterThan(NamedExpression(name, _), Literal(value, dataType)) =>
146146
makeGt.lift(dataType).map(_(name, value))
147147
case GreaterThan(Literal(value, dataType), NamedExpression(name, _)) =>
148-
makeGt.lift(dataType).map(_(name, value))
148+
makeLt.lift(dataType).map(_(name, value))
149149

150150
case GreaterThanOrEqual(NamedExpression(name, _), Literal(value, dataType)) =>
151151
makeGtEq.lift(dataType).map(_(name, value))
152152
case GreaterThanOrEqual(Literal(value, dataType), NamedExpression(name, _)) =>
153-
makeGtEq.lift(dataType).map(_(name, value))
153+
makeLtEq.lift(dataType).map(_(name, value))
154154

155155
case And(lhs, rhs) =>
156156
(createFilter(lhs) ++ createFilter(rhs)).reduceOption(FilterApi.and)

sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,9 +461,21 @@ class ParquetQuerySuite extends QueryTest with FunSuiteLike with BeforeAndAfterA
461461
}
462462

463463
checkFilter[Operators.Eq[Integer]]('a.int === 1)
464+
checkFilter[Operators.Eq[Integer]](Literal(1) === 'a.int)
465+
464466
checkFilter[Operators.Lt[Integer]]('a.int < 4)
467+
checkFilter[Operators.Lt[Integer]](Literal(4) > 'a.int)
468+
checkFilter[Operators.LtEq[Integer]]('a.int <= 4)
469+
checkFilter[Operators.LtEq[Integer]](Literal(4) >= 'a.int)
470+
471+
checkFilter[Operators.Gt[Integer]]('a.int > 4)
472+
checkFilter[Operators.Gt[Integer]](Literal(4) < 'a.int)
473+
checkFilter[Operators.GtEq[Integer]]('a.int >= 4)
474+
checkFilter[Operators.GtEq[Integer]](Literal(4) <= 'a.int)
475+
465476
checkFilter[Operators.And]('a.int === 1 && 'a.int < 4)
466477
checkFilter[Operators.Or]('a.int === 1 || 'a.int < 4)
478+
checkFilter[Operators.Not](!('a.int === 1))
467479

468480
checkFilter('a.int > 'b.int, defined = false)
469481
checkFilter(('a.int > 'b.int) && ('a.int > 'b.int), defined = false)

0 commit comments

Comments
 (0)