Skip to content

[SPARK-27604][SQL] Add filter reduction #24495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 12 commits into from
Closed

[SPARK-27604][SQL] Add filter reduction #24495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
dd09f51
[SPARK-27604][SQL] Enhance constant propagation to constraint propaga…
peter-toth Apr 28, 2019
d1ae18c
normalize only once
peter-toth Apr 30, 2019
8aa8e36
fix null handling regarding EqualNullSafe
peter-toth Apr 30, 2019
3737801
fix review findings
peter-toth May 3, 2019
9ee3be3
revert unnecessary changes, minor fix
peter-toth May 6, 2019
169563b
revert accidentally removed logic, add more tests
peter-toth May 7, 2019
3c982b6
use map of equality predicates in ConstantPropagation
peter-toth May 7, 2019
b0786e1
minor fix
peter-toth May 8, 2019
693cdbb
remove constant propagation changes
peter-toth May 8, 2019
9dc1422
fix review findings
peter-toth May 9, 2019
699fe2f
fix null issues
peter-toth May 10, 2019
0972a29
plantest fix
peter-toth May 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog)
TransposeWindow,
NullPropagation,
ConstantPropagation,
FilterReduction,
FoldablePropagation,
OptimizeIn,
ConstantFolding,
Expand Down
243 changes: 243 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,249 @@ object ConstantPropagation extends Rule[LogicalPlan] with PredicateHelper {
}
}

/**
* Substitutes expressions which can be statically reduced by constraints.
* eg.
* {{{
* SELECT * FROM table WHERE i <= 5 AND i = 5 => ... WHERE i = 5
* SELECT * FROM table WHERE i < j AND ... AND i > j => ... WHERE false
* }}}
*/
object FilterReduction extends Rule[LogicalPlan] with ConstraintHelper {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case f: Filter =>
val newCondition = normalizeAndReduceWithConstraints(f.condition)
if (newCondition fastEquals f.condition) {
f
} else {
f.copy(condition = newCondition)
}
}

private def normalizeAndReduceWithConstraints(expression: Expression): Expression =
reduceWithConstraints(normalize(expression), true)._1

private def normalize(expression: Expression) = expression transform {
case GreaterThan(x, y) => LessThan(y, x)
case GreaterThanOrEqual(x, y) => LessThanOrEqual(y, x)
}

/**
* Traverse a condition as a tree and simplify expressions with constraints.
* - This functions assumes that the plan has been normalized using [[normalize()]]
* - On matching [[And]], recursively traverse both children, simplify child expressions with
* propagated constraints from sibling and propagate up union of constraints.
* - If a child of [[And]] is [[LessThan]], [[LessThanOrEqual]], [[EqualTo]] or [[EqualNullSafe]],
* propagate the constraint.
* - On matching [[Or]], [[If]], [[CaseWhen]] or [[Not]] recursively traverse each children, but
* propagate up no constraints.
* - Starting off from a condition expression of a [[Filter]] as top node, to the bottom of the
* expression tree, [[And]], [[Or]], [[If]] and [[CaseWhen]] nodes are considered as safe,
* non-[[NullIntolerant]] nodes where reduction rules can be executed without nullability check.
* - Otherwise, stop traversal and propagate no constraints.
* @param expression expression to be traversed
* @param nullIsFalse defines if a null value can be considered as false
* @return A tuple including:
* 1. Expression: optionally changed expression after traversal
* 2. Seq[Expression]: propagated constraints
*/
private def reduceWithConstraints(
expression: Expression,
nullIsFalse: Boolean): (Expression, Seq[Expression]) =
expression match {
case e @ (_: LessThan | _: LessThanOrEqual | _: EqualTo | _: EqualNullSafe)
if e.deterministic => (e, Seq(e))
case a @ And(left, right) =>
val (newLeft, leftConstraints) = reduceWithConstraints(left, nullIsFalse)
val reducedRight = reduceWithConstraints(right, leftConstraints, nullIsFalse)
val (reducedNewRight, rightConstraints) =
reduceWithConstraints(reducedRight, nullIsFalse)
val reducedNewLeft = reduceWithConstraints(newLeft, rightConstraints, nullIsFalse)
val newAnd = if ((reducedNewLeft fastEquals left) &&
(reducedNewRight fastEquals right)) {
a
} else {
And(reducedNewLeft, reducedNewRight)
}
(newAnd, leftConstraints ++ rightConstraints)
case o @ (_: Or | _: If | _: CaseWhen) =>
(o.mapChildren(reduceWithConstraints(_, nullIsFalse)._1), Seq.empty)
case n: Not =>
(n.mapChildren(reduceWithConstraints(_, false)._1), Seq.empty)
case _ => (expression, Seq.empty)
}

private def reduceWithConstraints(
expression: Expression,
constraints: Seq[Expression],
nullIsFalse: Boolean) =
constraints
.foldLeft(expression)((e, constraint) => reduceWithConstraint(e, constraint, nullIsFalse))

private def planEqual(x: Expression, y: Expression) =
!x.foldable && !y.foldable && x.canonicalized == y.canonicalized

private def valueEqual(x: Expression, y: Expression) =
x.foldable && y.foldable && EqualTo(x, y).eval(EmptyRow).asInstanceOf[Boolean]

private def valueLessThan(x: Expression, y: Expression) =
x.foldable && y.foldable && LessThan(x, y).eval(EmptyRow).asInstanceOf[Boolean]

private def valueLessThanOrEqual(x: Expression, y: Expression) =
x.foldable && y.foldable && LessThanOrEqual(x, y).eval(EmptyRow).asInstanceOf[Boolean]

private def reduceWithConstraint(
expression: Expression,
constraint: Expression,
nullIsFalse: Boolean): Expression =
if (nullIsFalse || constraint.children.forall(!_.nullable)) {
constraint match {
case a LessThan b => expression transformUp {
case c LessThan d if planEqual(b, d) && (planEqual(a, c) || valueLessThanOrEqual(c, a)) =>
Literal.TrueLiteral
case c LessThan d if planEqual(b, c) && (planEqual(a, d) || valueLessThanOrEqual(d, a)) =>
Literal.FalseLiteral
case c LessThan d if planEqual(a, c) && (planEqual(b, d) || valueLessThanOrEqual(b, d)) =>
Literal.TrueLiteral
case c LessThan d if planEqual(a, d) && (planEqual(b, c) || valueLessThanOrEqual(b, c)) =>
Literal.FalseLiteral

case c LessThanOrEqual d
if planEqual(b, d) && (planEqual(a, c) || valueLessThanOrEqual(c, a)) =>
Literal.TrueLiteral
case c LessThanOrEqual d
if planEqual(b, c) && (planEqual(a, d) || valueLessThanOrEqual(d, a)) =>
Literal.FalseLiteral
case c LessThanOrEqual d
if planEqual(a, c) && (planEqual(b, d) || valueLessThanOrEqual(b, d)) =>
Literal.TrueLiteral
case c LessThanOrEqual d
if planEqual(a, d) && (planEqual(b, c) || valueLessThanOrEqual(b, c)) =>
Literal.FalseLiteral

case c EqualTo d if planEqual(b, d) && (planEqual(a, c) || valueLessThanOrEqual(c, a)) =>
Literal.FalseLiteral
case c EqualTo d if planEqual(b, c) && (planEqual(a, d) || valueLessThanOrEqual(d, a)) =>
Literal.FalseLiteral
case c EqualTo d if planEqual(a, c) && (planEqual(b, d) || valueLessThanOrEqual(b, d)) =>
Literal.FalseLiteral
case c EqualTo d if planEqual(a, d) && (planEqual(b, c) || valueLessThanOrEqual(b, c)) =>
Literal.FalseLiteral

case c EqualNullSafe d
if planEqual(b, d) && (planEqual(a, c) || valueLessThanOrEqual(c, a)) =>
Literal.FalseLiteral
case c EqualNullSafe d
if planEqual(b, c) && (planEqual(a, d) || valueLessThanOrEqual(d, a)) =>
Literal.FalseLiteral
case c EqualNullSafe d
if planEqual(a, c) && (planEqual(b, d) || valueLessThanOrEqual(b, d)) =>
Literal.FalseLiteral
case c EqualNullSafe d
if planEqual(a, d) && (planEqual(b, c) || valueLessThanOrEqual(b, c)) =>
Literal.FalseLiteral

case c EqualNullSafe d if planEqual(b, d) => EqualTo(c, d)
case c EqualNullSafe d if planEqual(b, c) => EqualTo(c, d)
case c EqualNullSafe d if planEqual(a, c) => EqualTo(c, d)
case c EqualNullSafe d if planEqual(a, d) => EqualTo(c, d)
}
case a LessThanOrEqual b => expression transformUp {
case c LessThan d if planEqual(b, d) && valueLessThan(c, a) =>
Literal.TrueLiteral
case c LessThan d if planEqual(b, c) && (planEqual(a, d) || valueLessThanOrEqual(d, a)) =>
Literal.FalseLiteral
case c LessThan d if planEqual(a, c) && valueLessThan(b, d) =>
Literal.TrueLiteral
case c LessThan d if planEqual(a, d) && (planEqual(b, c) || valueLessThanOrEqual(b, c)) =>
Literal.FalseLiteral

case c LessThanOrEqual d
if planEqual(b, d) && (planEqual(a, c) || valueLessThanOrEqual(c, a)) =>
Literal.TrueLiteral
case c LessThanOrEqual d if planEqual(b, c) && valueLessThan(d, a) =>
Literal.FalseLiteral
case c LessThanOrEqual d if planEqual(b, c) && (planEqual(a, d) || valueEqual(a, d)) =>
EqualTo(c, d)
case c LessThanOrEqual d
if planEqual(a, c) && (planEqual(b, d) || valueLessThanOrEqual(b, d)) =>
Literal.TrueLiteral
case c LessThanOrEqual d if planEqual(a, d) && valueLessThan(b, c) =>
Literal.FalseLiteral
case c LessThanOrEqual d if planEqual(a, d) && (planEqual(b, c) || valueEqual(b, c)) =>
EqualTo(c, d)

case c EqualTo d if planEqual(b, d) && valueLessThan(c, a) => Literal.FalseLiteral
case c EqualTo d if planEqual(b, c) && valueLessThan(d, a) => Literal.FalseLiteral
case c EqualTo d if planEqual(a, c) && valueLessThan(b, d) => Literal.FalseLiteral
case c EqualTo d if planEqual(a, d) && valueLessThan(b, c) => Literal.FalseLiteral

case c EqualNullSafe d if planEqual(b, d) && valueLessThan(c, a) => Literal.FalseLiteral
case c EqualNullSafe d if planEqual(b, c) && valueLessThan(d, a) => Literal.FalseLiteral
case c EqualNullSafe d if planEqual(a, c) && valueLessThan(b, d) => Literal.FalseLiteral
case c EqualNullSafe d if planEqual(a, d) && valueLessThan(b, c) => Literal.FalseLiteral

case c EqualNullSafe d if planEqual(b, d) => EqualTo(c, d)
case c EqualNullSafe d if planEqual(b, c) => EqualTo(c, d)
case c EqualNullSafe d if planEqual(a, c) => EqualTo(c, d)
case c EqualNullSafe d if planEqual(a, d) => EqualTo(c, d)
}
case a EqualTo b => expression transformUp {
case c LessThan d if planEqual(b, d) && planEqual(a, c) => Literal.FalseLiteral
case c LessThan d if planEqual(b, c) && planEqual(a, d) => Literal.FalseLiteral
case c LessThan d if planEqual(a, d) && planEqual(b, c) => Literal.FalseLiteral
case c LessThan d if planEqual(a, c) && planEqual(b, d) => Literal.FalseLiteral

case c LessThanOrEqual d if planEqual(b, d) && planEqual(a, c) => Literal.TrueLiteral
case c LessThanOrEqual d if planEqual(b, c) && planEqual(a, d) => Literal.TrueLiteral
case c LessThanOrEqual d if planEqual(a, d) && planEqual(b, c) => Literal.TrueLiteral
case c LessThanOrEqual d if planEqual(a, c) && planEqual(b, d) => Literal.TrueLiteral

case c EqualTo d if planEqual(b, d) && planEqual(a, c) => Literal.TrueLiteral
case c EqualTo d if planEqual(b, c) && planEqual(a, d) => Literal.TrueLiteral
case c EqualTo d if planEqual(a, d) && planEqual(b, c) => Literal.TrueLiteral
case c EqualTo d if planEqual(a, c) && planEqual(b, d) => Literal.TrueLiteral

case c EqualNullSafe d if planEqual(b, d) =>
if (planEqual(a, c)) Literal.TrueLiteral else EqualTo(c, d)
case c EqualNullSafe d if planEqual(b, c) =>
if (planEqual(a, d)) Literal.TrueLiteral else EqualTo(c, d)
case c EqualNullSafe d if planEqual(a, d) =>
if (planEqual(b, c)) Literal.TrueLiteral else EqualTo(c, d)
case c EqualNullSafe d if planEqual(a, c) =>
if (planEqual(b, d)) Literal.TrueLiteral else EqualTo(c, d)
}
case a EqualNullSafe b => expression transformUp {
case c LessThan d if planEqual(b, d) && planEqual(a, c) => Literal.FalseLiteral
case c LessThan d if planEqual(b, c) && planEqual(d, a) => Literal.FalseLiteral
case c LessThan d if planEqual(a, d) && planEqual(b, c) => Literal.FalseLiteral
case c LessThan d if planEqual(a, c) && planEqual(d, b) => Literal.FalseLiteral

case c LessThanOrEqual d if planEqual(b, d) && planEqual(a, c) => EqualTo(c, d)
case c LessThanOrEqual d if planEqual(b, c) && planEqual(a, d) => EqualTo(c, d)
case c LessThanOrEqual d if planEqual(a, d) && planEqual(b, c) => EqualTo(c, d)
case c LessThanOrEqual d if planEqual(a, c) && planEqual(b, d) => EqualTo(c, d)

case c EqualNullSafe d if planEqual(b, d) && planEqual(a, c) => Literal.TrueLiteral
case c EqualNullSafe d if planEqual(b, c) && planEqual(a, d) => Literal.TrueLiteral
case c EqualNullSafe d if planEqual(a, d) && planEqual(b, c) => Literal.TrueLiteral
case c EqualNullSafe d if planEqual(a, c) && planEqual(b, d) => Literal.TrueLiteral
}
case _ => expression
}
} else {
constraint match {
case a EqualNullSafe b => expression transformUp {
case c EqualNullSafe d if planEqual(b, d) && planEqual(a, c) => Literal.TrueLiteral
case c EqualNullSafe d if planEqual(b, c) && planEqual(a, d) => Literal.TrueLiteral
case c EqualNullSafe d if planEqual(a, d) && planEqual(b, c) => Literal.TrueLiteral
case c EqualNullSafe d if planEqual(a, c) && planEqual(b, d) => Literal.TrueLiteral
}
case _ => expression
}
}
}

/**
* Reorder associative integral-type operators and fold all constants into one.
*/
Expand Down
Loading