From 49b6d166969e191c98223ebafb772850f528403f Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sat, 25 Apr 2015 18:15:48 +0530 Subject: [PATCH 1/4] SPARK-7142: Minor enhancement to BooleanSimplification Optimizer rule, using these rules: A and (not(A) or B) => A and B not(A and B) => not(A) or not(B) not(A or B) => not(A) and not(B) --- .../spark/sql/catalyst/optimizer/Optimizer.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index a430000bef653..e3b6aec34d3c5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -434,8 +434,13 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper { case (_, Literal(false, BooleanType)) => Literal(false) // a && a => a case (l, r) if l fastEquals r => l + // a && (not(a) || b) => a && b + case (l, Or(l1, r)) if (Not(l) fastEquals l1) => And(l, r) + case (l, Or(r, l1)) if (Not(l) fastEquals l1) => And(l, r) + case (Or(l, l1), r) if (l1 fastEquals Not(r)) => And(l, r) + case (Or(l1, l), r) if (l1 fastEquals Not(r)) => And(l, r) // (a || b) && (a || c) => a || (b && c) - case _ => + case _ => // 1. Split left and right to get the disjunctive predicates, // i.e. lhs = (a, b), rhs = (a, c) // 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a) @@ -512,6 +517,10 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper { case LessThan(l, r) => GreaterThanOrEqual(l, r) // not(l <= r) => l > r case LessThanOrEqual(l, r) => GreaterThan(l, r) + // not(l || r) => not(l) && not(r) + case Or(l, r) => And(Not(l), Not(r)) + // not(l && r) => not(l) or not(r) + case And(l, r) => Or(Not(l), Not(r)) // not(not(e)) => e case Not(e) => e case _ => not From 7ada093214359973f12284e3fc368c8e36bb1b17 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sat, 25 Apr 2015 18:20:39 +0530 Subject: [PATCH 2/4] SPARK-7142: Fix whitespace --- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index e3b6aec34d3c5..d9b50f3c97da0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -440,7 +440,7 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper { case (Or(l, l1), r) if (l1 fastEquals Not(r)) => And(l, r) case (Or(l1, l), r) if (l1 fastEquals Not(r)) => And(l, r) // (a || b) && (a || c) => a || (b && c) - case _ => + case _ => // 1. Split left and right to get the disjunctive predicates, // i.e. lhs = (a, b), rhs = (a, c) // 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a) From de2dbdcf889add5602496a34bc796ef12242fdb0 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Thu, 10 Sep 2015 12:37:18 +0530 Subject: [PATCH 3/4] SPARK-7142: Adding test cases --- .../optimizer/BooleanSimplificationSuite.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index 1877cff1334bd..5cbfbb1080fea 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -89,6 +89,22 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { ('a === 'b || 'b > 3 && 'a > 3 && 'a < 5)) } + test("a && (!a || b)") { + checkCondition(('a && (!('a) || 'b )), ('a && 'b)) + + checkCondition(('a && ('b || !('a) )), ('a && 'b)) + + checkCondition(((!('a) || 'b ) && 'a), ('b && 'a)) + + checkCondition((('b || !('a) ) && 'a), ('b && 'a)) + } + + test("!(a && b) , !(a || b)") { + checkCondition((!('a && 'b)), (!('a) || !('b))) + + checkCondition(!('a || 'b), (!('a) && !('b))) + } + private val caseInsensitiveAnalyzer = new Analyzer(EmptyCatalog, EmptyFunctionRegistry, new SimpleCatalystConf(false)) From 2dbfba691283457f0367271d8348951123b8229d Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Thu, 10 Sep 2015 13:45:01 +0530 Subject: [PATCH 4/4] SPARK-7142: Fix whitespace --- .../sql/catalyst/optimizer/BooleanSimplificationSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index 5cbfbb1080fea..cde346e99eb17 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -104,7 +104,7 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { checkCondition(!('a || 'b), (!('a) && !('b))) } - + private val caseInsensitiveAnalyzer = new Analyzer(EmptyCatalog, EmptyFunctionRegistry, new SimpleCatalystConf(false))