Skip to content

Commit b558549

Browse files
committed
address comments
1 parent 214842b commit b558549

File tree

2 files changed

+12
-7
lines changed

2 files changed

+12
-7
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,20 +39,23 @@ object DefaultOptimizer extends Optimizer {
3939
Batch("Distinct", FixedPoint(100),
4040
ReplaceDistinctWithAggregate) ::
4141
Batch("Operator Optimizations", FixedPoint(100),
42+
// Operator push down
4243
UnionPushDown,
43-
CombineFilters,
44+
LimitPushDown,
45+
PushPredicateThroughJoin,
4446
PushPredicateThroughProject,
4547
PushPredicateThroughGenerate,
4648
ColumnPruning,
47-
LimitPushDown,
49+
// Operator combine
4850
ProjectCollapsing,
51+
CombineFilters,
4952
CombineLimits,
53+
// Constant folding
5054
NullPropagation,
5155
OptimizeIn,
5256
ConstantFolding,
5357
LikeSimplification,
5458
BooleanSimplification,
55-
PushPredicateThroughJoin,
5659
RemovePositive,
5760
SimplifyFilters,
5861
SimplifyCasts,
@@ -111,12 +114,14 @@ object UnionPushDown extends Rule[LogicalPlan] {
111114

112115
object LimitPushDown extends Rule[LogicalPlan] {
113116
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
114-
// Push down limit when the child is project on limit
117+
// Push down limit when the child is project on limit.
115118
case Limit(expr, Project(projectList, l: Limit)) =>
116119
Project(projectList, Limit(expr, l))
117120

118-
// Push down limit when the child is project on sort
119-
case Limit(expr, Project(projectList, s: Sort)) =>
121+
// Push down limit when the child is project on sort,
122+
// and we cannot push down this project through sort.
123+
case Limit(expr, p @ Project(projectList, s: Sort))
124+
if !s.references.subsetOf(p.outputSet) =>
120125
Project(projectList, Limit(expr, s))
121126
}
122127
}
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.rules.RuleExecutor
2424
import org.apache.spark.sql.catalyst.dsl.expressions._
2525
import org.apache.spark.sql.catalyst.dsl.plans._
2626

27-
class LimitPushDownSuit extends PlanTest {
27+
class LimitPushDownSuite extends PlanTest {
2828

2929
object Optimize extends RuleExecutor[LogicalPlan] {
3030
val batches =

0 commit comments

Comments
 (0)