Skip to content

Commit b4e19a2

Browse files
costincostin
authored
[7.x] SQL: Pushdown WHERE clause inside subqueries (#71362) (#71398)
Push down filters inside subqueries, even when dealing with aggregates. The rule already existed however it was not being used inside SQL. When dealing with Aggregates, keep the aggregate functions in place but try and push down conjunctions on non-aggregates.
1 parent 9d04e61 commit b4e19a2

File tree

8 files changed

+241
-40
lines changed

8 files changed

+241
-40
lines changed

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.xpack.ql.expression.Order.NullsPosition;
2828
import org.elasticsearch.xpack.ql.expression.Order.OrderDirection;
2929
import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
30-
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
3130
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
3231
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
3332
import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
@@ -47,6 +46,7 @@
4746
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.OptimizerRule;
4847
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals;
4948
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy;
49+
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters;
5050
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceSurrogateFunction;
5151
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized;
5252
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SimplifyComparisonsArithmetics;
@@ -93,13 +93,14 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
9393
new PropagateNullable(),
9494
new CombineBinaryComparisons(),
9595
new CombineDisjunctionsToIn(),
96-
new PushDownAndCombineFilters(),
9796
new SimplifyComparisonsArithmetics(DataTypes::areCompatible),
9897
// prune/elimination
9998
new PruneFilters(),
10099
new PruneLiteralsInOrderBy(),
101100
new PruneCast(),
102-
new CombineLimits());
101+
new CombineLimits(),
102+
new PushDownAndCombineFilters()
103+
);
103104

104105
Batch constraints = new Batch("Infer constraints", Limiter.ONCE,
105106
new PropagateJoinKeyConstraints());
@@ -189,25 +190,6 @@ protected LogicalPlan rule(Filter filter) {
189190
}
190191
}
191192

192-
static class PushDownAndCombineFilters extends OptimizerRule<Filter> {
193-
194-
@Override
195-
protected LogicalPlan rule(Filter filter) {
196-
LogicalPlan child = filter.child();
197-
LogicalPlan plan = filter;
198-
199-
if (child instanceof Filter) {
200-
Filter f = (Filter) child;
201-
plan = new Filter(f.source(), f.child(), new And(f.source(), f.condition(), filter.condition()));
202-
} else if (child instanceof UnaryPlan) {
203-
UnaryPlan up = (UnaryPlan) child;
204-
plan = child.replaceChildrenSameSize(singletonList(new Filter(filter.source(), up.child(), filter.condition())));
205-
}
206-
207-
return plan;
208-
}
209-
}
210-
211193
static class ReplaceRegexMatch extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceRegexMatch {
212194
@Override
213195
protected Expression regexToEquals(RegexMatch<?> regexMatch, Literal literal) {

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG;
6868
import static org.elasticsearch.xpack.ql.TestUtils.UTC;
6969
import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
70+
import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters;
7071
import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
7172
import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER;
7273

@@ -334,7 +335,7 @@ public void testCombineFilters() {
334335
Filter filterChild = basicFilter(left);
335336
Filter filterParent = new Filter(EMPTY, filterChild, right);
336337

337-
LogicalPlan result = new Optimizer.PushDownAndCombineFilters().apply(filterParent);
338+
LogicalPlan result = new PushDownAndCombineFilters().apply(filterParent);
338339

339340
assertEquals(Filter.class, result.getClass());
340341
Expression condition = ((Filter) result).condition();
@@ -359,7 +360,7 @@ public void testPushDownFilterUnary() {
359360
OrderBy order = new OrderBy(EMPTY, rel(), emptyList());
360361
Filter filter = new Filter(EMPTY, order, left);
361362

362-
LogicalPlan result = new Optimizer.PushDownAndCombineFilters().apply(filter);
363+
LogicalPlan result = new PushDownAndCombineFilters().apply(filter);
363364

364365
assertEquals(OrderBy.class, result.getClass());
365366
OrderBy o = (OrderBy) result;
@@ -386,7 +387,7 @@ public void testPushDownFilterDoesNotApplyOnNonUnary() {
386387
Sequence s = sequence(rule1, rule2);
387388
Filter filter = new Filter(EMPTY, s, left);
388389

389-
LogicalPlan result = new Optimizer.PushDownAndCombineFilters().apply(filter);
390+
LogicalPlan result = new PushDownAndCombineFilters().apply(filter);
390391

391392
assertEquals(Filter.class, result.getClass());
392393
Filter f = (Filter) result;

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.xpack.ql.expression.Nullability;
1414
import org.elasticsearch.xpack.ql.expression.Order;
1515
import org.elasticsearch.xpack.ql.expression.function.Function;
16+
import org.elasticsearch.xpack.ql.expression.function.Functions;
1617
import org.elasticsearch.xpack.ql.expression.function.scalar.SurrogateFunction;
1718
import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator;
1819
import org.elasticsearch.xpack.ql.expression.predicate.BinaryPredicate;
@@ -40,10 +41,12 @@
4041
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
4142
import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch;
4243
import org.elasticsearch.xpack.ql.expression.predicate.regex.StringPattern;
44+
import org.elasticsearch.xpack.ql.plan.logical.Aggregate;
4345
import org.elasticsearch.xpack.ql.plan.logical.Filter;
4446
import org.elasticsearch.xpack.ql.plan.logical.Limit;
4547
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
4648
import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
49+
import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan;
4750
import org.elasticsearch.xpack.ql.rule.Rule;
4851
import org.elasticsearch.xpack.ql.type.DataType;
4952
import org.elasticsearch.xpack.ql.type.DataTypes;
@@ -63,6 +66,7 @@
6366

6467
import static java.lang.Math.signum;
6568
import static java.util.Arrays.asList;
69+
import static java.util.Collections.singletonList;
6670
import static org.elasticsearch.xpack.ql.expression.Literal.FALSE;
6771
import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
6872
import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.combineAnd;
@@ -1139,6 +1143,51 @@ protected In createIn(Expression key, List<Expression> values, ZoneId zoneId) {
11391143
}
11401144
}
11411145

1146+
public static class PushDownAndCombineFilters extends OptimizerRule<Filter> {
1147+
1148+
@Override
1149+
protected LogicalPlan rule(Filter filter) {
1150+
LogicalPlan plan = filter;
1151+
LogicalPlan child = filter.child();
1152+
Expression condition = filter.condition();
1153+
1154+
if (child instanceof Filter) {
1155+
Filter f = (Filter) child;
1156+
plan = f.with(new And(f.source(), f.condition(), condition));
1157+
}
1158+
// as it stands, all other unary plans should allow filters to be pushed down
1159+
else if (child instanceof UnaryPlan) {
1160+
UnaryPlan unary = (UnaryPlan) child;
1161+
// in case of aggregates, worry about filters that contain aggregations
1162+
if (unary instanceof Aggregate && condition.anyMatch(Functions::isAggregate)) {
1163+
Aggregate agg = (Aggregate) unary;
1164+
List<Expression> conjunctions = new ArrayList<>(splitAnd(condition));
1165+
List<Expression> inPlace = new ArrayList<>();
1166+
// extract all conjunctions containing aggregates
1167+
for (Iterator<Expression> iterator = conjunctions.iterator(); iterator.hasNext();) {
1168+
Expression conjunction = iterator.next();
1169+
if (conjunction.anyMatch(Functions::isAggregate)) {
1170+
inPlace.add(conjunction);
1171+
iterator.remove();
1172+
}
1173+
}
1174+
// if at least one expression can be pushed down, update the tree
1175+
if (conjunctions.size() > 0) {
1176+
child = child.replaceChildrenSameSize(
1177+
singletonList(filter.with(unary.child(), Predicates.combineAnd(conjunctions)))
1178+
);
1179+
plan = filter.with(child, Predicates.combineAnd(inPlace));
1180+
}
1181+
} else {
1182+
// push down filter
1183+
plan = child.replaceChildrenSameSize(singletonList(filter.with(unary.child(), condition)));
1184+
}
1185+
}
1186+
1187+
return plan;
1188+
}
1189+
}
1190+
11421191
public static class ReplaceSurrogateFunction extends OptimizerExpressionRule {
11431192

11441193
public ReplaceSurrogateFunction() {

‎x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
1414
import org.elasticsearch.xpack.ql.expression.Literal;
1515
import org.elasticsearch.xpack.ql.expression.Nullability;
16+
import org.elasticsearch.xpack.ql.expression.function.aggregate.Count;
1617
import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator;
1718
import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
1819
import org.elasticsearch.xpack.ql.expression.predicate.Range;
@@ -45,6 +46,10 @@
4546
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
4647
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ConstantFolding;
4748
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals;
49+
import org.elasticsearch.xpack.ql.plan.logical.Aggregate;
50+
import org.elasticsearch.xpack.ql.plan.logical.EsRelation;
51+
import org.elasticsearch.xpack.ql.plan.logical.Filter;
52+
import org.elasticsearch.xpack.ql.plan.logical.Project;
4853
import org.elasticsearch.xpack.ql.tree.NodeInfo;
4954
import org.elasticsearch.xpack.ql.tree.Source;
5055
import org.elasticsearch.xpack.ql.type.DataType;
@@ -56,6 +61,7 @@
5661
import java.util.List;
5762

5863
import static java.util.Arrays.asList;
64+
import static java.util.Collections.emptyList;
5965
import static java.util.Collections.emptyMap;
6066
import static java.util.Collections.singletonList;
6167
import static org.elasticsearch.xpack.ql.TestUtils.equalsOf;
@@ -68,11 +74,13 @@
6874
import static org.elasticsearch.xpack.ql.TestUtils.nullEqualsOf;
6975
import static org.elasticsearch.xpack.ql.TestUtils.of;
7076
import static org.elasticsearch.xpack.ql.TestUtils.rangeOf;
77+
import static org.elasticsearch.xpack.ql.TestUtils.relation;
7178
import static org.elasticsearch.xpack.ql.expression.Literal.FALSE;
7279
import static org.elasticsearch.xpack.ql.expression.Literal.NULL;
7380
import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
7481
import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineDisjunctionsToIn;
7582
import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateNullable;
83+
import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters;
7684
import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceRegexMatch;
7785
import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
7886
import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
@@ -1609,7 +1617,48 @@ public void testIsNullDisjunction() throws Exception {
16091617
assertEquals(and, new PropagateNullable().rule(and));
16101618
}
16111619

1612-
public void testSkipNull() throws Exception {
1620+
public void testCombineFilters() throws Exception {
1621+
EsRelation relation = relation();
1622+
GreaterThan conditionA = greaterThanOf(getFieldAttribute("a"), ONE);
1623+
LessThan conditionB = lessThanOf(getFieldAttribute("b"), TWO);
1624+
1625+
Filter fa = new Filter(EMPTY, relation, conditionA);
1626+
Filter fb = new Filter(EMPTY, fa, conditionB);
1627+
1628+
assertEquals(new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)), new PushDownAndCombineFilters().apply(fb));
1629+
}
1630+
1631+
public void testPushDownFilter() throws Exception {
1632+
EsRelation relation = relation();
1633+
GreaterThan conditionA = greaterThanOf(getFieldAttribute("a"), ONE);
1634+
LessThan conditionB = lessThanOf(getFieldAttribute("b"), TWO);
1635+
1636+
Filter fa = new Filter(EMPTY, relation, conditionA);
1637+
List<FieldAttribute> projections = singletonList(getFieldAttribute("b"));
1638+
Project project = new Project(EMPTY, fa, projections);
1639+
Filter fb = new Filter(EMPTY, project, conditionB);
1640+
1641+
Filter combinedFilter = new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB));
1642+
assertEquals(new Project(EMPTY, combinedFilter, projections), new PushDownAndCombineFilters().apply(fb));
1643+
}
1644+
1645+
public void testPushDownFilterThroughAgg() throws Exception {
1646+
EsRelation relation = relation();
1647+
GreaterThan conditionA = greaterThanOf(getFieldAttribute("a"), ONE);
1648+
LessThan conditionB = lessThanOf(getFieldAttribute("b"), TWO);
1649+
GreaterThanOrEqual aggregateCondition = greaterThanOrEqualOf(new Count(EMPTY, ONE, false), THREE);
1650+
1651+
Filter fa = new Filter(EMPTY, relation, conditionA);
1652+
List<FieldAttribute> projections = singletonList(getFieldAttribute("b"));
1653+
// invalid aggregate but that's fine cause its properties are not used by this rule
1654+
Aggregate aggregate = new Aggregate(EMPTY, fa, emptyList(), emptyList());
1655+
Filter fb = new Filter(EMPTY, aggregate, new And(EMPTY, aggregateCondition, conditionB));
1656+
1657+
Filter combinedFilter = new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB));
1658+
1659+
// expected
1660+
Filter expected = new Filter(EMPTY, new Aggregate(EMPTY, combinedFilter, emptyList(), emptyList()), aggregateCondition);
1661+
assertEquals(expected, new PushDownAndCombineFilters().apply(fb));
16131662

16141663
}
16151664
}

0 commit comments

Comments
 (0)