From cb2ec5c424433ec9b5ae49436d3d823964c7d509 Mon Sep 17 00:00:00 2001 From: Nick Young Date: Fri, 7 Jun 2024 15:32:19 -0700 Subject: [PATCH 1/2] Make ineffectiveRules lazy --- .../apache/spark/sql/catalyst/trees/TreeNode.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala index dd39f3182bfb8..23052a353fafb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala @@ -120,7 +120,14 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] * ineffective for subsequent apply calls on this tree because query plan structures are * immutable. */ - private val ineffectiveRules: BitSet = new BitSet(RuleIdCollection.NumRules) + private[this] var _ineffectiveRules: BitSet = null + private def ineffectiveRules: BitSet = { + if (_ineffectiveRules eq null) { + _ineffectiveRules = new BitSet(RuleIdCollection.NumRules) + } + _ineffectiveRules + } + private def isIneffectiveRulesEmpty = _ineffectiveRules == null /** * @return a sequence of tree pattern enums in a TreeNode T. It does not include propagated @@ -149,7 +156,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] * UnknownId, it returns false. */ protected def isRuleIneffective(ruleId : RuleId): Boolean = { - if (ruleId eq UnknownRuleId) { + if (isIneffectiveRulesEmpty || (ruleId eq UnknownRuleId)) { return false } ineffectiveRules.get(ruleId.id) From 5d01eb58de16de1e585bbb11401d1a219a881404 Mon Sep 17 00:00:00 2001 From: Nick Young Date: Mon, 10 Jun 2024 09:19:37 -0700 Subject: [PATCH 2/2] Update sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala Co-authored-by: Josh Rosen --- .../scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala index 23052a353fafb..23d26854a767a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala @@ -127,7 +127,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] } _ineffectiveRules } - private def isIneffectiveRulesEmpty = _ineffectiveRules == null + private def isIneffectiveRulesEmpty = _ineffectiveRules eq null /** * @return a sequence of tree pattern enums in a TreeNode T. It does not include propagated