Skip to content

Commit 84207c7

Browse files
committed
Skip automatically broadcast a plan when it contains ObjectProducer.
1 parent df1953f commit 84207c7

File tree

1 file changed

+5
-2
lines changed

1 file changed

+5
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.encoders.RowEncoder
2323
import org.apache.spark.sql.catalyst.expressions._
2424
import org.apache.spark.sql.catalyst.planning._
2525
import org.apache.spark.sql.catalyst.plans._
26-
import org.apache.spark.sql.catalyst.plans.logical.{BroadcastHint, LogicalPlan}
26+
import org.apache.spark.sql.catalyst.plans.logical.{BroadcastHint, LogicalPlan, ObjectProducer}
2727
import org.apache.spark.sql.catalyst.plans.physical._
2828
import org.apache.spark.sql.execution
2929
import org.apache.spark.sql.execution.columnar.{InMemoryColumnarTableScan, InMemoryRelation}
@@ -82,7 +82,10 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
8282
*/
8383
object CanBroadcast {
8484
def unapply(plan: LogicalPlan): Option[LogicalPlan] = {
85-
if (plan.statistics.sizeInBytes <= conf.autoBroadcastJoinThreshold) {
85+
// We can't estimate the size of ObjectType
86+
if (plan.find(_.isInstanceOf[ObjectProducer]).isDefined) {
87+
None
88+
} else if (plan.statistics.sizeInBytes <= conf.autoBroadcastJoinThreshold) {
8689
Some(plan)
8790
} else {
8891
None

0 commit comments

Comments
 (0)