Skip to content

Commit cd71aac

Browse files
committed
doc update
1 parent bd9ce83 commit cd71aac

File tree

1 file changed

+17
-11
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/internal

1 file changed

+17
-11
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -355,29 +355,33 @@ object SQLConf {
355355

356356
val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
357357
buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
358-
.doc("The target post-shuffle input size in bytes of a task.")
358+
.doc("The target post-shuffle input size in bytes of a task. This configuration only has " +
359+
s"an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
359360
.bytesConf(ByteUnit.BYTE)
360361
.createWithDefault(64 * 1024 * 1024)
361362

362363
val FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED =
363364
buildConf("spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch.enabled")
364365
.doc("Whether to fetch the continuous shuffle blocks in batch. Instead of fetching blocks " +
365366
"one by one, fetching continuous shuffle blocks for the same map task in batch can " +
366-
"reduce IO and improve performance. Note, this feature also depends on a relocatable " +
367-
"serializer and the concatenation support codec in use.")
367+
"reduce IO and improve performance. Note, multiple continuous blocks exist in single " +
368+
s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, " +
369+
"this feature also depends on a relocatable serializer and the concatenation support " +
370+
"codec in use.")
368371
.booleanConf
369372
.createWithDefault(true)
370373

371374
val REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED =
372375
buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled")
373-
.doc("When true and adaptive execution is enabled, this enables reducing the number of " +
374-
"post-shuffle partitions based on map output statistics.")
376+
.doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables reducing " +
377+
"the number of post-shuffle partitions based on map output statistics.")
375378
.booleanConf
376379
.createWithDefault(true)
377380

378381
val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS =
379382
buildConf("spark.sql.adaptive.shuffle.minNumPostShufflePartitions")
380-
.doc("The advisory minimum number of post-shuffle partitions used in adaptive execution.")
383+
.doc("The advisory minimum number of post-shuffle partitions used when " +
384+
s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
381385
.intConf
382386
.checkValue(_ > 0, "The minimum shuffle partition number " +
383387
"must be a positive integer.")
@@ -387,25 +391,27 @@ object SQLConf {
387391
buildConf("spark.sql.adaptive.shuffle.maxNumPostShufflePartitions")
388392
.doc("The advisory maximum number of post-shuffle partitions used in adaptive execution. " +
389393
"This is used as the initial number of pre-shuffle partitions. By default it equals to " +
390-
"spark.sql.shuffle.partitions")
394+
"spark.sql.shuffle.partitions. This configuration only has an effect when " +
395+
s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
391396
.intConf
392397
.checkValue(_ > 0, "The maximum shuffle partition number " +
393398
"must be a positive integer.")
394399
.createOptional
395400

396401
val LOCAL_SHUFFLE_READER_ENABLED =
397402
buildConf("spark.sql.adaptive.shuffle.localShuffleReader.enabled")
398-
.doc("When true and adaptive execution is enabled, this enables the optimization of" +
399-
" converting the shuffle reader to local shuffle reader for the shuffle exchange" +
400-
" of the broadcast hash join in probe side.")
403+
.doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables the " +
404+
"optimization of converting the shuffle reader to local shuffle reader for the shuffle " +
405+
"exchange of the broadcast hash join in probe side.")
401406
.booleanConf
402407
.createWithDefault(true)
403408

404409
val NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN =
405410
buildConf("spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin")
406411
.doc("The relation with a non-empty partition ratio lower than this config will not be " +
407412
"considered as the build side of a broadcast-hash join in adaptive execution regardless " +
408-
"of its size.")
413+
"of its size.This configuration only has an effect when " +
414+
s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
409415
.doubleConf
410416
.checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
411417
.createWithDefault(0.2)

0 commit comments

Comments
 (0)