Skip to content

Commit c711c86

Browse files
committed
Renamed isPushBasedShuffleEnabled to isPushShuffleEnabled and fixed the comment
1 parent 3c84e07 commit c711c86

File tree

3 files changed

+3
-5
lines changed

3 files changed

+3
-5
lines changed

core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ private[spark] class ShuffleWriteProcessor extends Serializable with Logging {
6464
// The map task only takes care of converting the shuffle data file into multiple
6565
// block push requests and pushing the first batch of blocks to the remote shuffle
6666
// service. The remaining block pushes is handled by netty client threads.
67-
if (Utils.isPushBasedShuffleEnabled(SparkEnv.get.conf) && dep.getMergerLocs.nonEmpty) {
67+
if (Utils.isPushShuffleEnabled(SparkEnv.get.conf) && dep.getMergerLocs.nonEmpty) {
6868
writer.initiateBlockPush(
6969
manager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver],
7070
writer.getPartitionLengths(), dep, partition.index, mapId, SparkEnv.get.conf)

core/src/main/scala/org/apache/spark/shuffle/ShuffleWriter.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -461,7 +461,7 @@ private[spark] object ShuffleWriter {
461461

462462
private val BLOCK_PUSHER_POOL: ExecutorService = {
463463
val conf = SparkEnv.get.conf
464-
if (Utils.isPushBasedShuffleEnabled(conf)) {
464+
if (Utils.isPushShuffleEnabled(conf)) {
465465
val numThreads = conf.get(PUSH_SHUFFLE_NUM_PUSH_THREADS)
466466
.getOrElse(conf.getInt(SparkLauncher.EXECUTOR_CORES, 1))
467467
ThreadUtils.newDaemonFixedThreadPool(numThreads, "block-push-thread")

core/src/main/scala/org/apache/spark/util/Utils.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2543,10 +2543,8 @@ private[spark] object Utils extends Logging {
25432543

25442544
/**
25452545
* Push based shuffle can only be enabled when external shuffle service is enabled.
2546-
* In the initial version, we cannot support pushed based shuffle and adaptive execution
2547-
* at the same time. Will improve this in a later version.
25482546
*/
2549-
def isPushBasedShuffleEnabled(conf: SparkConf): Boolean = {
2547+
def isPushShuffleEnabled(conf: SparkConf): Boolean = {
25502548
conf.get(PUSH_SHUFFLE_ENABLED) && conf.get(SHUFFLE_SERVICE_ENABLED)
25512549
}
25522550

0 commit comments

Comments
 (0)