diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index af8a24553a461..3019846143d9f 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1837,26 +1837,31 @@ private[spark] object Utils extends Logging { * Attempt to start a service on the given port, or fail after a number of attempts. * Each subsequent attempt uses 1 + the port used in the previous attempt (unless the port is 0). * - * @param startPort The initial port to start the service on. + * @param port The initial port to start the service on. + * @param maxRetries Maximum number of retries to attempt. + * A value of 3 means attempting ports n, n+1, n+2, and n+3, for example. * @param startService Function to start service on a given port. * This is expected to throw java.net.BindException on port collision. * @param conf A SparkConf used to get the maximum number of retries when binding to a port. * @param serviceName Name of the service. */ def startServiceOnPort[T]( - startPort: Int, + port: Int, startService: Int => (T, Int), conf: SparkConf, serviceName: String = ""): (T, Int) = { val serviceString = if (serviceName.isEmpty) "" else s" '$serviceName'" val maxRetries = portMaxRetries(conf) + val startPort = conf.getInt("spark.port.min", 1024) + val endPort = conf.getInt("spark.port.max", 65536) for (offset <- 0 to maxRetries) { - // Do not increment port if startPort is 0, which is treated as a special port - val tryPort = if (startPort == 0) { - startPort + // specific a random port between 'spark.port.min' and 'spark.port.max' + // if port is 0 + val tryPort = if (port == 0) { + (startPort + Math.random() * (endPort - startPort + 1)).toInt } else { // If the new port wraps around, do not try a privilege port - ((startPort + offset - 1024) % (65536 - 1024)) + 1024 + ((port + offset - 1024) % (65536 - 1024)) + 1024 } try { val (service, port) = startService(tryPort) diff --git a/docs/configuration.md b/docs/configuration.md index 7fe11475212b3..9be51ba63fead 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -165,6 +165,20 @@ of the most common options to set are: Logs the effective SparkConf as INFO when a SparkContext is started. +
spark.port.minspark.port.maxspark.master