From 5f5fda8244c59970d117a0e9d32bd8bddde348f2 Mon Sep 17 00:00:00 2001 From: w00228970 Date: Mon, 17 Nov 2014 10:34:17 +0800 Subject: [PATCH 1/5] https -> http in pom --- pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 639ea22a1fda3..26fa0e823edbf 100644 --- a/pom.xml +++ b/pom.xml @@ -156,7 +156,7 @@ central Maven Repository - https://repo1.maven.org/maven2 + http://repo1.maven.org/maven2 true @@ -167,7 +167,7 @@ apache-repo Apache Repository - https://repository.apache.org/content/repositories/releases + http://repository.apache.org/content/repositories/releases true @@ -178,7 +178,7 @@ jboss-repo JBoss Repository - https://repository.jboss.org/nexus/content/repositories/releases + http://repository.jboss.org/nexus/content/repositories/releases true @@ -189,7 +189,7 @@ mqtt-repo MQTT Repository - https://repo.eclipse.org/content/repositories/paho-releases + http://repo.eclipse.org/content/repositories/paho-releases true @@ -200,7 +200,7 @@ cloudera-repo Cloudera Repository - https://repository.cloudera.com/artifactory/cloudera-repos + http://repository.cloudera.com/artifactory/cloudera-repos true @@ -222,7 +222,7 @@ spring-releases Spring Release Repository - https://repo.spring.io/libs-release + http://repo.spring.io/libs-release true @@ -234,7 +234,7 @@ spark-staging Spring Staging Repository - https://oss.sonatype.org/content/repositories/orgspark-project-1085 + http://oss.sonatype.org/content/repositories/orgspark-project-1085 true @@ -246,7 +246,7 @@ spark-staging-hive13 Spring Staging Repository Hive 13 - https://oss.sonatype.org/content/repositories/orgspark-project-1089/ + http://oss.sonatype.org/content/repositories/orgspark-project-1089/ true @@ -258,7 +258,7 @@ central - https://repo1.maven.org/maven2 + http://repo1.maven.org/maven2 true From 4203b7f0061e18e5057cf500e43f765c5ae1820d Mon Sep 17 00:00:00 2001 From: w00228970 Date: Mon, 17 Nov 2014 18:23:17 +0800 Subject: [PATCH 2/5] start port in specified rang ports --- .../main/scala/org/apache/spark/HttpServer.scala | 3 ++- .../scala/org/apache/spark/SecurityManager.scala | 3 ++- .../spark/network/nio/ConnectionManager.scala | 2 +- .../scala/org/apache/spark/ui/JettyUtils.scala | 2 +- .../scala/org/apache/spark/util/AkkaUtils.scala | 2 +- .../main/scala/org/apache/spark/util/Utils.scala | 15 +++++++++------ 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/HttpServer.scala b/core/src/main/scala/org/apache/spark/HttpServer.scala index 912558d0cab7d..310e87ec56576 100644 --- a/core/src/main/scala/org/apache/spark/HttpServer.scala +++ b/core/src/main/scala/org/apache/spark/HttpServer.scala @@ -57,7 +57,8 @@ private[spark] class HttpServer( } else { logInfo("Starting HTTP Server") val (actualServer, actualPort) = - Utils.startServiceOnPort[Server](requestedPort, doStart, serverName) + Utils.startServiceOnPort[Server]( + requestedPort, doStart, securityManager.sparkConf, serverName) server = actualServer port = actualPort } diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala index dbff9d12b5ad7..abdbba0c7b907 100644 --- a/core/src/main/scala/org/apache/spark/SecurityManager.scala +++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala @@ -144,7 +144,8 @@ import org.apache.spark.network.sasl.SecretKeyHolder * can take place. */ -private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging with SecretKeyHolder { +private[spark] class SecurityManager( + val sparkConf: SparkConf) extends Logging with SecretKeyHolder { // key used to store the spark secret in the Hadoop UGI private val sparkSecretLookupKey = "sparkCookie" diff --git a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala index df4b085d2251e..302b496b8a849 100644 --- a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala +++ b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala @@ -164,7 +164,7 @@ private[nio] class ConnectionManager( serverChannel.socket.bind(new InetSocketAddress(port)) (serverChannel, serverChannel.socket.getLocalPort) } - Utils.startServiceOnPort[ServerSocketChannel](port, startService, name) + Utils.startServiceOnPort[ServerSocketChannel](port, startService, conf, name) serverChannel.register(selector, SelectionKey.OP_ACCEPT) val id = new ConnectionManagerId(Utils.localHostName, serverChannel.socket.getLocalPort) diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 2a27d49d2de05..88fed833f922d 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -201,7 +201,7 @@ private[spark] object JettyUtils extends Logging { } } - val (server, boundPort) = Utils.startServiceOnPort[Server](port, connect, serverName) + val (server, boundPort) = Utils.startServiceOnPort[Server](port, connect, conf, serverName) ServerInfo(server, boundPort, collection) } diff --git a/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala b/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala index 10010bdfa1a51..3505346ac44ba 100644 --- a/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala +++ b/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala @@ -53,7 +53,7 @@ private[spark] object AkkaUtils extends Logging { val startService: Int => (ActorSystem, Int) = { actualPort => doCreateActorSystem(name, host, actualPort, conf, securityManager) } - Utils.startServiceOnPort(port, startService, name) + Utils.startServiceOnPort(port, startService, conf, name) } private def doCreateActorSystem( diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index eb4a598dbf857..f830d4bd42905 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1652,25 +1652,28 @@ private[spark] object Utils extends Logging { * Attempt to start a service on the given port, or fail after a number of attempts. * Each subsequent attempt uses 1 + the port used in the previous attempt (unless the port is 0). * - * @param startPort The initial port to start the service on. + * @param port The initial port to start the service on. * @param maxRetries Maximum number of retries to attempt. * A value of 3 means attempting ports n, n+1, n+2, and n+3, for example. * @param startService Function to start service on a given port. * This is expected to throw java.net.BindException on port collision. */ def startServiceOnPort[T]( - startPort: Int, + port: Int, startService: Int => (T, Int), + conf: SparkConf = new SparkConf(), serviceName: String = "", maxRetries: Int = portMaxRetries): (T, Int) = { val serviceString = if (serviceName.isEmpty) "" else s" '$serviceName'" + val startPort = conf.getInt("spark.port.min", 0) + val endPort = conf.getInt("spark.port.max", 65536) for (offset <- 0 to maxRetries) { - // Do not increment port if startPort is 0, which is treated as a special port - val tryPort = if (startPort == 0) { - startPort + // Do not increment port if port is 0, which is treated as a special port + val tryPort = if (port == 0) { + (startPort + Math.random() * (endPort - startPort)).toInt } else { // If the new port wraps around, do not try a privilege port - ((startPort + offset - 1024) % (65536 - 1024)) + 1024 + ((port + offset - 1024) % (65536 - 1024)) + 1024 } try { val (service, port) = startService(tryPort) From 1773b12115db5242feed4532126ff5a26cd35c7a Mon Sep 17 00:00:00 2001 From: w00228970 Date: Mon, 17 Nov 2014 19:58:08 +0800 Subject: [PATCH 3/5] Revert "https -> http in pom" This reverts commit 5f5fda8244c59970d117a0e9d32bd8bddde348f2. --- pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 26fa0e823edbf..639ea22a1fda3 100644 --- a/pom.xml +++ b/pom.xml @@ -156,7 +156,7 @@ central Maven Repository - http://repo1.maven.org/maven2 + https://repo1.maven.org/maven2 true @@ -167,7 +167,7 @@ apache-repo Apache Repository - http://repository.apache.org/content/repositories/releases + https://repository.apache.org/content/repositories/releases true @@ -178,7 +178,7 @@ jboss-repo JBoss Repository - http://repository.jboss.org/nexus/content/repositories/releases + https://repository.jboss.org/nexus/content/repositories/releases true @@ -189,7 +189,7 @@ mqtt-repo MQTT Repository - http://repo.eclipse.org/content/repositories/paho-releases + https://repo.eclipse.org/content/repositories/paho-releases true @@ -200,7 +200,7 @@ cloudera-repo Cloudera Repository - http://repository.cloudera.com/artifactory/cloudera-repos + https://repository.cloudera.com/artifactory/cloudera-repos true @@ -222,7 +222,7 @@ spring-releases Spring Release Repository - http://repo.spring.io/libs-release + https://repo.spring.io/libs-release true @@ -234,7 +234,7 @@ spark-staging Spring Staging Repository - http://oss.sonatype.org/content/repositories/orgspark-project-1085 + https://oss.sonatype.org/content/repositories/orgspark-project-1085 true @@ -246,7 +246,7 @@ spark-staging-hive13 Spring Staging Repository Hive 13 - http://oss.sonatype.org/content/repositories/orgspark-project-1089/ + https://oss.sonatype.org/content/repositories/orgspark-project-1089/ true @@ -258,7 +258,7 @@ central - http://repo1.maven.org/maven2 + https://repo1.maven.org/maven2 true From cd1a88dd462373ce890bfc0eaaa455fc82ee76b4 Mon Sep 17 00:00:00 2001 From: w00228970 Date: Tue, 18 Nov 2014 11:42:22 +0800 Subject: [PATCH 4/5] fix 'else' branch and doc update --- .../main/scala/org/apache/spark/util/Utils.scala | 6 +++--- docs/configuration.md | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index f830d4bd42905..59148fe1ea7da 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1665,15 +1665,15 @@ private[spark] object Utils extends Logging { serviceName: String = "", maxRetries: Int = portMaxRetries): (T, Int) = { val serviceString = if (serviceName.isEmpty) "" else s" '$serviceName'" - val startPort = conf.getInt("spark.port.min", 0) + val startPort = conf.getInt("spark.port.min", 1024) val endPort = conf.getInt("spark.port.max", 65536) for (offset <- 0 to maxRetries) { // Do not increment port if port is 0, which is treated as a special port val tryPort = if (port == 0) { (startPort + Math.random() * (endPort - startPort)).toInt } else { - // If the new port wraps around, do not try a privilege port - ((port + offset - 1024) % (65536 - 1024)) + 1024 + // If the new port wraps around, ensure it is in range(startPort, endPort) + ((port + offset - startPort) % (endPort - startPort)) + startPort } try { val (service, port) = startService(tryPort) diff --git a/docs/configuration.md b/docs/configuration.md index 8839162c3a13e..e29f48364f300 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -183,6 +183,20 @@ of the most common options to set are: Logs the effective SparkConf as INFO when a SparkContext is started. + + spark.port.min + 1024 + + Min port for spark(UI, HttpServer. ConnectionManager, Akka) + + + + spark.port.max + 65536 + + Max port for spark(UI, HttpServer. ConnectionManager, Akka) + + Apart from these, the following properties are also available, and may be useful in some situations: From da05d64f6ab0b97f86a481430ebcc3a7dc8099c3 Mon Sep 17 00:00:00 2001 From: scwf Date: Sat, 22 Nov 2014 22:42:13 +0800 Subject: [PATCH 5/5] inclusive-exclusive issue fix --- core/src/main/scala/org/apache/spark/util/Utils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 59148fe1ea7da..6ff75aed51421 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1670,10 +1670,10 @@ private[spark] object Utils extends Logging { for (offset <- 0 to maxRetries) { // Do not increment port if port is 0, which is treated as a special port val tryPort = if (port == 0) { - (startPort + Math.random() * (endPort - startPort)).toInt + port } else { // If the new port wraps around, ensure it is in range(startPort, endPort) - ((port + offset - startPort) % (endPort - startPort)) + startPort + ((port + offset) % (endPort - startPort + 1)) + startPort } try { val (service, port) = startService(tryPort)