From f060c02e51b6f1558ae28b0f1becb4271b94cf1d Mon Sep 17 00:00:00 2001 From: Benoy Antony Date: Thu, 4 Sep 2014 10:19:14 -0700 Subject: [PATCH 1/4] =?UTF-8?q?[SPARK-3286]=20-=20Cannot=20view=20Applicat?= =?UTF-8?q?ionMaster=20UI=20when=20Yarn=E2=80=99s=20url=20scheme=20is=20ht?= =?UTF-8?q?tps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala | 2 +- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala index cc5392192ec51..ca83ad5af3741 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala @@ -96,7 +96,7 @@ private class YarnRMClientImpl(args: ApplicationMasterArguments) extends YarnRMC // Users can then monitor stderr/stdout on that node if required. appMasterRequest.setHost(Utils.localHostName()) appMasterRequest.setRpcPort(0) - appMasterRequest.setTrackingUrl(uiAddress) + appMasterRequest.setTrackingUrl(uiAddress.replaceAll("^http(\\w)*://", "")) resourceManager.registerApplicationMaster(appMasterRequest) } diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 98039a20de245..ff9ddff25bf15 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -188,7 +188,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, if (sc == null) { finish(FinalApplicationStatus.FAILED, "Timed out waiting for SparkContext.") } else { - registerAM(sc.ui.appUIHostPort) + registerAM(sc.ui.appUIAddress) try { userThread.join() } finally { From 450c5369cf21392ff312731f46f15b358ae69bbc Mon Sep 17 00:00:00 2001 From: Benoy Antony Date: Fri, 5 Sep 2014 16:32:56 -0700 Subject: [PATCH 2/4] =?UTF-8?q?[SPARK-3286]=20-=20Cannot=20view=20Applicat?= =?UTF-8?q?ionMaster=20UI=20when=20Yarn=E2=80=99s=20url=20scheme=20is=20ht?= =?UTF-8?q?tps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/spark/deploy/yarn/YarnRMClientImpl.scala | 1 + .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala index ca83ad5af3741..2d536faf57897 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala @@ -96,6 +96,7 @@ private class YarnRMClientImpl(args: ApplicationMasterArguments) extends YarnRMC // Users can then monitor stderr/stdout on that node if required. appMasterRequest.setHost(Utils.localHostName()) appMasterRequest.setRpcPort(0) + //remove the scheme from the url if it exists since Hadoop does not expect scheme appMasterRequest.setTrackingUrl(uiAddress.replaceAll("^http(\\w)*://", "")) resourceManager.registerApplicationMaster(appMasterRequest) } diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index ff9ddff25bf15..1a6bcd591fb8d 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -188,7 +188,11 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, if (sc == null) { finish(FinalApplicationStatus.FAILED, "Timed out waiting for SparkContext.") } else { - registerAM(sc.ui.appUIAddress) + var uiAddress = sc.ui.appUIHostPort + if (yarnConf.get("yarn.http.policy").equals("HTTPS_ONLY")) { + uiAddress = sc.ui.appUIAddress + } + registerAM(uiAddress) try { userThread.join() } finally { From e82f94ede50a5aae986ff53a9c5a63d5de226f24 Mon Sep 17 00:00:00 2001 From: Benoy Antony Date: Tue, 9 Sep 2014 12:06:56 -0700 Subject: [PATCH 3/4] Use address with scheme, but Allpha version removes the scheme --- .../org/apache/spark/deploy/yarn/YarnRMClientImpl.scala | 3 ++- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala index 14c6c5b0c4d1d..fc30953011812 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala @@ -18,6 +18,7 @@ package org.apache.spark.deploy.yarn import scala.collection.{Map, Set} +import java.net.URI; import org.apache.hadoop.net.NetUtils import org.apache.hadoop.yarn.api._ @@ -98,7 +99,7 @@ private class YarnRMClientImpl(args: ApplicationMasterArguments) extends YarnRMC appMasterRequest.setHost(Utils.localHostName()) appMasterRequest.setRpcPort(0) //remove the scheme from the url if it exists since Hadoop does not expect scheme - appMasterRequest.setTrackingUrl(uiAddress.replaceAll("^http(\\w)*://", "")) + appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority()) resourceManager.registerApplicationMaster(appMasterRequest) } diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index c5e393d6a0d5a..0337af05da1d6 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -189,11 +189,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, if (sc == null) { finish(FinalApplicationStatus.FAILED, "Timed out waiting for SparkContext.") } else { - var uiAddress = sc.ui.appUIHostPort - if (yarnConf.get("yarn.http.policy").equals("HTTPS_ONLY")) { - uiAddress = sc.ui.appUIAddress - } - registerAM(uiAddress, securityMgr) + registerAM(sc.ui.appUIAddress, securityMgr) try { userThread.join() } finally { From c3d51ee040b68191af8f9c6a91a8163fba589862 Mon Sep 17 00:00:00 2001 From: Benoy Antony Date: Tue, 9 Sep 2014 12:11:13 -0700 Subject: [PATCH 4/4] Use address with scheme, but Allpha version removes the scheme --- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 0337af05da1d6..5756263e89e21 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -189,7 +189,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, if (sc == null) { finish(FinalApplicationStatus.FAILED, "Timed out waiting for SparkContext.") } else { - registerAM(sc.ui.appUIAddress, securityMgr) + registerAM(sc.ui.appUIAddress, securityMgr) try { userThread.join() } finally {