From 884f2024fb4dd777cfc4849dc52ef76e978abec6 Mon Sep 17 00:00:00 2001 From: gaurangi Date: Mon, 4 May 2020 21:37:07 +0000 Subject: [PATCH] Add Host name to history server web page Change-Id: I59a467a49d417aa76548fb8e00f1582b09b9e041 --- .../jobhistory/JobHistoryEventHandler.java | 13 +++++++++++++ .../hadoop/mapreduce/v2/app/job/Job.java | 1 + .../mapreduce/v2/app/job/impl/JobImpl.java | 7 ++++++- .../hadoop/mapreduce/v2/app/MockJobs.java | 5 +++++ .../v2/app/TestRuntimeEstimators.java | 5 +++++ .../v2/jobhistory/FileNameIndexUtils.java | 18 ++++++++++++++++++ .../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 ++++++- .../mapreduce/v2/jobhistory/JobIndexInfo.java | 8 ++++++++ .../hadoop/mapreduce/v2/hs/CompletedJob.java | 5 +++++ .../hadoop/mapreduce/v2/hs/PartialJob.java | 5 +++++ .../hadoop/mapreduce/v2/hs/UnparsedJob.java | 5 +++++ .../mapreduce/v2/hs/webapp/HsJobsBlock.java | 10 ++++++++++ .../mapreduce/v2/hs/webapp/TestBlocks.java | 1 + .../v2/hs/webapp/TestHsWebServicesAcls.java | 5 +++++ 14 files changed, 93 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index c7e69af8a8140..f67197aaae85c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -1588,6 +1588,19 @@ protected class MetaInfo { this.jobIndexInfo = new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null, queueName); + + if (getConfig().getBoolean( + JHAdminConfig.MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED, + JHAdminConfig.DEFAULT_MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED)) { + + String hostName = getConfig().get(YarnConfiguration.RM_HOSTNAME, ""); + if (!hostName.isEmpty()) { + this.jobIndexInfo.setResourceManagerHost(hostName); + } else { + LOG.warn("Could not retreive a valid YARN host name"); + } + } + this.jobSummary = new JobSummary(); this.flushTimer = new Timer("FlushTimer", true); this.forcedJobStateOnShutDown = forcedJobStateOnShutDown; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java index 437707b7019d6..c80f8b3709205 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java @@ -73,6 +73,7 @@ public interface Job { boolean isUber(); String getUserName(); String getQueueName(); + String getResourceManagerHost(); /** * @return a path to where the config file for this job is located. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 8ee097fe243e1..64b280c480743 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -1162,7 +1162,12 @@ public String getUserName() { public String getQueueName() { return queueName; } - + + @Override + public String getResourceManagerHost() { + throw new UnsupportedOperationException(); + } + @Override public void setQueueName(String queueName) { this.queueName = queueName; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index bfb8d79de9391..7d50501eb13d1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -605,6 +605,11 @@ public String getQueueName() { return "mockqueue"; } + @Override + public String getResourceManagerHost() { + return "mockresourcemanager"; + } + @Override public Path getConfFile() { return configFile; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 0031598da5b33..0eda32ae95584 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -490,6 +490,11 @@ public String getQueueName() { throw new UnsupportedOperationException("Not supported yet."); } + @Override + public String getResourceManagerHost() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public int getTotalMaps() { return mapTasks.size(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java index c5f4dbba665e3..457dd45faa739 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java @@ -52,6 +52,7 @@ public class FileNameIndexUtils { private static final int JOB_STATUS_INDEX = 7; private static final int QUEUE_NAME_INDEX = 8; private static final int JOB_START_TIME_INDEX = 9; + private static final int RESOURCE_MANAGER_INDEX = 10; /** * Constructs the job history file name from the JobIndexInfo. @@ -116,6 +117,13 @@ public static String getDoneFileName(JobIndexInfo indexInfo, sb.append(encodeJobHistoryFileName( String.valueOf(indexInfo.getJobStartTime()))); + String resourceManagerName = getResourceManagerHost(indexInfo); + if (resourceManagerName != null) { + sb.append(DELIMITER); + sb.append(escapeDelimiters(encodeJobHistoryFileName( + getResourceManagerHost(indexInfo)))); + } + sb.append(encodeJobHistoryFileName( JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION)); return sb.toString(); @@ -198,6 +206,12 @@ public static JobIndexInfo getIndexInfo(String jhFileName) LOG.warn("Unable to parse start time from job history file " + jhFileName + " : " + e); } + + if (jobDetails.length > RESOURCE_MANAGER_INDEX) { + indexInfo.setResourceManagerHost(decodeJobHistoryFileName(jobDetails[RESOURCE_MANAGER_INDEX])); + } else { + LOG.warn("Could not parse cluster name from job history file " + jhFileName ); + } } catch (IndexOutOfBoundsException e) { LOG.warn("Parsing job history file with partial data encoded into name: " + jhFileName); @@ -292,6 +306,10 @@ private static String getQueueName(JobIndexInfo indexInfo) { return getNonEmptyString(indexInfo.getQueueName()); } + private static String getResourceManagerHost(JobIndexInfo indexInfo) { + return getNonEmptyString(indexInfo.getResourceManagerHost()); + } + //TODO Maybe handle default values for longs and integers here? private static String getNonEmptyString(String in) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 1c11e7728b24a..09718e26bd4c3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -102,7 +102,12 @@ public class JHAdminConfig { MR_HISTORY_PREFIX + "intermediate-user-done-dir.permissions"; public static final short DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = 0770; - + + public static final String MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED = + MR_HISTORY_PREFIX + "append-rm-host-to-history-file-name.enabled"; + public static final boolean DEFAULT_MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED = + true; + /** Size of the job list cache.*/ public static final String MR_HISTORY_JOBLIST_CACHE_SIZE = MR_HISTORY_PREFIX + "joblist.cache.size"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java index ef10294a5ac69..056f4bafe5b72 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java @@ -36,6 +36,7 @@ public class JobIndexInfo { private int numReduces; private String jobStatus; private long jobStartTime; + private String resourceManagerHost; public JobIndexInfo() { } @@ -59,6 +60,7 @@ public JobIndexInfo(long submitTime, long finishTime, String user, this.jobStatus = jobStatus; this.jobStartTime = -1; this.queueName = queueName; + this.resourceManagerHost = null; } public long getSubmitTime() { @@ -121,6 +123,12 @@ public long getJobStartTime() { public void setJobStartTime(long lTime) { this.jobStartTime = lTime; } + public String getResourceManagerHost() { + return resourceManagerHost; + } + public void setResourceManagerHost(String resourceManagerHost) { + this.resourceManagerHost = resourceManagerHost; + } @Override public String toString() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index e9a6a641b5cf7..12545648a4481 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -416,6 +416,11 @@ public String getQueueName() { return jobInfo.getJobQueueName(); } + @Override + public String getResourceManagerHost() { + throw new UnsupportedOperationException(); + } + @Override public int getTotalMaps() { return (int) jobInfo.getTotalMaps(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java index 26755d7a8afa8..336c5c60ce970 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java @@ -76,6 +76,11 @@ public String getQueueName() { return jobIndexInfo.getQueueName(); } + @Override + public String getResourceManagerHost() { + return jobIndexInfo.getResourceManagerHost(); + } + @Override public JobState getState() { JobState js = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java index ecc494552831c..b049a92b8e47b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java @@ -159,6 +159,11 @@ public String getQueueName() { return jobIndexInfo.getQueueName(); } + @Override + public String getResourceManagerHost() { + return jobIndexInfo.getResourceManagerHost(); + } + @Override public Path getConfFile() { return jhfInfo.getConfFile(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java index 6a83ac2625be2..cf014ce93abff 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java @@ -76,6 +76,7 @@ public class HsJobsBlock extends HtmlBlock { th(".id", "Job ID"). th(".name", "Name"). th("User"). + th(".resourceManagerHost", "Resource Manager Host"). th("Queue"). th(".state", "State"). th("Maps Total"). @@ -96,6 +97,11 @@ public class HsJobsBlock extends HtmlBlock { .checkAccess(ugi, JobACL.VIEW_JOB, job.getUserName(), null)) { continue; } + + String resourceManagerHost = ""; + if (j.getResourceManagerHost() != null) { + resourceManagerHost = j.getResourceManagerHost(); + } jobsTableData.append("[\"") .append(dateFormat.format(new Date(job.getSubmitTime()))).append("\",\"") .append(job.getFormattedStartTimeStr(dateFormat)).append("\",\"") @@ -106,6 +112,8 @@ public class HsJobsBlock extends HtmlBlock { job.getName()))).append("\",\"") .append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils.escapeHtml4( job.getUserName()))).append("\",\"") + .append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils.escapeHtml4( + resourceManagerHost))).append("\",\"") .append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils.escapeHtml4( job.getQueueName()))).append("\",\"") .append(job.getState()).append("\",\"") @@ -140,6 +148,8 @@ public class HsJobsBlock extends HtmlBlock { .$name("name").$value("Name").__().__(). th().input("search_init").$type(InputType.text) .$name("user").$value("User").__().__(). + th().input("search_init").$type(InputType.text) + .$name("resourceManagerHost").$value("Resource Manager Host").__().__(). th().input("search_init").$type(InputType.text) .$name("queue").$value("Queue").__().__(). th().input("search_init").$type(InputType.text) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java index 337f182c681ee..0f43c4e9bd23d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java @@ -398,6 +398,7 @@ private Job getJob() { when(job.getReport()).thenReturn(report); when(job.getName()).thenReturn("JobName"); when(job.getUserName()).thenReturn("UserName"); + when(job.getResourceManagerHost()).thenReturn("ResourceManagerHost"); when(job.getQueueName()).thenReturn("QueueName"); when(job.getState()).thenReturn(JobState.SUCCEEDED); when(job.getTotalMaps()).thenReturn(3); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java index 960993ed7f706..c41a5a5ead8e0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java @@ -378,6 +378,11 @@ public String getQueueName() { return mockJob.getQueueName(); } + @Override + public String getResourceManagerHost() { + return mockJob.getResourceManagerHost(); + } + @Override public Path getConfFile() { return new Path("/some/path/to/conf");