From 33044679cb666b03e87084df32e0564d20566aff Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Oct 2023 11:57:24 +0800 Subject: [PATCH 1/9] YARN-11593. [Federation] Improve command line help information. --- .../hadoop/yarn/client/cli/RouterCLI.java | 219 ++++++++++++------ 1 file changed, 149 insertions(+), 70 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 0aa02c8124a3a..d1f26fffa58b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -23,6 +23,8 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -63,10 +65,7 @@ import java.io.PrintWriter; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid; @@ -77,25 +76,75 @@ public class RouterCLI extends Configured implements Tool { private static final Logger LOG = LoggerFactory.getLogger(RouterCLI.class); - protected final static Map ADMIN_USAGE = - ImmutableMap.builder() - // Command1: deregisterSubCluster - .put("-deregisterSubCluster", new UsageInfo( - "[-sc|--subClusterId [subCluster Id]]", - "Deregister SubCluster, If the interval between the heartbeat time of the subCluster " + - "and the current time exceeds the timeout period, " + - "set the state of the subCluster to SC_LOST.")) - // Command2: policy - .put("-policy", new UsageInfo( - "[-s|--save [queue;router weight;amrm weight;headroomalpha]] " + - "[-bs|--batch-save [--format xml] [-f|--input-file fileName]]" + - "[-l|--list [--pageSize][--currentPage][--queue][--queues]]", - "We provide a set of commands for Policy:" + - " Include list policies, save policies, batch save policies. " + - " (Note: The policy type will be directly read from the" + - " yarn.federation.policy-manager in the local yarn-site.xml.)" + - " eg. (routeradmin -policy [-s|--save] root.a;SC-1:0.7,SC-2:0.3;SC-1:0.7,SC-2:0.3;1.0)")) - .build(); + protected final static UsageInfo subClusterId = new UsageInfo("<-sc|--subClusterId>", + "'-sc' option allows you to specify the sub-cluster to operate on, " + + "while the '--subClusterId' option is the long format of -sc and serves the same purpose."); + protected final static String dsExample_1 = "yarn routeradmin -deregisterSubCluster -sc SC-1"; + protected final static String dsExample_2 = "yarn routeradmin -deregisterSubCluster --subClusterId SC-1"; + + protected final static RouterExampleUsageInfos dsExampleUsageInfos_1 = new RouterExampleUsageInfos( + Arrays.asList(dsExample_1, dsExample_2), + Arrays.asList("At this point we can use the following command:")); + protected final static RouterUsageInfos deregisterSubCluster = + new RouterUsageInfos(Arrays.asList(subClusterId), + Arrays.asList("deregister subCluster, If the interval between the heartbeat time of the subCluster" + + "and the current time exceeds the timeout period, set the state of the subCluster to SC_LOST."), + ImmutableMap.builder() + .put("<--deregisterSubCluster>", dsExampleUsageInfos_1) + .build()); + + protected final static UsageInfo policySaveUsage = new UsageInfo( + "<-s|--save [queue;router weight;amrm weight;headroomalpha]>", + "We will save the policy information of the queue, including queue and weight information."); + protected final static UsageInfo policyBatchSaveUsage = new UsageInfo( + "<-bs|--batch-save [--format xml] [-f|--input-file fileName]]>", + "We will save queue policies in bulk, where users can provide XML files containing the policies."); + protected final static UsageInfo policyListUsage = new UsageInfo("<-l|--list [--pageSize][--currentPage][--queue][--queues]>", + "We can display the configured queue policies."); + + protected final static String pExample_1 = "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + protected final static String pExample_2 = "yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + protected final static RouterExampleUsageInfos pExampleUsageInfos_1 = new RouterExampleUsageInfos( + Arrays.asList(pExample_1, pExample_2), + Arrays.asList("We have two subClusters, SC-1 and SC-2. For queue root.a, we want the ratio to be set to SC-1:SC-2=0.7:0.3 when routing.", + "Additionally, when AM applies containers, we want the allocation ratio to be SC-1:SC-2=0.6:0.4.", + "Headroomalpha can be set to the default value of 1.0.", + "At this point we can use the following command:")); + + protected final static String pExample_3 = "yarn routeradmin -policy -bs --format xml -f queue.xml"; + protected final static String pExample_4 = "yarn routeradmin -policy --batch-save --format xml -f queue.xml"; + protected final static RouterExampleUsageInfos pExampleUsageInfos_2 = new RouterExampleUsageInfos( + Arrays.asList(pExample_3, pExample_4), + Arrays.asList("We offer the capability to batch input queue policies. ", + "Currently, this is supported exclusively through XML. " + + "(We can find federationWeights.xml in the Federation.md on the website.) ", + "We can use the batch-save command to input proportion information. ", + "At this point we can use the following command:")); + + protected final static String pExample_5 = "yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a"; + protected final static String pExample_6 = "yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b"; + protected final static RouterExampleUsageInfos pExampleUsageInfos_3 = new RouterExampleUsageInfos( + Arrays.asList(pExample_5, pExample_6), + Arrays.asList("We can print the Configured Queue Policies on the command line. ", + "We can specify a queue through --queue, and we can specify a queue list through --queues.", + "At this point we can use the following command:")); + + protected final static RouterUsageInfos policy = + new RouterUsageInfos(Arrays.asList(policySaveUsage, policyBatchSaveUsage, policyListUsage), + Arrays.asList("We provide a set of commands for Policy Include list policies, save policies, batch save policies."), + ImmutableMap.builder() + .put("<-s|--save>", pExampleUsageInfos_1) + .put("<-bs|--batch-save>", pExampleUsageInfos_2) + .put("<-l|--list>", pExampleUsageInfos_3) + .build()); + + protected final static Map ADMIN_USAGE = + ImmutableMap.builder() + // Command1: deregisterSubCluster + .put("-deregisterSubCluster", deregisterSubCluster) + // Command2: policy + .put("-policy", policy) + .build(); // Common Constant private static final String SEMICOLON = ";"; @@ -135,7 +184,7 @@ public class RouterCLI extends Configured implements Tool { private static final String CMD_POLICY = "-policy"; private static final String FORMAT_XML = "xml"; - private static final String FORMAT_JSON = "json"; + private static final String XML_TAG_SUBCLUSTERIDINFO = "subClusterIdInfo"; private static final String XML_TAG_AMRMPOLICYWEIGHTS = "amrmPolicyWeights"; private static final String XML_TAG_ROUTERPOLICYWEIGHTS = "routerPolicyWeights"; @@ -159,43 +208,59 @@ public RouterCLI(Configuration conf) { } private static void buildHelpMsg(String cmd, StringBuilder builder) { - UsageInfo usageInfo = ADMIN_USAGE.get(cmd); - if (usageInfo == null) { + RouterUsageInfos routerUsageInfo = ADMIN_USAGE.get(cmd); + + if (routerUsageInfo == null) { return; } + builder.append("[" + cmd + "]\n"); - if (usageInfo.args != null) { - String space = (usageInfo.args == "") ? "" : " "; - builder.append(" ") - .append(cmd) - .append(space) - .append(usageInfo.args) - .append(": ") - .append(usageInfo.help); - } else { - builder.append(" ") - .append(cmd) - .append(": ") - .append(usageInfo.help); + if (!routerUsageInfo.helpInfos.isEmpty()) { + builder.append("\t Description: \n"); + for (String helpInfo : routerUsageInfo.helpInfos) { + builder.append("\t\t" + helpInfo).append("\n\n"); + } } - } - private static void buildIndividualUsageMsg(String cmd, StringBuilder builder) { - UsageInfo usageInfo = ADMIN_USAGE.get(cmd); - if (usageInfo == null) { - return; + if (!routerUsageInfo.usageInfos.isEmpty()) { + builder.append("\t UsageInfos: \n"); + for (UsageInfo usageInfo : routerUsageInfo.usageInfos) { + builder.append("\t\t"+usageInfo.args) + .append(": ") + .append("\n\t\t") + .append(usageInfo.help).append("\n\n"); + } } - if (usageInfo.args == null) { - builder.append("Usage: routeradmin [") - .append(cmd) - .append("]\n"); - } else { - String space = (usageInfo.args == "") ? "" : " "; - builder.append("Usage: routeradmin [") - .append(cmd) - .append(space) - .append(usageInfo.args) - .append("]\n"); + + if (MapUtils.isNotEmpty(routerUsageInfo.examples)) { + builder.append("\t Examples: \n"); + int count = 1; + for (Map.Entry example : + routerUsageInfo.examples.entrySet()) { + String key = example.getKey(); + builder.append("\t\t").append("Cmd:" + count + ". ").append(key).append(": \n\n"); + RouterExampleUsageInfos values = example.getValue(); + if (values == null) { + return; + } + // Print Command Description + if (CollectionUtils.isNotEmpty(values.desc)) { + builder.append("\t\t").append("Cmd Requirement Description:\n"); + for (String value : values.desc) { + builder.append("\t\t").append(value).append("\n"); + } + } + builder.append("\n"); + // Print Command example + if (CollectionUtils.isNotEmpty(values.examples)) { + builder.append("\t\t").append("Cmd Examples:\n"); + for (String value : values.examples) { + builder.append("\t\t").append(value).append("\n"); + } + } + builder.append("\n"); + count++; + } } } @@ -204,12 +269,7 @@ private static void printHelp() { summary.append("routeradmin is the command to execute ") .append("YARN Federation administrative commands.\n") .append("The full syntax is: \n\n") - .append("routeradmin\n") - .append(" [-deregisterSubCluster [-sc|--subClusterId [subCluster Id]]\n") - .append(" [-policy [-s|--save [queue;router weight;amrm weight;headroomalpha] " + - "[-bs|--batch-save [--format xml,json] [-f|--input-file fileName]]] " + - "[-l|--list [--pageSize][--currentPage][--queue][--queues]]\n") - .append(" [-help [cmd]]").append("\n"); + .append("routeradmin\n"); StringBuilder helpBuilder = new StringBuilder(); System.out.println(summary); @@ -235,13 +295,9 @@ protected ResourceManagerAdministrationProtocol createAdminProtocol() private static void buildUsageMsg(StringBuilder builder) { builder.append("routeradmin is only used in Yarn Federation Mode.\n"); builder.append("Usage: routeradmin\n"); - for (Map.Entry cmdEntry : ADMIN_USAGE.entrySet()) { - UsageInfo usageInfo = cmdEntry.getValue(); - builder.append(" ") - .append(cmdEntry.getKey()) - .append(" ") - .append(usageInfo.args) - .append("\n"); + for (String cmdKey : ADMIN_USAGE.keySet()) { + buildHelpMsg(cmdKey, builder); + builder.append("\n"); } builder.append(" -help [cmd]\n"); } @@ -249,7 +305,7 @@ private static void buildUsageMsg(StringBuilder builder) { private static void printUsage(String cmd) { StringBuilder usageBuilder = new StringBuilder(); if (ADMIN_USAGE.containsKey(cmd)) { - buildIndividualUsageMsg(cmd, usageBuilder); + buildHelpMsg(cmd, usageBuilder); } else { buildUsageMsg(usageBuilder); } @@ -353,7 +409,7 @@ private int handlePolicy(String[] args) saveOpt.setOptionalArg(true); Option batchSaveOpt = new Option(OPTION_BATCH_S, OPTION_BATCH_SAVE, false, "We will save queue policies in bulk, " + - "where users can provide XML or JSON files containing the policies. " + + "where users can provide XML files containing the policies. " + "This command will parse the file contents and store the results " + "in the FederationStateStore."); Option formatOpt = new Option(null, "format", true, @@ -748,6 +804,29 @@ public int run(String[] args) throws Exception { return EXIT_SUCCESS; } + private static class RouterExampleUsageInfos { + public List examples; + public List desc; + + public RouterExampleUsageInfos(List examples, List desc) { + this.examples = examples; + this.desc = desc; + } + } + + private static class RouterUsageInfos { + public List usageInfos; + public List helpInfos; + public Map examples; + + public RouterUsageInfos(List usageInfos, List helpInfos, + Map examples) { + this.usageInfos = usageInfos; + this.helpInfos = helpInfos; + this.examples = examples; + } + } + public static void main(String[] args) throws Exception { int result = ToolRunner.run(new RouterCLI(), args); System.exit(result); From 306f6525dd33a19d2834bcd8c6dc985b462d9db4 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Oct 2023 12:33:52 +0800 Subject: [PATCH 2/9] YARN-11593. Fix CheckStyle. --- .../java/org/apache/hadoop/yarn/client/cli/RouterCLI.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index d1f26fffa58b2..da29d5eca75b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -65,7 +65,10 @@ import java.io.PrintWriter; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid; From 8ccd9af8b0cee1edfea514e0a4380f6f268f604d Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Oct 2023 19:24:13 +0800 Subject: [PATCH 3/9] YARN-11593. Fix CheckStyle. --- .../hadoop/yarn/client/cli/RouterCLI.java | 86 ++++++++++--------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index da29d5eca75b1..2188e0321d1f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -79,76 +79,70 @@ public class RouterCLI extends Configured implements Tool { private static final Logger LOG = LoggerFactory.getLogger(RouterCLI.class); - protected final static UsageInfo subClusterId = new UsageInfo("<-sc|--subClusterId>", + protected final static UsageInfo SUBCLUSTER_ID = new UsageInfo("<-sc|--subClusterId>", "'-sc' option allows you to specify the sub-cluster to operate on, " + "while the '--subClusterId' option is the long format of -sc and serves the same purpose."); - protected final static String dsExample_1 = "yarn routeradmin -deregisterSubCluster -sc SC-1"; - protected final static String dsExample_2 = "yarn routeradmin -deregisterSubCluster --subClusterId SC-1"; + protected final static String DSEXAMPLE_1 = "yarn routeradmin -deregisterSubCluster -sc SC-1"; + protected final static String DSEXAMPLE_2 = "yarn routeradmin -deregisterSubCluster --subClusterId SC-1"; - protected final static RouterExampleUsageInfos dsExampleUsageInfos_1 = new RouterExampleUsageInfos( - Arrays.asList(dsExample_1, dsExample_2), + protected final static RouterExampleUsageInfos DSEXAMPLEUSAGEINFOS_1 = new RouterExampleUsageInfos( + Arrays.asList(DSEXAMPLE_1, DSEXAMPLE_2), Arrays.asList("At this point we can use the following command:")); - protected final static RouterUsageInfos deregisterSubCluster = - new RouterUsageInfos(Arrays.asList(subClusterId), - Arrays.asList("deregister subCluster, If the interval between the heartbeat time of the subCluster" + + protected final static RouterUsageInfos DEREGISTER_SUBCLUSTER = + new RouterUsageInfos(Arrays.asList(SUBCLUSTER_ID), + Arrays.asList("deregister subCluster, " + + "If the interval between the heartbeat time of the subCluster" + "and the current time exceeds the timeout period, set the state of the subCluster to SC_LOST."), ImmutableMap.builder() - .put("<--deregisterSubCluster>", dsExampleUsageInfos_1) + .put("<--deregisterSubCluster>", DSEXAMPLEUSAGEINFOS_1) .build()); - protected final static UsageInfo policySaveUsage = new UsageInfo( + protected final static UsageInfo POLICY_SAVE_USAGE = new UsageInfo( "<-s|--save [queue;router weight;amrm weight;headroomalpha]>", "We will save the policy information of the queue, including queue and weight information."); - protected final static UsageInfo policyBatchSaveUsage = new UsageInfo( + protected final static UsageInfo POLICY_BATCH_SAVE_USAGE = new UsageInfo( "<-bs|--batch-save [--format xml] [-f|--input-file fileName]]>", "We will save queue policies in bulk, where users can provide XML files containing the policies."); - protected final static UsageInfo policyListUsage = new UsageInfo("<-l|--list [--pageSize][--currentPage][--queue][--queues]>", + protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo("<-l|--list [--pageSize][--currentPage][--queue][--queues]>", "We can display the configured queue policies."); - protected final static String pExample_1 = "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; - protected final static String pExample_2 = "yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; - protected final static RouterExampleUsageInfos pExampleUsageInfos_1 = new RouterExampleUsageInfos( - Arrays.asList(pExample_1, pExample_2), - Arrays.asList("We have two subClusters, SC-1 and SC-2. For queue root.a, we want the ratio to be set to SC-1:SC-2=0.7:0.3 when routing.", + protected final static String PEXAMPLE_1 = "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + protected final static String PEXAMPLE_2 = "yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_1 = new RouterExampleUsageInfos( + Arrays.asList(PEXAMPLE_1, PEXAMPLE_2), + Arrays.asList("We have two subClusters, SC-1 and SC-2. " + + "For queue root.a, we want the ratio to be set to SC-1:SC-2=0.7:0.3 when routing.", "Additionally, when AM applies containers, we want the allocation ratio to be SC-1:SC-2=0.6:0.4.", "Headroomalpha can be set to the default value of 1.0.", "At this point we can use the following command:")); - protected final static String pExample_3 = "yarn routeradmin -policy -bs --format xml -f queue.xml"; - protected final static String pExample_4 = "yarn routeradmin -policy --batch-save --format xml -f queue.xml"; - protected final static RouterExampleUsageInfos pExampleUsageInfos_2 = new RouterExampleUsageInfos( - Arrays.asList(pExample_3, pExample_4), + protected final static String PEXAMPLE_3 = "yarn routeradmin -policy -bs --format xml -f queue.xml"; + protected final static String PEXAMPLE_4 = "yarn routeradmin -policy --batch-save --format xml -f queue.xml"; + protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_2 = new RouterExampleUsageInfos( + Arrays.asList(PEXAMPLE_3, PEXAMPLE_4), Arrays.asList("We offer the capability to batch input queue policies. ", "Currently, this is supported exclusively through XML. " + "(We can find federationWeights.xml in the Federation.md on the website.) ", "We can use the batch-save command to input proportion information. ", "At this point we can use the following command:")); - protected final static String pExample_5 = "yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a"; - protected final static String pExample_6 = "yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b"; - protected final static RouterExampleUsageInfos pExampleUsageInfos_3 = new RouterExampleUsageInfos( - Arrays.asList(pExample_5, pExample_6), + protected final static String PEXAMPLE_5 = "yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a"; + protected final static String PEXAMPLE_6 = "yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b"; + protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_3 = new RouterExampleUsageInfos( + Arrays.asList(PEXAMPLE_5, PEXAMPLE_6), Arrays.asList("We can print the Configured Queue Policies on the command line. ", "We can specify a queue through --queue, and we can specify a queue list through --queues.", "At this point we can use the following command:")); - protected final static RouterUsageInfos policy = - new RouterUsageInfos(Arrays.asList(policySaveUsage, policyBatchSaveUsage, policyListUsage), + protected final static RouterUsageInfos POLICY = + new RouterUsageInfos(Arrays.asList(POLICY_SAVE_USAGE, POLICY_BATCH_SAVE_USAGE, POLICY_LIST_USAGE), Arrays.asList("We provide a set of commands for Policy Include list policies, save policies, batch save policies."), ImmutableMap.builder() - .put("<-s|--save>", pExampleUsageInfos_1) - .put("<-bs|--batch-save>", pExampleUsageInfos_2) - .put("<-l|--list>", pExampleUsageInfos_3) + .put("<-s|--save>", PEXAMPLEUSAGEINFOS_1) + .put("<-bs|--batch-save>", PEXAMPLEUSAGEINFOS_2) + .put("<-l|--list>", PEXAMPLEUSAGEINFOS_3) .build()); - protected final static Map ADMIN_USAGE = - ImmutableMap.builder() - // Command1: deregisterSubCluster - .put("-deregisterSubCluster", deregisterSubCluster) - // Command2: policy - .put("-policy", policy) - .build(); - // Common Constant private static final String SEMICOLON = ";"; @@ -202,6 +196,14 @@ public class RouterCLI extends Configured implements Tool { private static final List LIST_POLICIES_HEADER = Arrays.asList( "Queue Name", "AMRM Weight", "Router Weight"); + protected final static Map ADMIN_USAGE = + ImmutableMap.builder() + // Command1: deregisterSubCluster + .put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER) + // Command2: policy + .put(CMD_POLICY, POLICY) + .build(); + public RouterCLI() { super(); } @@ -216,19 +218,19 @@ private static void buildHelpMsg(String cmd, StringBuilder builder) { if (routerUsageInfo == null) { return; } - builder.append("[" + cmd + "]\n"); + builder.append("[").append(cmd).append("]\n"); if (!routerUsageInfo.helpInfos.isEmpty()) { builder.append("\t Description: \n"); for (String helpInfo : routerUsageInfo.helpInfos) { - builder.append("\t\t" + helpInfo).append("\n\n"); + builder.append("\t\t").append(helpInfo).append("\n\n"); } } if (!routerUsageInfo.usageInfos.isEmpty()) { builder.append("\t UsageInfos: \n"); for (UsageInfo usageInfo : routerUsageInfo.usageInfos) { - builder.append("\t\t"+usageInfo.args) + builder.append("\t\t").append(usageInfo.args) .append(": ") .append("\n\t\t") .append(usageInfo.help).append("\n\n"); @@ -241,7 +243,7 @@ private static void buildHelpMsg(String cmd, StringBuilder builder) { for (Map.Entry example : routerUsageInfo.examples.entrySet()) { String key = example.getKey(); - builder.append("\t\t").append("Cmd:" + count + ". ").append(key).append(": \n\n"); + builder.append("\t\t").append("Cmd:").append(count).append(". ").append(key).append(": \n\n"); RouterExampleUsageInfos values = example.getValue(); if (values == null) { return; From 7da7960b1514fcd76146063e81100ae86f5287f0 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Oct 2023 20:17:25 +0800 Subject: [PATCH 4/9] YARN-11593. Fix CheckStyle. --- .../hadoop/yarn/client/cli/RouterCLI.java | 13 ++++----- .../src/site/markdown/Federation.md | 27 ++++++++++++++++++- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 2188e0321d1f8..5cebbbbe20fc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -98,12 +98,13 @@ public class RouterCLI extends Configured implements Tool { .build()); protected final static UsageInfo POLICY_SAVE_USAGE = new UsageInfo( - "<-s|--save [queue;router weight;amrm weight;headroomalpha]>", + "-s|--save ()", "We will save the policy information of the queue, including queue and weight information."); protected final static UsageInfo POLICY_BATCH_SAVE_USAGE = new UsageInfo( - "<-bs|--batch-save [--format xml] [-f|--input-file fileName]]>", + "-bs|--batch-save (--format ) (-f|--input-file )", "We will save queue policies in bulk, where users can provide XML files containing the policies."); - protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo("<-l|--list [--pageSize][--currentPage][--queue][--queues]>", + protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo( + "-l|--list [--pageSize][--currentPage][--queue][--queues]", "We can display the configured queue policies."); protected final static String PEXAMPLE_1 = "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; @@ -138,9 +139,9 @@ public class RouterCLI extends Configured implements Tool { new RouterUsageInfos(Arrays.asList(POLICY_SAVE_USAGE, POLICY_BATCH_SAVE_USAGE, POLICY_LIST_USAGE), Arrays.asList("We provide a set of commands for Policy Include list policies, save policies, batch save policies."), ImmutableMap.builder() - .put("<-s|--save>", PEXAMPLEUSAGEINFOS_1) - .put("<-bs|--batch-save>", PEXAMPLEUSAGEINFOS_2) - .put("<-l|--list>", PEXAMPLEUSAGEINFOS_3) + .put("-s|--save () ", PEXAMPLEUSAGEINFOS_1) + .put("-bs|--batch-save (--format ) (-f|--input-file )", PEXAMPLEUSAGEINFOS_2) + .put("-l|--list [--pageSize][--currentPage][--queue][--queues]", PEXAMPLEUSAGEINFOS_3) .build()); // Common Constant diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 66c79c94cc9f9..1d1f435e74c5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -465,9 +465,34 @@ If we want to use JCache, we can configure `yarn.federation.cache.class` to `org This is a Cache implemented based on the Guava framework. If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`. +Router command line: + +- deregisterSubCluster + +This command is used to `deregister subCluster`, If the interval between the heartbeat time of the subCluster, and the current time exceeds the timeout period, set the state of the subCluster to `SC_LOST`. + +Uasge: + +`yarn routeradmin -deregisterSubCluster [-sc|--subClusterId ]` + +Options: + +| Property | Example | +|:------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -sc, --subClusterId | '-sc' option allows you to specify the sub-cluster to operate on, while the '--subClusterId' option is the long format of -sc and serves the same purpose." | + +Examples: + +If we want to deregisterSubCluster `SC-1` + +- yarn routeradmin -deregisterSubCluster -sc SC-1 +- yarn routeradmin -deregisterSubCluster --subClusterId SC-1 + +- policy + ### ON GPG: -GlobalPolicyGenerator, abbreviated as "GPG," is used for the automatic generation of global policies for subClusters. +GlobalPolicyGenerator, abbreviated as "GPG", is used for the automatic generation of global policies for subClusters. These are extra configurations that should appear in the **conf/yarn-site.xml** for GPG. We allow only one GPG. From 9c4a501b35362f0d76e1bcd9e36e28ee67654fc1 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Oct 2023 21:21:59 +0800 Subject: [PATCH 5/9] YARN-11593. Fix CheckStyle. --- .../hadoop/yarn/client/cli/RouterCLI.java | 10 +- .../src/site/markdown/Federation.md | 132 +++++++++++++++++- 2 files changed, 134 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 5cebbbbe20fc0..6249c5d045110 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -117,13 +117,13 @@ public class RouterCLI extends Configured implements Tool { "Headroomalpha can be set to the default value of 1.0.", "At this point we can use the following command:")); - protected final static String PEXAMPLE_3 = "yarn routeradmin -policy -bs --format xml -f queue.xml"; - protected final static String PEXAMPLE_4 = "yarn routeradmin -policy --batch-save --format xml -f queue.xml"; + protected final static String PEXAMPLE_3 = "yarn routeradmin -policy -bs --format xml -f federation-weights.xml"; + protected final static String PEXAMPLE_4 = "yarn routeradmin -policy --batch-save --format xml -f federation-weights.xml"; protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_2 = new RouterExampleUsageInfos( Arrays.asList(PEXAMPLE_3, PEXAMPLE_4), Arrays.asList("We offer the capability to batch input queue policies. ", "Currently, this is supported exclusively through XML. " + - "(We can find federationWeights.xml in the Federation.md on the website.) ", + "(We can find federation-weights.xml in the Federation.md on the website.) ", "We can use the batch-save command to input proportion information. ", "At this point we can use the following command:")); @@ -139,9 +139,9 @@ public class RouterCLI extends Configured implements Tool { new RouterUsageInfos(Arrays.asList(POLICY_SAVE_USAGE, POLICY_BATCH_SAVE_USAGE, POLICY_LIST_USAGE), Arrays.asList("We provide a set of commands for Policy Include list policies, save policies, batch save policies."), ImmutableMap.builder() - .put("-s|--save () ", PEXAMPLEUSAGEINFOS_1) + .put("-s|--save ()", PEXAMPLEUSAGEINFOS_1) .put("-bs|--batch-save (--format ) (-f|--input-file )", PEXAMPLEUSAGEINFOS_2) - .put("-l|--list [--pageSize][--currentPage][--queue][--queues]", PEXAMPLEUSAGEINFOS_3) + .put("-l|--list ([--pageSize][--currentPage][--queue][--queues])", PEXAMPLEUSAGEINFOS_3) .build()); // Common Constant diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 1d1f435e74c5c..8c0c25637202f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -477,9 +477,9 @@ Uasge: Options: -| Property | Example | -|:------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -sc, --subClusterId | '-sc' option allows you to specify the sub-cluster to operate on, while the '--subClusterId' option is the long format of -sc and serves the same purpose." | +| Property | Description | +|:--------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-sc, --subClusterId [subCluster Id]` | `'-sc' option allows you to specify the sub-cluster to operate on, while the '--subClusterId' option is the long format of -sc and serves the same purpose.` | Examples: @@ -490,6 +490,132 @@ If we want to deregisterSubCluster `SC-1` - policy +We provide a set of commands for Policy Include list policies, save policies, batch save policies. + +Uasge: + +`yarn routeradmin -policy -s|--save (queue;router weight;amrm weight;headroomalpha)` + +`yarn routeradmin -policy -bs|--batch-save (--format xml) (-f|--input-file fileName)` + +`yarn routeradmin -policy -l|--list ([--pageSize][--currentPage][--queue][--queues])` + +- -s|--save () + +This command is used to save the policy information of the queue, including queue and weight information. + +How to configure `queue;router weight;amrm weight;headroomalpha` + +the sum of weights for all sub-clusters in routerWeight/amrmWeight should be 1. + +| Property | Description | +|:----------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `queue` | `Scheduled queue` | +| `router weight` | `Weight for routing applications to different subclusters.` | +| `amrm weight` | `Weight for resource request from ApplicationMaster (AM) to different subclusters' Resource Manager (RM).` | +| `headroomalpha` | `Used by policies that balance weight-based and load-based considerations in their decisions. It is recommended to use 1.0 because the load-base function is not yet complete.` | + +Example: + +We have two subClusters, `SC-1` and `SC-2`. For queue `root.a`, we want the ratio to be set to `SC-1:SC-2=0.7:0.3` when routing., +Additionally, when AM applies containers, we want the allocation ratio to be `SC-1:SC-2=0.6:0.4.`, headroomalpha can be set to the default value of `1.0`. + +yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 + +yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 + +- -bs|--batch-save (--format xml) (-f|--input-file fileName) + +This command can batch load weight information for queues based on the provided `federation-weights.xml` file. + +| Property | Description | +|:--------------------------|:----------------------------------------------------------------------------------------------| +| `--format [xml]` | `Configuration file format, we currently only support xml format` | +| `-f, --input-file [path]` | `The path to the configuration file. Please use the absolute path of the configuration file.` | + +How to configure `federation-weights.xml` + ```xml + + + + root.a + + + SC-1 + 0.7 + + + SC-2 + 0.3 + + + + + SC-1 + 0.6 + + + SC-2 + 0.4 + + + 1.0 + + + + + root.b + + + SC-1 + 0.8 + + + SC-2 + 0.2 + + + + + SC-1 + 0.6 + + + SC-2 + 0.4 + + + 1.0 + + + + ``` + +Example: + +The file name can be any file name, but it is recommended to use `federation-weights.xml` + +yarn routeradmin -policy -bs --format xml -f /path/federation-weights.xml + +yarn routeradmin -policy --batch-save --format xml -f /path/federation-weights.xml + +- -l|--list (--pageSize --currentPage --queue --queues) + +This command is used to display the configured queue weight information. + +| Property | Description | +|:----------------|:-------------------------------------------------------------| +| `--pageSize` | `The number of policies displayed per page.` | +| `--currentPage` | `This parameter represents the page number to be displayed.` | +| `--queue` | `the queue we need to filter. example: root.a` | +| `--queues` | `list of queues to filter. example: root.a,root.b,root.c` | + +Example: + +yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a + +yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b + ### ON GPG: GlobalPolicyGenerator, abbreviated as "GPG", is used for the automatic generation of global policies for subClusters. From 13ad7209620a873af80c4a9670269490d19cd7ff Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 20 Oct 2023 00:59:47 +0800 Subject: [PATCH 6/9] YARN-11593. Fix CheckStyle. --- .../hadoop/yarn/client/cli/RouterCLI.java | 267 +++++++++++------- .../src/site/markdown/Federation.md | 2 +- 2 files changed, 159 insertions(+), 110 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 6249c5d045110..065a196dc15b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -69,6 +69,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.LinkedHashMap; import java.util.stream.Collectors; import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid; @@ -76,74 +77,8 @@ public class RouterCLI extends Configured implements Tool { - private static final Logger LOG = LoggerFactory.getLogger(RouterCLI.class); - protected final static UsageInfo SUBCLUSTER_ID = new UsageInfo("<-sc|--subClusterId>", - "'-sc' option allows you to specify the sub-cluster to operate on, " + - "while the '--subClusterId' option is the long format of -sc and serves the same purpose."); - protected final static String DSEXAMPLE_1 = "yarn routeradmin -deregisterSubCluster -sc SC-1"; - protected final static String DSEXAMPLE_2 = "yarn routeradmin -deregisterSubCluster --subClusterId SC-1"; - - protected final static RouterExampleUsageInfos DSEXAMPLEUSAGEINFOS_1 = new RouterExampleUsageInfos( - Arrays.asList(DSEXAMPLE_1, DSEXAMPLE_2), - Arrays.asList("At this point we can use the following command:")); - protected final static RouterUsageInfos DEREGISTER_SUBCLUSTER = - new RouterUsageInfos(Arrays.asList(SUBCLUSTER_ID), - Arrays.asList("deregister subCluster, " + - "If the interval between the heartbeat time of the subCluster" + - "and the current time exceeds the timeout period, set the state of the subCluster to SC_LOST."), - ImmutableMap.builder() - .put("<--deregisterSubCluster>", DSEXAMPLEUSAGEINFOS_1) - .build()); - - protected final static UsageInfo POLICY_SAVE_USAGE = new UsageInfo( - "-s|--save ()", - "We will save the policy information of the queue, including queue and weight information."); - protected final static UsageInfo POLICY_BATCH_SAVE_USAGE = new UsageInfo( - "-bs|--batch-save (--format ) (-f|--input-file )", - "We will save queue policies in bulk, where users can provide XML files containing the policies."); - protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo( - "-l|--list [--pageSize][--currentPage][--queue][--queues]", - "We can display the configured queue policies."); - - protected final static String PEXAMPLE_1 = "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; - protected final static String PEXAMPLE_2 = "yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; - protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_1 = new RouterExampleUsageInfos( - Arrays.asList(PEXAMPLE_1, PEXAMPLE_2), - Arrays.asList("We have two subClusters, SC-1 and SC-2. " + - "For queue root.a, we want the ratio to be set to SC-1:SC-2=0.7:0.3 when routing.", - "Additionally, when AM applies containers, we want the allocation ratio to be SC-1:SC-2=0.6:0.4.", - "Headroomalpha can be set to the default value of 1.0.", - "At this point we can use the following command:")); - - protected final static String PEXAMPLE_3 = "yarn routeradmin -policy -bs --format xml -f federation-weights.xml"; - protected final static String PEXAMPLE_4 = "yarn routeradmin -policy --batch-save --format xml -f federation-weights.xml"; - protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_2 = new RouterExampleUsageInfos( - Arrays.asList(PEXAMPLE_3, PEXAMPLE_4), - Arrays.asList("We offer the capability to batch input queue policies. ", - "Currently, this is supported exclusively through XML. " + - "(We can find federation-weights.xml in the Federation.md on the website.) ", - "We can use the batch-save command to input proportion information. ", - "At this point we can use the following command:")); - - protected final static String PEXAMPLE_5 = "yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a"; - protected final static String PEXAMPLE_6 = "yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b"; - protected final static RouterExampleUsageInfos PEXAMPLEUSAGEINFOS_3 = new RouterExampleUsageInfos( - Arrays.asList(PEXAMPLE_5, PEXAMPLE_6), - Arrays.asList("We can print the Configured Queue Policies on the command line. ", - "We can specify a queue through --queue, and we can specify a queue list through --queues.", - "At this point we can use the following command:")); - - protected final static RouterUsageInfos POLICY = - new RouterUsageInfos(Arrays.asList(POLICY_SAVE_USAGE, POLICY_BATCH_SAVE_USAGE, POLICY_LIST_USAGE), - Arrays.asList("We provide a set of commands for Policy Include list policies, save policies, batch save policies."), - ImmutableMap.builder() - .put("-s|--save ()", PEXAMPLEUSAGEINFOS_1) - .put("-bs|--batch-save (--format ) (-f|--input-file )", PEXAMPLEUSAGEINFOS_2) - .put("-l|--list ([--pageSize][--currentPage][--queue][--queues])", PEXAMPLEUSAGEINFOS_3) - .build()); - // Common Constant private static final String SEMICOLON = ";"; @@ -151,6 +86,7 @@ public class RouterCLI extends Configured implements Tool { private static final String CMD_EMPTY = ""; private static final int EXIT_SUCCESS = 0; private static final int EXIT_ERROR = -1; + private static final String CMD_HELP = "-help"; // Command1: deregisterSubCluster private static final String DEREGISTER_SUBCLUSTER_TITLE = @@ -162,15 +98,42 @@ public class RouterCLI extends Configured implements Tool { private static final String OPTION_SC = "sc"; private static final String OPTION_SUBCLUSTERID = "subClusterId"; private static final String CMD_DEREGISTERSUBCLUSTER = "-deregisterSubCluster"; - private static final String CMD_HELP = "-help"; + + // DeregisterSubCluster Command Parameters + protected final static UsageInfo SUBCLUSTER_ID = new UsageInfo("<-sc|--subClusterId>", + "'-sc' option allows you to specify the sub-cluster to operate on, " + + "while the '--subClusterId' option is the long format of -sc and serves the same purpose."); + + // DeregisterSubCluster Command Examples + protected final static String DEREGISTER_SUBCLUSTER_EXAMPLE_1 = + "yarn routeradmin -deregisterSubCluster -sc SC-1"; + protected final static String DEREGISTER_SUBCLUSTER_EXAMPLE_2 = + "yarn routeradmin -deregisterSubCluster --subClusterId SC-1"; + + // DeregisterSubCluster Command Help Information + protected final static String DEREGISTER_SUBCLUSTER_HELP_INFO = + "deregister subCluster, If the interval between the heartbeat time of the subCluster and" + + "the current time exceeds the timeout period, set the state of the subCluster to SC_LOST."; + + protected final static RouterCmdUsageInfos DEREGISTER_SUBCLUSTER_USAGEINFOS = + new RouterCmdUsageInfos() + .addUsageInfo(SUBCLUSTER_ID) + .addHelpInfo(DEREGISTER_SUBCLUSTER_HELP_INFO) + .addExampleDescs(CMD_DEREGISTERSUBCLUSTER, "If we want to deregisterSubCluster SC-1") + .addExample(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_EXAMPLE_1) + .addExample(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_EXAMPLE_2); // Command2: policy + + private static final String CMD_POLICY = "-policy"; + // save policy private static final String OPTION_S = "s"; - private static final String OPTION_BATCH_S = "bs"; private static final String OPTION_SAVE = "save"; + private static final String OPTION_BATCH_S = "bs"; private static final String OPTION_BATCH_SAVE = "batch-save"; private static final String OPTION_FORMAT = "format"; + private static final String FORMAT_XML = "xml"; private static final String OPTION_FILE = "f"; private static final String OPTION_INPUT_FILE = "input-file"; private static final String OPTION_L = "l"; @@ -180,9 +143,6 @@ public class RouterCLI extends Configured implements Tool { private static final String OPTION_QUEUE = "queue"; private static final String OPTION_QUEUES = "queues"; - private static final String CMD_POLICY = "-policy"; - private static final String FORMAT_XML = "xml"; - private static final String XML_TAG_SUBCLUSTERIDINFO = "subClusterIdInfo"; private static final String XML_TAG_AMRMPOLICYWEIGHTS = "amrmPolicyWeights"; private static final String XML_TAG_ROUTERPOLICYWEIGHTS = "routerPolicyWeights"; @@ -193,17 +153,84 @@ public class RouterCLI extends Configured implements Tool { private static final String LIST_POLICIES_TITLE = "Yarn Federation Queue Policies"; + // Columns information private static final List LIST_POLICIES_HEADER = Arrays.asList( "Queue Name", "AMRM Weight", "Router Weight"); - protected final static Map ADMIN_USAGE = - ImmutableMap.builder() - // Command1: deregisterSubCluster - .put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER) - // Command2: policy - .put(CMD_POLICY, POLICY) - .build(); + // Policy Commands + protected final static UsageInfo POLICY_SAVE_USAGE = new UsageInfo( + "-s|--save ()", + "This command is used to save the policy information of the queue, " + + "including queue and weight information."); + + protected final static String POLICY_SAVE_USAGE_EXAMPLE_DESC = + "We have two sub-clusters, SC-1 and SC-2. \\" + + "We want to configure a weight policy for the 'root.a' queue. \\" + + "The Router Weight is set to SC-1 with a weight of 0.7 and SC-2 with a weight of 0.3. \\" + + "The AMRM Weight is set SC-1 to 0.6 and SC-2 to 0.4. \\" + + "We are using the default value of 0.1 for headroomalpha."; + + protected final static String POLICY_SAVE_USAGE_EXAMPLE_1 = + "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + protected final static String POLICY_SAVE_USAGE_EXAMPLE_2 = + "yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + + protected final static UsageInfo POLICY_BATCH_SAVE_USAGE = new UsageInfo( + "-bs|--batch-save (--format ) (-f|--input-file )", + "This command can batch load weight information for queues " + + "based on the provided `federation-weights.xml` file."); + + protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_DESC = + "We have two sub-clusters, SC-1 and SC-2. \\" + + "We would like to configure weights for 'root.a' and 'root.b' queues. \\" + + "We can set the weights for 'root.a' and 'root.b' in the 'federation-weights.xml' file. \\" + + "and then use the batch-save command to save the configurations in bulk."; + + protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_1 = + "yarn routeradmin -policy -bs --format xml -f federation-weights.xml"; + protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_2 = + "yarn routeradmin -policy --batch-save --format xml -f federation-weights.xml"; + + protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo( + "-l|--list [--pageSize][--currentPage][--queue][--queues]", + "This command is used to display the configured queue weight information."); + + protected final static String POLICY_LIST_USAGE_DESC = + "We can display the list of already configured queue weight information. \\" + + "We can use the --queue option to query the weight information for a specific queue \\" + + " or use the --queues option to query the weight information for multiple queues. \\"; + + protected final static String POLICY_LIST_USAGE_EXAMPLE_1 = + "yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a"; + + protected final static String POLICY_LIST_USAGE_EXAMPLE_2 = + "yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b"; + + protected final static RouterCmdUsageInfos POLICY_USAGEINFOS = new RouterCmdUsageInfos() + // Policy Save + .addUsageInfo(POLICY_SAVE_USAGE) + .addExampleDescs(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_DESC) + .addExample(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_1) + .addExample(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_2) + // Policy Batch Save + .addUsageInfo(POLICY_BATCH_SAVE_USAGE) + .addExampleDescs(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_DESC) + .addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_1) + .addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_2) + // Policy List Save + .addUsageInfo(POLICY_LIST_USAGE) + .addExampleDescs(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_DESC) + .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_1) + .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_2); + + protected final static Map ADMIN_USAGE = + ImmutableMap.builder() + // Command1: deregisterSubCluster + .put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_USAGEINFOS) + // Command2: policy + .put(CMD_POLICY, POLICY_USAGEINFOS) + .build(); public RouterCLI() { super(); @@ -214,7 +241,7 @@ public RouterCLI(Configuration conf) { } private static void buildHelpMsg(String cmd, StringBuilder builder) { - RouterUsageInfos routerUsageInfo = ADMIN_USAGE.get(cmd); + RouterCmdUsageInfos routerUsageInfo = ADMIN_USAGE.get(cmd); if (routerUsageInfo == null) { return; @@ -241,27 +268,34 @@ private static void buildHelpMsg(String cmd, StringBuilder builder) { if (MapUtils.isNotEmpty(routerUsageInfo.examples)) { builder.append("\t Examples: \n"); int count = 1; - for (Map.Entry example : - routerUsageInfo.examples.entrySet()) { - String key = example.getKey(); - builder.append("\t\t").append("Cmd:").append(count).append(". ").append(key).append(": \n\n"); - RouterExampleUsageInfos values = example.getValue(); - if (values == null) { - return; - } + for (Map.Entry> example : routerUsageInfo.examples.entrySet()) { + + String keyCmd = example.getKey(); + builder.append("\t\t") + .append("Cmd:").append(count) + .append(". ").append(keyCmd) + .append(": \n\n"); + // Print Command Description - if (CollectionUtils.isNotEmpty(values.desc)) { + List exampleDescs = routerUsageInfo.exampleDescs.get(keyCmd); + if (CollectionUtils.isNotEmpty(exampleDescs)) { builder.append("\t\t").append("Cmd Requirement Description:\n"); - for (String value : values.desc) { - builder.append("\t\t").append(value).append("\n"); + for (String value : exampleDescs) { + String[] valueDescs = StringUtils.split(value, "\\"); + for (String valueDesc : valueDescs) { + builder.append("\t\t").append(valueDesc).append("\n"); + } } } + builder.append("\n"); + // Print Command example - if (CollectionUtils.isNotEmpty(values.examples)) { + List valueExamples = example.getValue(); + if (CollectionUtils.isNotEmpty(valueExamples)) { builder.append("\t\t").append("Cmd Examples:\n"); - for (String value : values.examples) { - builder.append("\t\t").append(value).append("\n"); + for (String valueExample : valueExamples) { + builder.append("\t\t").append(valueExample).append("\n"); } } builder.append("\n"); @@ -810,26 +844,41 @@ public int run(String[] args) throws Exception { return EXIT_SUCCESS; } - private static class RouterExampleUsageInfos { - public List examples; - public List desc; + private static class RouterCmdUsageInfos { + public List usageInfos; + public List helpInfos; + public Map> examples; + public Map> exampleDescs; + + public RouterCmdUsageInfos() { + this.usageInfos = new ArrayList<>(); + this.helpInfos = new ArrayList<>(); + this.examples = new LinkedHashMap<>(); + this.exampleDescs = new LinkedHashMap<>(); + } - public RouterExampleUsageInfos(List examples, List desc) { - this.examples = examples; - this.desc = desc; + public RouterCmdUsageInfos addUsageInfo(UsageInfo usageInfo) { + this.usageInfos.add(usageInfo); + return this; } - } - private static class RouterUsageInfos { - public List usageInfos; - public List helpInfos; - public Map examples; + public RouterCmdUsageInfos addHelpInfo(String helpInfo) { + this.helpInfos.add(helpInfo); + return this; + } + + private RouterCmdUsageInfos addExample(String cmd, String example) { + List exampleList = this.examples.getOrDefault(cmd, new ArrayList<>()); + exampleList.add(example); + this.examples.put(cmd, exampleList); + return this; + } - public RouterUsageInfos(List usageInfos, List helpInfos, - Map examples) { - this.usageInfos = usageInfos; - this.helpInfos = helpInfos; - this.examples = examples; + private RouterCmdUsageInfos addExampleDescs(String cmd, String exampleDesc) { + List exampleDescList = this.exampleDescs.getOrDefault(cmd, new ArrayList<>()); + exampleDescList.add(exampleDesc); + this.exampleDescs.put(cmd, exampleDescList); + return this; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 8c0c25637202f..fb6305a0af1f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -504,7 +504,7 @@ Uasge: This command is used to save the policy information of the queue, including queue and weight information. -How to configure `queue;router weight;amrm weight;headroomalpha` +How to configure `queue;router weight;amrm weight;headroomalpha` the sum of weights for all sub-clusters in routerWeight/amrmWeight should be 1. From d7edf773247609260df5e2aa54381e30e7ba0fd7 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 20 Oct 2023 02:57:43 +0800 Subject: [PATCH 7/9] YARN-11593. Fix CheckStyle. --- .../hadoop/yarn/client/cli/RouterCLI.java | 14 +++++++++- .../hadoop/yarn/client/cli/TestRouterCLI.java | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 065a196dc15b5..06beb0874a6e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.ha.HAAdmin.UsageInfo; @@ -130,12 +131,14 @@ public class RouterCLI extends Configured implements Tool { // save policy private static final String OPTION_S = "s"; private static final String OPTION_SAVE = "save"; + // batch save policy private static final String OPTION_BATCH_S = "bs"; private static final String OPTION_BATCH_SAVE = "batch-save"; private static final String OPTION_FORMAT = "format"; private static final String FORMAT_XML = "xml"; private static final String OPTION_FILE = "f"; private static final String OPTION_INPUT_FILE = "input-file"; + // list policy private static final String OPTION_L = "l"; private static final String OPTION_LIST = "list"; private static final String OPTION_PAGE_SIZE = "pageSize"; @@ -844,7 +847,11 @@ public int run(String[] args) throws Exception { return EXIT_SUCCESS; } - private static class RouterCmdUsageInfos { + public static UsageInfo getPolicyBatchSaveUsage() { + return POLICY_BATCH_SAVE_USAGE; + } + + static class RouterCmdUsageInfos { public List usageInfos; public List helpInfos; public Map> examples; @@ -886,4 +893,9 @@ public static void main(String[] args) throws Exception { int result = ToolRunner.run(new RouterCLI(), args); System.exit(result); } + + @VisibleForTesting + public Map getAdminUsage(){ + return ADMIN_USAGE; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index 6ed83826dfa58..4affeae185a73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -266,4 +267,29 @@ public void testListPolicies() throws Exception { String[] args = {"-policy", "-l", "--queue", "root.a"}; assertEquals(0, rmAdminCLI.run(args)); } + + @Test + public void testBuildHelpMsg() throws Exception { + Map adminUsage = rmAdminCLI.getAdminUsage(); + assertEquals(2, adminUsage.size()); + + RouterCLI.RouterCmdUsageInfos deregisterSubClusterUsageInfos = + adminUsage.get("-deregisterSubCluster"); + assertNotNull(deregisterSubClusterUsageInfos); + Map> dsExamplesMap = deregisterSubClusterUsageInfos.examples; + assertNotNull(dsExamplesMap); + assertEquals(1, dsExamplesMap.size()); + List dsExamples = dsExamplesMap.get("-deregisterSubCluster"); + assertNotNull(dsExamples); + assertEquals(2, dsExamples.size()); + + RouterCLI.RouterCmdUsageInfos policyUsageInfos = adminUsage.get("-policy"); + assertNotNull(policyUsageInfos); + Map> policyExamplesMap = policyUsageInfos.examples; + assertNotNull(policyExamplesMap); + assertEquals(3, policyExamplesMap.size()); + policyExamplesMap.forEach((cmd, cmdExamples) -> { + assertEquals(2, cmdExamples.size()); + }); + } } From 8a287a3696e670d516bfe159bb37c04b0f41c099 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 20 Oct 2023 16:52:46 +0800 Subject: [PATCH 8/9] YARN-11593. Fix CheckStyle. --- .../apache/hadoop/yarn/client/cli/RouterCLI.java | 14 +++++++------- .../src/site/markdown/Federation.md | 9 +++++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 06beb0874a6e5..f7cc104e24f64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -199,7 +199,7 @@ public class RouterCLI extends Configured implements Tool { "-l|--list [--pageSize][--currentPage][--queue][--queues]", "This command is used to display the configured queue weight information."); - protected final static String POLICY_LIST_USAGE_DESC = + protected final static String POLICY_LIST_USAGE_EXAMPLE_DESC = "We can display the list of already configured queue weight information. \\" + "We can use the --queue option to query the weight information for a specific queue \\" + " or use the --queues option to query the weight information for multiple queues. \\"; @@ -223,7 +223,7 @@ public class RouterCLI extends Configured implements Tool { .addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_2) // Policy List Save .addUsageInfo(POLICY_LIST_USAGE) - .addExampleDescs(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_DESC) + .addExampleDescs(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_DESC) .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_1) .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_2); @@ -852,12 +852,12 @@ public static UsageInfo getPolicyBatchSaveUsage() { } static class RouterCmdUsageInfos { - public List usageInfos; - public List helpInfos; - public Map> examples; - public Map> exampleDescs; + protected List usageInfos; + protected List helpInfos; + protected Map> examples; + protected Map> exampleDescs; - public RouterCmdUsageInfos() { + RouterCmdUsageInfos() { this.usageInfos = new ArrayList<>(); this.helpInfos = new ArrayList<>(); this.examples = new LinkedHashMap<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index fb6305a0af1f3..5d5dc786e13b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -517,8 +517,8 @@ the sum of weights for all sub-clusters in routerWeight/amrmWeight should be 1. Example: -We have two subClusters, `SC-1` and `SC-2`. For queue `root.a`, we want the ratio to be set to `SC-1:SC-2=0.7:0.3` when routing., -Additionally, when AM applies containers, we want the allocation ratio to be `SC-1:SC-2=0.6:0.4.`, headroomalpha can be set to the default value of `1.0`. +We have two sub-clusters, `SC-1` and `SC-2`. We want to configure a weight policy for the `root.a` queue. The Router Weight is set to `SC-1` with a weight of `0.7` and `SC-2` with a weight of `0.3`. +The AMRM Weight is set `SC-1` to `0.6` and `SC-2` to `0.4`. We are using the default value of `0.1` for `headroomalpha`. yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 @@ -593,6 +593,9 @@ How to configure `federation-weights.xml` Example: +We have two sub-clusters, `SC-1` and `SC-2`. We would like to configure weights for `root.a` and `root.b` queues. We can set the weights for `root.a` and `root.b` in the `federation-weights.xml` file. +and then use the batch-save command to save the configurations in bulk. + The file name can be any file name, but it is recommended to use `federation-weights.xml` yarn routeradmin -policy -bs --format xml -f /path/federation-weights.xml @@ -612,6 +615,8 @@ This command is used to display the configured queue weight information. Example: +We can display the list of already configured queue weight information. We can use the `--queue` option to query the weight information for a specific queue or use the `--queues` option to query the weight information for multiple queues. + yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b From 4ec36f8888d4f10d3b30ecac877cc25cae249f7c Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 20 Oct 2023 22:01:55 +0800 Subject: [PATCH 9/9] YARN-11593. Fix CheckStyle. --- .../org/apache/hadoop/yarn/client/cli/RouterCLI.java | 10 +++++++--- .../apache/hadoop/yarn/client/cli/TestRouterCLI.java | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index f7cc104e24f64..2da584f9d61bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -852,9 +852,9 @@ public static UsageInfo getPolicyBatchSaveUsage() { } static class RouterCmdUsageInfos { - protected List usageInfos; - protected List helpInfos; - protected Map> examples; + private List usageInfos; + private List helpInfos; + private Map> examples; protected Map> exampleDescs; RouterCmdUsageInfos() { @@ -887,6 +887,10 @@ private RouterCmdUsageInfos addExampleDescs(String cmd, String exampleDesc) { this.exampleDescs.put(cmd, exampleDescList); return this; } + + public Map> getExamples() { + return examples; + } } public static void main(String[] args) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index 4affeae185a73..a86878dac3f81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -276,7 +276,7 @@ public void testBuildHelpMsg() throws Exception { RouterCLI.RouterCmdUsageInfos deregisterSubClusterUsageInfos = adminUsage.get("-deregisterSubCluster"); assertNotNull(deregisterSubClusterUsageInfos); - Map> dsExamplesMap = deregisterSubClusterUsageInfos.examples; + Map> dsExamplesMap = deregisterSubClusterUsageInfos.getExamples(); assertNotNull(dsExamplesMap); assertEquals(1, dsExamplesMap.size()); List dsExamples = dsExamplesMap.get("-deregisterSubCluster"); @@ -285,7 +285,7 @@ public void testBuildHelpMsg() throws Exception { RouterCLI.RouterCmdUsageInfos policyUsageInfos = adminUsage.get("-policy"); assertNotNull(policyUsageInfos); - Map> policyExamplesMap = policyUsageInfos.examples; + Map> policyExamplesMap = policyUsageInfos.getExamples(); assertNotNull(policyExamplesMap); assertEquals(3, policyExamplesMap.size()); policyExamplesMap.forEach((cmd, cmdExamples) -> {