From c7f90d87a076fe896631122ba7faadd30c2f58a9 Mon Sep 17 00:00:00 2001 From: Prathyusha Garre Date: Thu, 22 Jun 2023 21:53:21 +0530 Subject: [PATCH 1/3] HBASE-27938 - load test classes by name and support command specific properties --- .../hadoop/hbase/PerformanceEvaluation.java | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 83d6d2f7cba9..b68798e2ad7c 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -36,6 +36,7 @@ import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Properties; import java.util.Queue; import java.util.Random; import java.util.TreeMap; @@ -357,13 +358,11 @@ static boolean checkTable(Admin admin, TestOptions opts) throws IOException { // recreate the table when user has requested presplit or when existing // {RegionSplitPolicy,replica count} does not match requested, or when the // number of column families does not match requested. - if ( - (exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions) - || (!isReadCmd && desc != null - && !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy)) + if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions + && opts.presplitRegions != admin.getRegions(tableName).size()) + || (!isReadCmd && desc != null && !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy)) || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas) - || (desc != null && desc.getColumnFamilyCount() != opts.families) - ) { + || (desc != null && desc.getColumnFamilyCount() != opts.families)) { needsDelete = true; // wait, why did it delete my table?!? LOG.debug(MoreObjects.toStringHelper("needsDelete").add("needsDelete", needsDelete) @@ -719,6 +718,7 @@ static class TestOptions { boolean cacheBlocks = true; Scan.ReadType scanReadType = Scan.ReadType.DEFAULT; long bufferSize = 2l * 1024l * 1024l; + Properties commandProperties; public TestOptions() { } @@ -775,8 +775,13 @@ public TestOptions(TestOptions that) { this.cacheBlocks = that.cacheBlocks; this.scanReadType = that.scanReadType; this.bufferSize = that.bufferSize; + this.commandProperties = that.commandProperties; } - + + public Properties getCommandProperties() { + return commandProperties; + } + public int getCaching() { return this.caching; } @@ -1140,10 +1145,10 @@ private static long nextRandomSeed() { protected final Configuration conf; protected final TestOptions opts; - private final Status status; + protected final Status status; private String testName; - private Histogram latencyHistogram; + protected Histogram latencyHistogram; private Histogram replicaLatencyHistogram; private Histogram valueSizeHistogram; private Histogram rpcCallsHistogram; @@ -2300,7 +2305,7 @@ protected byte[] generateRow(final int i) { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + protected boolean testRow(final int i, final long startTime) throws IOException { byte[] row = generateRow(i); Put put = new Put(row); for (int family = 0; family < opts.families; family++) { @@ -2968,6 +2973,19 @@ static TestOptions parseOpts(Queue args) { continue; } + final String commandPropertiesFile = "--commandPropertiesFile="; + if (cmd.startsWith(commandPropertiesFile)) { + String fileName = String.valueOf(cmd.substring(commandPropertiesFile.length())); + Properties properties = new Properties(); + try { + properties.load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName)); + opts.commandProperties = properties; + } catch (IOException e) { + LOG.error("Failed to load metricIds from properties file", e); + } + continue; + } + validateParsedOpts(opts); if (isCommandClass(cmd)) { @@ -3081,9 +3099,21 @@ public int run(String[] args) throws Exception { } private static boolean isCommandClass(String cmd) { - return COMMANDS.containsKey(cmd); + return !COMMANDS.containsKey(cmd) ? isCustomTestClass(cmd) : true; } + private static boolean isCustomTestClass(String cmd) { + Class cmdClass; + try { + cmdClass = (Class) PerformanceEvaluation.class.getClassLoader().loadClass(cmd); + addCommandDescriptor(cmdClass, cmd, "custom command"); + return true; + } catch (Throwable th) { + LOG.info("No class found for command: " + cmd, th); + return false; + } + } + private static Class determineCommandClass(String cmd) { CmdDescriptor descriptor = COMMANDS.get(cmd); return descriptor != null ? descriptor.getCmdClass() : null; From 2ccd5e8e72835eff86bdcc8d50ad735c21d3618f Mon Sep 17 00:00:00 2001 From: gvprathyusha6 <70918688+gvprathyusha6@users.noreply.github.com> Date: Tue, 27 Jun 2023 01:07:11 +0530 Subject: [PATCH 2/3] HBASE-27938 - update printUsage to include custom command --- .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index b68798e2ad7c..ed5f707a4a73 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -2678,6 +2678,7 @@ protected static void printUsage(final String shortName, final String message) { for (CmdDescriptor command : COMMANDS.values()) { System.err.println(String.format(" %-20s %s", command.getName(), command.getDescription())); } + System.err.println(String.format(" %-20s %s", "? extends Test", "Run custom implementation of provided Test present in classpath")); System.err.println(); System.err.println("Args:"); System.err.println(" nclients Integer. Required. Total number of clients " From 8b36b15dbe9b751cb7c2df60985a24e0c4bb61d0 Mon Sep 17 00:00:00 2001 From: Prathyusha Garre Date: Tue, 27 Jun 2023 15:39:45 +0530 Subject: [PATCH 3/3] HBASE-27938 - fix checkstyle errors --- .../hadoop/hbase/PerformanceEvaluation.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index ed5f707a4a73..891d5d7a098b 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -358,11 +358,14 @@ static boolean checkTable(Admin admin, TestOptions opts) throws IOException { // recreate the table when user has requested presplit or when existing // {RegionSplitPolicy,replica count} does not match requested, or when the // number of column families does not match requested. - if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions + if ( + (exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions && opts.presplitRegions != admin.getRegions(tableName).size()) - || (!isReadCmd && desc != null && !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy)) + || (!isReadCmd && desc != null + && !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy)) || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas) - || (desc != null && desc.getColumnFamilyCount() != opts.families)) { + || (desc != null && desc.getColumnFamilyCount() != opts.families) + ) { needsDelete = true; // wait, why did it delete my table?!? LOG.debug(MoreObjects.toStringHelper("needsDelete").add("needsDelete", needsDelete) @@ -777,11 +780,11 @@ public TestOptions(TestOptions that) { this.bufferSize = that.bufferSize; this.commandProperties = that.commandProperties; } - + public Properties getCommandProperties() { return commandProperties; } - + public int getCaching() { return this.caching; } @@ -2678,7 +2681,8 @@ protected static void printUsage(final String shortName, final String message) { for (CmdDescriptor command : COMMANDS.values()) { System.err.println(String.format(" %-20s %s", command.getName(), command.getDescription())); } - System.err.println(String.format(" %-20s %s", "? extends Test", "Run custom implementation of provided Test present in classpath")); + System.err.println(String.format(" %-20s %s", "? extends Test", + "Run custom implementation of provided Test present in classpath")); System.err.println(); System.err.println("Args:"); System.err.println(" nclients Integer. Required. Total number of clients " @@ -2979,14 +2983,15 @@ static TestOptions parseOpts(Queue args) { String fileName = String.valueOf(cmd.substring(commandPropertiesFile.length())); Properties properties = new Properties(); try { - properties.load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName)); + properties + .load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName)); opts.commandProperties = properties; } catch (IOException e) { LOG.error("Failed to load metricIds from properties file", e); } continue; } - + validateParsedOpts(opts); if (isCommandClass(cmd)) { @@ -3106,7 +3111,8 @@ private static boolean isCommandClass(String cmd) { private static boolean isCustomTestClass(String cmd) { Class cmdClass; try { - cmdClass = (Class) PerformanceEvaluation.class.getClassLoader().loadClass(cmd); + cmdClass = + (Class) PerformanceEvaluation.class.getClassLoader().loadClass(cmd); addCommandDescriptor(cmdClass, cmd, "custom command"); return true; } catch (Throwable th) { @@ -3114,7 +3120,7 @@ private static boolean isCustomTestClass(String cmd) { return false; } } - + private static Class determineCommandClass(String cmd) { CmdDescriptor descriptor = COMMANDS.get(cmd); return descriptor != null ? descriptor.getCmdClass() : null;