diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index aa7b1302231f..83d339fefc74 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -441,9 +441,9 @@ protected static byte[][] getSplits(TestOptions opts) { int numSplitPoints = opts.presplitRegions - 1; byte[][] splits = new byte[numSplitPoints][]; - int jump = opts.totalRows / opts.presplitRegions; + long jump = opts.totalRows / opts.presplitRegions; for (int i = 0; i < numSplitPoints; i++) { - int rowkey = jump * (1 + i); + long rowkey = jump * (1 + i); splits[i] = format(rowkey); } return splits; @@ -620,7 +620,7 @@ static Path writeInputFile(final Configuration c, final TestOptions opts, final // Make input random. Map m = new TreeMap<>(); Hash h = MurmurHash.getInstance(); - int perClientRows = (opts.totalRows / opts.numClientThreads); + long perClientRows = (opts.totalRows / opts.numClientThreads); try { for (int j = 0; j < opts.numClientThreads; j++) { TestOptions next = new TestOptions(opts); @@ -679,11 +679,11 @@ static class TestOptions { String cmdName = null; boolean nomapred = false; boolean filterAll = false; - int startRow = 0; + long startRow = 0; float size = 1.0f; - int perClientRunRows = DEFAULT_ROWS_PER_GB; + long perClientRunRows = DEFAULT_ROWS_PER_GB; int numClientThreads = 1; - int totalRows = DEFAULT_ROWS_PER_GB; + long totalRows = DEFAULT_ROWS_PER_GB; int measureAfter = 0; float sampleRate = 1.0f; /** @@ -715,7 +715,7 @@ static class TestOptions { boolean valueRandom = false; boolean valueZipf = false; int valueSize = DEFAULT_VALUE_LENGTH; - int period = (this.perClientRunRows / 10) == 0 ? perClientRunRows : perClientRunRows / 10; + long period = (this.perClientRunRows / 10) == 0 ? perClientRunRows : perClientRunRows / 10; int cycles = 1; int columns = 1; int families = 1; @@ -872,7 +872,7 @@ public void setFilterAll(boolean filterAll) { this.filterAll = filterAll; } - public void setStartRow(int startRow) { + public void setStartRow(long startRow) { this.startRow = startRow; } @@ -888,7 +888,7 @@ public void setNumClientThreads(int numClientThreads) { this.numClientThreads = numClientThreads; } - public void setTotalRows(int totalRows) { + public void setTotalRows(long totalRows) { this.totalRows = totalRows; } @@ -1000,7 +1000,7 @@ public boolean isFilterAll() { return filterAll; } - public int getStartRow() { + public long getStartRow() { return startRow; } @@ -1008,7 +1008,7 @@ public float getSize() { return size; } - public int getPerClientRunRows() { + public long getPerClientRunRows() { return perClientRunRows; } @@ -1016,7 +1016,7 @@ public int getNumClientThreads() { return numClientThreads; } - public int getTotalRows() { + public long getTotalRows() { return totalRows; } @@ -1092,7 +1092,7 @@ public int getValueSize() { return valueSize; } - public int getPeriod() { + public long getPeriod() { return period; } @@ -1141,17 +1141,8 @@ public long getBufferSize() { * A test. Subclass to particularize what happens per row. */ static abstract class TestBase { - // Below is make it so when Tests are all running in the one - // jvm, that they each have a differently seeded Random. - private static final Random randomSeed = new Random(EnvironmentEdgeManager.currentTime()); + private final long everyN; - private static long nextRandomSeed() { - return randomSeed.nextLong(); - } - - private final int everyN; - - protected final Random rand = new Random(nextRandomSeed()); protected final Configuration conf; protected final TestOptions opts; @@ -1180,16 +1171,17 @@ private static long nextRandomSeed() { this.opts = options; this.status = status; this.testName = this.getClass().getSimpleName(); - everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate)); + everyN = (long) (opts.totalRows / (opts.totalRows * opts.sampleRate)); if (options.isValueZipf()) { - this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.2); + this.zipf = + new RandomDistribution.Zipf(ThreadLocalRandom.current(), 1, options.getValueSize(), 1.2); } LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows."); } - int getValueLength(final Random r) { + int getValueLength() { if (this.opts.isValueRandom()) { - return r.nextInt(opts.valueSize); + return ThreadLocalRandom.current().nextInt(opts.valueSize); } else if (this.opts.isValueZipf()) { return Math.abs(this.zipf.nextInt()); } else { @@ -1261,7 +1253,7 @@ void updateScanMetrics(final ScanMetrics metrics) { } } - String generateStatus(final int sr, final int i, final int lr) { + String generateStatus(final long sr, final long i, final long lr) { return "row [start=" + sr + ", current=" + i + ", last=" + lr + "], latency [" + getShortLatencyReport() + "]" + (!isRandomValueSize() ? "" : ", value size [" + getShortValueSizeReport() + "]"); @@ -1271,7 +1263,7 @@ boolean isRandomValueSize() { return opts.valueRandom; } - protected int getReportingPeriod() { + protected long getReportingPeriod() { return opts.period; } @@ -1374,11 +1366,11 @@ long test() throws IOException, InterruptedException { return (System.nanoTime() - startTime) / 1000000; } - int getStartRow() { + long getStartRow() { return opts.startRow; } - int getLastRow() { + long getLastRow() { return getStartRow() + opts.perClientRunRows; } @@ -1386,12 +1378,12 @@ int getLastRow() { * Provides an extension point for tests that don't want a per row invocation. */ void testTimed() throws IOException, InterruptedException { - int startRow = getStartRow(); - int lastRow = getLastRow(); + long startRow = getStartRow(); + long lastRow = getLastRow(); // Report on completion of 1/10th of total. for (int ii = 0; ii < opts.cycles; ii++) { if (opts.cycles > 1) LOG.info("Cycle=" + ii + " of " + opts.cycles); - for (int i = startRow; i < lastRow; i++) { + for (long i = startRow; i < lastRow; i++) { if (i % everyN != 0) continue; long startTime = System.nanoTime(); boolean requestSent = false; @@ -1437,7 +1429,7 @@ public String getShortValueSizeReport() { * @return true if the row was sent to server and need to record metrics. False if not, multiGet * and multiPut e.g., the rows are sent to server only if enough gets/puts are gathered. */ - abstract boolean testRow(final int i, final long startTime) + abstract boolean testRow(final long i, final long startTime) throws IOException, InterruptedException; } @@ -1485,7 +1477,7 @@ static abstract class MetaTest extends TableTest { MetaTest(Connection con, TestOptions options, Status status) { super(con, options, status); - keyLength = Integer.toString(opts.perClientRunRows).length(); + keyLength = Long.toString(opts.perClientRunRows).length(); } @Override @@ -1496,7 +1488,7 @@ void onTakedown() throws IOException { /* * Generates Lexicographically ascending strings */ - protected byte[] getSplitKey(final int i) { + protected byte[] getSplitKey(final long i) { return Bytes.toBytes(String.format("%0" + keyLength + "d", i)); } @@ -1533,11 +1525,11 @@ static class AsyncRandomReadTest extends AsyncTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { if (opts.randomSleep > 0) { Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep)); } - Get get = new Get(getRandomRow(this.rand, opts.totalRows)); + Get get = new Get(getRandomRow(opts.totalRows)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { @@ -1590,8 +1582,8 @@ public static V propagate(Callable callable) { } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 10; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 10; return period == 0 ? opts.perClientRunRows : period; } @@ -1612,8 +1604,8 @@ static class AsyncRandomWriteTest extends AsyncSequentialWriteTest { } @Override - protected byte[] generateRow(final int i) { - return getRandomRow(this.rand, opts.totalRows); + protected byte[] generateRow(final long i) { + return getRandomRow(opts.totalRows); } } @@ -1641,7 +1633,7 @@ void testTakedown() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) @@ -1674,7 +1666,7 @@ static class AsyncSequentialReadTest extends AsyncTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { Get get = new Get(format(i)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); @@ -1710,22 +1702,22 @@ static class AsyncSequentialWriteTest extends AsyncTableTest { } } - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return format(i); } @Override @SuppressWarnings("ReturnValueIgnored") - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { byte[] row = generateRow(i); Put put = new Put(row); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); for (int column = 0; column < opts.columns; column++) { byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" + column); - byte[] value = generateData(this.rand, getValueLength(this.rand)); + byte[] value = generateData(getValueLength()); if (opts.useTags) { - byte[] tag = generateData(this.rand, TAG_LENGTH); + byte[] tag = generateData(TAG_LENGTH); Tag[] tags = new Tag[opts.noOfTags]; for (int n = 0; n < opts.noOfTags; n++) { Tag t = new ArrayBackedTag((byte) n, tag); @@ -1791,11 +1783,10 @@ static class RandomSeekScanTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { - Scan scan = - new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows)).setCaching(opts.caching) - .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) - .setReadType(opts.scanReadType).setScanMetricsEnabled(true); + boolean testRow(final long i, final long startTime) throws IOException { + Scan scan = new Scan().withStartRow(getRandomRow(opts.totalRows)).setCaching(opts.caching) + .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) + .setReadType(opts.scanReadType).setScanMetricsEnabled(true); FilterList list = new FilterList(); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); @@ -1826,8 +1817,8 @@ boolean testRow(final int i, final long startTime) throws IOException { } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 100; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 100; return period == 0 ? opts.perClientRunRows : period; } @@ -1839,7 +1830,7 @@ static abstract class RandomScanWithRangeTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Pair startAndStopRow = getStartAndStopRow(); Scan scan = new Scan().withStartRow(startAndStopRow.getFirst()) .withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching) @@ -1881,15 +1872,15 @@ boolean testRow(final int i, final long startTime) throws IOException { protected abstract Pair getStartAndStopRow(); - protected Pair generateStartAndStopRows(int maxRange) { - int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows; - int stop = start + maxRange; + protected Pair generateStartAndStopRows(long maxRange) { + long start = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) % opts.totalRows; + long stop = start + maxRange; return new Pair<>(format(start), format(stop)); } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 100; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 100; return period == 0 ? opts.perClientRunRows : period; } } @@ -1952,11 +1943,11 @@ static class RandomReadTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { if (opts.randomSleep > 0) { Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep)); } - Get get = new Get(getRandomRow(this.rand, opts.totalRows)); + Get get = new Get(getRandomRow(opts.totalRows)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { @@ -2000,8 +1991,8 @@ boolean testRow(final int i, final long startTime) throws IOException, Interrupt } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 10; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 10; return period == 0 ? opts.perClientRunRows : period; } @@ -2034,19 +2025,19 @@ void onStartup() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { if (opts.randomSleep > 0) { Thread.sleep(rd.nextInt(opts.randomSleep)); } - HRegionLocation hRegionLocation = - regionLocator.getRegionLocation(getSplitKey(rd.nextInt(opts.perClientRunRows)), true); + HRegionLocation hRegionLocation = regionLocator.getRegionLocation( + getSplitKey(ThreadLocalRandom.current().nextLong(opts.perClientRunRows)), true); LOG.debug("get location for region: " + hRegionLocation); return true; } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 10; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 10; return period == 0 ? opts.perClientRunRows : period; } @@ -2062,8 +2053,8 @@ static class RandomWriteTest extends SequentialWriteTest { } @Override - protected byte[] generateRow(final int i) { - return getRandomRow(this.rand, opts.totalRows); + protected byte[] generateRow(final long i) { + return getRandomRow(opts.totalRows); } } @@ -2074,8 +2065,8 @@ static class RandomDeleteTest extends SequentialDeleteTest { } @Override - protected byte[] generateRow(final int i) { - return getRandomRow(this.rand, opts.totalRows); + protected byte[] generateRow(final long i) { + return getRandomRow(opts.totalRows); } } @@ -2096,7 +2087,7 @@ void testTakedown() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) @@ -2139,7 +2130,7 @@ void testTakedown() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) @@ -2187,12 +2178,12 @@ byte[] getQualifier() { } @Override - int getStartRow() { + long getStartRow() { return 0; } @Override - int getLastRow() { + long getLastRow() { return opts.perClientRunRows; } } @@ -2203,7 +2194,7 @@ static class IncrementTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Increment increment = new Increment(format(i)); // unlike checkAndXXX tests, which make most sense to do on a single value, // if multiple families are specified for an increment test we assume it is @@ -2223,7 +2214,7 @@ static class AppendTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { byte[] bytes = format(i); Append append = new Append(bytes); // unlike checkAndXXX tests, which make most sense to do on a single value, @@ -2244,7 +2235,7 @@ static class CheckAndMutateTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { final byte[] bytes = format(i); // checkAndXXX tests operate on only a single value // Put a known value so when we go to check it, it is there. @@ -2265,7 +2256,7 @@ static class CheckAndPutTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { final byte[] bytes = format(i); // checkAndXXX tests operate on only a single value // Put a known value so when we go to check it, it is there. @@ -2284,7 +2275,7 @@ static class CheckAndDeleteTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { final byte[] bytes = format(i); // checkAndXXX tests operate on only a single value // Put a known value so when we go to check it, it is there. @@ -2308,7 +2299,7 @@ static class CleanMetaTest extends MetaTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { try { RegionInfo regionInfo = connection.getRegionLocator(table.getName()) .getRegionLocation(getSplitKey(i), false).getRegion(); @@ -2333,7 +2324,7 @@ static class SequentialReadTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Get get = new Get(format(i)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); @@ -2365,21 +2356,21 @@ static class SequentialWriteTest extends BufferedMutatorTest { } } - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return format(i); } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { byte[] row = generateRow(i); Put put = new Put(row); for (int family = 0; family < opts.families; family++) { byte familyName[] = Bytes.toBytes(FAMILY_NAME_BASE + family); for (int column = 0; column < opts.columns; column++) { byte[] qualifier = column == 0 ? COLUMN_ZERO : Bytes.toBytes("" + column); - byte[] value = generateData(this.rand, getValueLength(this.rand)); + byte[] value = generateData(getValueLength()); if (opts.useTags) { - byte[] tag = generateData(this.rand, TAG_LENGTH); + byte[] tag = generateData(TAG_LENGTH); Tag[] tags = new Tag[opts.noOfTags]; for (int n = 0; n < opts.noOfTags; n++) { Tag t = new ArrayBackedTag((byte) n, tag); @@ -2421,12 +2412,12 @@ static class SequentialDeleteTest extends BufferedMutatorTest { super(con, options, status); } - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return format(i); } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { byte[] row = generateRow(i); Delete delete = new Delete(row); for (int family = 0; family < opts.families; family++) { @@ -2453,7 +2444,7 @@ static class MetaWriteTest extends MetaTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { List regionInfos = new ArrayList(); RegionInfo regionInfo = (RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE_NAME)) .setStartKey(getSplitKey(i)).setEndKey(getSplitKey(i + 1)).build()); @@ -2462,7 +2453,7 @@ boolean testRow(final int i, final long startTime) throws IOException { // write the serverName columns MetaTableAccessor.updateRegionLocation(connection, regionInfo, - ServerName.valueOf("localhost", 60010, rand.nextLong()), i, + ServerName.valueOf("localhost", 60010, ThreadLocalRandom.current().nextLong()), i, EnvironmentEdgeManager.currentTime()); return true; } @@ -2480,8 +2471,8 @@ static class FilteredScanTest extends TableTest { } @Override - boolean testRow(int i, final long startTime) throws IOException { - byte[] value = generateData(this.rand, getValueLength(this.rand)); + boolean testRow(long i, final long startTime) throws IOException { + byte[] value = generateData(getValueLength()); Scan scan = constructScan(value); ResultScanner scanner = null; try { @@ -2528,7 +2519,7 @@ protected Scan constructScan(byte[] valuePrefix) throws IOException { * @param timeMs Time taken in milliseconds. * @return String value with label, ie '123.76 MB/s' */ - private static String calculateMbps(int rows, long timeMs, final int valueSize, int families, + private static String calculateMbps(long rows, long timeMs, final int valueSize, int families, int columns) { BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH + ((valueSize + (FAMILY_NAME_BASE.length() + 1) + COLUMN_ZERO.length) * columns) * families); @@ -2542,9 +2533,9 @@ private static String calculateMbps(int rows, long timeMs, final int valueSize, * @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of passed number (Does * absolute in case number is negative). */ - public static byte[] format(final int number) { + public static byte[] format(final long number) { byte[] b = new byte[ROW_LENGTH]; - int d = Math.abs(number); + long d = Math.abs(number); for (int i = b.length - 1; i >= 0; i--) { b[i] = (byte) ((d % 10) + '0'); d /= 10; @@ -2557,10 +2548,11 @@ public static byte[] format(final int number) { * test, generation of the key and value consumes about 30% of CPU time. * @return Generated random value to insert into a table cell. */ - public static byte[] generateData(final Random r, int length) { + public static byte[] generateData(int length) { byte[] b = new byte[length]; int i; + Random r = ThreadLocalRandom.current(); for (i = 0; i < (length - 8); i += 8) { b[i] = (byte) (65 + r.nextInt(26)); b[i + 1] = b[i]; @@ -2579,12 +2571,12 @@ public static byte[] generateData(final Random r, int length) { return b; } - static byte[] getRandomRow(final Random random, final int totalRows) { - return format(generateRandomRow(random, totalRows)); + static byte[] getRandomRow(final long totalRows) { + return format(generateRandomRow(totalRows)); } - static int generateRandomRow(final Random random, final int totalRows) { - return random.nextInt(Integer.MAX_VALUE) % totalRows; + static long generateRandomRow(final long totalRows) { + return ThreadLocalRandom.current().nextLong(Long.MAX_VALUE) % totalRows; } static RunResult runOneClient(final Class cmd, Configuration conf, @@ -2619,7 +2611,7 @@ static RunResult runOneClient(final Class cmd, Configuration status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" + " (" - + calculateMbps((int) (opts.perClientRunRows * opts.sampleRate), totalElapsedTime, + + calculateMbps((long) (opts.perClientRunRows * opts.sampleRate), totalElapsedTime, getAverageValueLength(opts), opts.families, opts.columns) + ")"); @@ -2814,7 +2806,7 @@ static TestOptions parseOpts(Queue args) { final String rows = "--rows="; if (cmd.startsWith(rows)) { - opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length())); + opts.perClientRunRows = Long.parseLong(cmd.substring(rows.length())); continue; } @@ -2838,7 +2830,7 @@ static TestOptions parseOpts(Queue args) { final String startRow = "--startRow="; if (cmd.startsWith(startRow)) { - opts.startRow = Integer.parseInt(cmd.substring(startRow.length())); + opts.startRow = Long.parseLong(cmd.substring(startRow.length())); continue; } @@ -3136,10 +3128,10 @@ static TestOptions calculateRowsAndSize(final TestOptions opts) { && (opts.getCmdName().equals(RANDOM_READ) || opts.getCmdName().equals(RANDOM_SEEK_SCAN))) && opts.size != DEFAULT_OPTS.size && opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows ) { - opts.totalRows = (int) opts.size * rowsPerGB; + opts.totalRows = (long) (opts.size * rowsPerGB); } else if (opts.size != DEFAULT_OPTS.size) { // total size in GB specified - opts.totalRows = (int) opts.size * rowsPerGB; + opts.totalRows = (long) (opts.size * rowsPerGB); opts.perClientRunRows = opts.totalRows / opts.numClientThreads; } else { opts.totalRows = opts.perClientRunRows * opts.numClientThreads; diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java index 7e4babe58b91..1e4493a47008 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java @@ -39,8 +39,6 @@ import java.util.NoSuchElementException; import java.util.Properties; import java.util.Queue; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -122,7 +120,7 @@ public void testWriteInputFile() throws IOException { public void testSizeCalculation() { TestOptions opts = new PerformanceEvaluation.TestOptions(); opts = PerformanceEvaluation.calculateRowsAndSize(opts); - int rows = opts.getPerClientRunRows(); + long rows = opts.getPerClientRunRows(); // Default row count final int defaultPerClientRunRows = 1024 * 1024; assertEquals(defaultPerClientRunRows, rows); @@ -144,7 +142,7 @@ public void testSizeCalculation() { public void testRandomReadCalculation() { TestOptions opts = new PerformanceEvaluation.TestOptions(); opts = PerformanceEvaluation.calculateRowsAndSize(opts); - int rows = opts.getPerClientRunRows(); + long rows = opts.getPerClientRunRows(); // Default row count final int defaultPerClientRunRows = 1024 * 1024; assertEquals(defaultPerClientRunRows, rows); @@ -160,9 +158,8 @@ public void testRandomReadCalculation() { assertEquals(1000, opts.getPerClientRunRows()); // assuming we will get one before this loop expires boolean foundValue = false; - Random rand = ThreadLocalRandom.current(); for (int i = 0; i < 10000000; i++) { - int randomRow = PerformanceEvaluation.generateRandomRow(rand, opts.totalRows); + long randomRow = PerformanceEvaluation.generateRandomRow(opts.totalRows); if (randomRow > 1000) { foundValue = true; break; @@ -184,7 +181,7 @@ public void testZipfian() throws NoSuchMethodException, SecurityException, Insta ctor.setAccessible(true); Histogram histogram = (Histogram) ctor.newInstance(new UniformReservoir(1024 * 500)); for (int i = 0; i < 100; i++) { - histogram.update(rrt.getValueLength(null)); + histogram.update(rrt.getValueLength()); } Snapshot snapshot = histogram.getSnapshot(); double stddev = snapshot.getStdDev(); @@ -400,7 +397,7 @@ void onTakedown() throws IOException { } @Override - boolean testRow(int i, long startTime) throws IOException, InterruptedException { + boolean testRow(long i, long startTime) throws IOException, InterruptedException { return false; } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java index f535ccf7e3e2..8c55f938147e 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedHFileOutputFormat2.java @@ -506,23 +506,19 @@ public void testJobConfiguration() throws Exception { } private byte[][] generateRandomStartKeys(int numKeys) { - Random random = new Random(); byte[][] ret = new byte[numKeys][]; // first region start key is always empty ret[0] = HConstants.EMPTY_BYTE_ARRAY; for (int i = 1; i < numKeys; i++) { - ret[i] = - PerformanceEvaluation.generateData(random, PerformanceEvaluation.DEFAULT_VALUE_LENGTH); + ret[i] = PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH); } return ret; } private byte[][] generateRandomSplitKeys(int numKeys) { - Random random = new Random(); byte[][] ret = new byte[numKeys][]; for (int i = 0; i < numKeys; i++) { - ret[i] = - PerformanceEvaluation.generateData(random, PerformanceEvaluation.DEFAULT_VALUE_LENGTH); + ret[i] = PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH); } return ret; } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 3c486a8a52f8..b5f48b08723a 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -34,14 +34,12 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Random; import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; @@ -543,23 +541,19 @@ public void testJobConfiguration() throws Exception { } private byte[][] generateRandomStartKeys(int numKeys) { - Random random = ThreadLocalRandom.current(); byte[][] ret = new byte[numKeys][]; // first region start key is always empty ret[0] = HConstants.EMPTY_BYTE_ARRAY; for (int i = 1; i < numKeys; i++) { - ret[i] = - PerformanceEvaluation.generateData(random, PerformanceEvaluation.DEFAULT_VALUE_LENGTH); + ret[i] = PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH); } return ret; } private byte[][] generateRandomSplitKeys(int numKeys) { - Random random = ThreadLocalRandom.current(); byte[][] ret = new byte[numKeys][]; for (int i = 0; i < numKeys; i++) { - ret[i] = - PerformanceEvaluation.generateData(random, PerformanceEvaluation.DEFAULT_VALUE_LENGTH); + ret[i] = PerformanceEvaluation.generateData(PerformanceEvaluation.DEFAULT_VALUE_LENGTH); } return ret; }