HBASE-28432 Refactor tools which are under test packaging to a new module hbase-tools

Author: NihalJain

hbase-assembly/src/main/assembly/hadoop-three-compat.xml

@@ -46,6 +46,7 @@
         <include>org.apache.hbase:hbase-it</include>
         <include>org.apache.hbase:hbase-logging</include>
         <include>org.apache.hbase:hbase-mapreduce</include>
+        <include>org.apache.hbase:hbase-diagnostics</include>
         <include>org.apache.hbase:hbase-metrics</include>
         <include>org.apache.hbase:hbase-metrics-api</include>
         <include>org.apache.hbase:hbase-procedure</include>

hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java

@@ -18,8 +18,6 @@
 package org.apache.hadoop.hbase.security;
 
 import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.hbase.AuthUtil;
@@ -172,23 +170,6 @@ public static void setSSLConfiguration(HBaseCommonTestingUtil utility, Class<?>
     KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
   }
 
-  public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
-    throws IOException {
-    String hostname = InetAddress.getLocalHost().getHostName();
-    String keyTabFileConfKey = "hbase." + username + ".keytab.file";
-    String keyTabFileLocation = conf.get(keyTabFileConfKey);
-    String principalConfKey = "hbase." + username + ".kerberos.principal";
-    String principal = org.apache.hadoop.security.SecurityUtil
-      .getServerPrincipal(conf.get(principalConfKey), hostname);
-    if (keyTabFileLocation == null || principal == null) {
-      LOG.warn(
-        "Principal or key tab file null for : " + principalConfKey + ", " + keyTabFileConfKey);
-    }
-    UserGroupInformation ugi =
-      UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
-    return ugi;
-  }
-
   public static UserGroupInformation loginKerberosPrincipal(String krbKeytab, String krbPrincipal)
     throws Exception {
     Configuration conf = new Configuration();

hbase-common/src/test/java/org/apache/hadoop/hbase/io/compress/CompressionTestBase.java

@@ -26,7 +26,7 @@
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.RandomDistribution;
+import org.apache.hadoop.hbase.util.RandomDistributionCopy;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -128,8 +128,8 @@ protected void codecSmallTest(final CompressionCodec codec) throws Exception {
    * Test with a large input (1MB) divided into blocks of 4KB.
    */
   protected void codecLargeTest(final CompressionCodec codec, final double sigma) throws Exception {
-    RandomDistribution.DiscreteRNG rng =
-      new RandomDistribution.Zipf(new Random(), 0, Byte.MAX_VALUE, sigma);
+    RandomDistributionCopy.DiscreteRNG rng =
+      new RandomDistributionCopy.Zipf(new Random(), 0, Byte.MAX_VALUE, sigma);
     final byte[][] input = new byte[LARGE_SIZE / BLOCK_SIZE][BLOCK_SIZE];
     fill(rng, input);
     codecTest(codec, input);
@@ -140,20 +140,20 @@ protected void codecLargeTest(final CompressionCodec codec, final double sigma)
    */
   protected void codecVeryLargeTest(final CompressionCodec codec, final double sigma)
     throws Exception {
-    RandomDistribution.DiscreteRNG rng =
-      new RandomDistribution.Zipf(new Random(), 0, Byte.MAX_VALUE, sigma);
+    RandomDistributionCopy.DiscreteRNG rng =
+      new RandomDistributionCopy.Zipf(new Random(), 0, Byte.MAX_VALUE, sigma);
     final byte[][] input = new byte[1][VERY_LARGE_SIZE];
     fill(rng, input);
     codecTest(codec, input);
   }
 
-  protected static void fill(RandomDistribution.DiscreteRNG rng, byte[][] input) {
+  protected static void fill(RandomDistributionCopy.DiscreteRNG rng, byte[][] input) {
     for (int i = 0; i < input.length; i++) {
       fill(rng, input[i]);
     }
   }
 
-  protected static void fill(RandomDistribution.DiscreteRNG rng, byte[] input) {
+  protected static void fill(RandomDistributionCopy.DiscreteRNG rng, byte[] input) {
     for (int i = 0; i < input.length; i++) {
       input[i] = (byte) rng.nextInt();
     }

hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTestingCopy.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.crypto;
+
+import java.security.Key;
+import javax.crypto.spec.SecretKeySpec;
+
+/**
+ * Return a fixed secret key for AES for testing.
+ */
+public class KeyProviderForTestingCopy implements KeyProvider {
+
+  @Override
+  public void init(String parameters) {
+  }
+
+  @Override
+  public Key getKey(String name) {
+    return new SecretKeySpec(Encryption.hash128(name), "AES");
+  }
+
+  @Override
+  public Key[] getKeys(String[] aliases) {
+    Key[] result = new Key[aliases.length];
+    for (int i = 0; i < aliases.length; i++) {
+      result[i] = new SecretKeySpec(Encryption.hash128(aliases[i]), "AES");
+    }
+    return result;
+  }
+}

hbase-common/src/test/java/org/apache/hadoop/hbase/util/RandomDistributionCopy.java

@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Random;
+
+/**
+ * A class that generates random numbers that follow some distribution.
+ * <p>
+ * Copied from <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
+ * Remove after tfile is committed and use the tfile version of this class instead.
+ * </p>
+ */
+public class RandomDistributionCopy {
+  /**
+   * Interface for discrete (integer) random distributions.
+   */
+  public interface DiscreteRNG {
+    /**
+     * Get the next random number
+     * @return the next random number.
+     */
+    int nextInt();
+  }
+
+  /**
+   * P(i)=1/(max-min)
+   */
+  public static final class Flat implements DiscreteRNG {
+    private final Random random;
+    private final int min;
+    private final int max;
+
+    /**
+     * Generate random integers from min (inclusive) to max (exclusive) following even distribution.
+     * The basic random number generator. Minimum integer maximum integer (exclusive).
+     */
+    public Flat(Random random, int min, int max) {
+      if (min >= max) {
+        throw new IllegalArgumentException("Invalid range");
+      }
+      this.random = random;
+      this.min = min;
+      this.max = max;
+    }
+
+    /**
+     * @see DiscreteRNG#nextInt()
+     */
+    @Override
+    public int nextInt() {
+      return random.nextInt(max - min) + min;
+    }
+  }
+
+  /**
+   * Zipf distribution. The ratio of the probabilities of integer i and j is defined as follows:
+   * P(i)/P(j)=((j-min+1)/(i-min+1))^sigma.
+   */
+  public static final class Zipf implements DiscreteRNG {
+    private static final double DEFAULT_EPSILON = 0.001;
+    private final Random random;
+    private final ArrayList<Integer> k;
+    private final ArrayList<Double> v;
+
+    /**
+     * Constructor The random number generator. minimum integer (inclusvie) maximum integer
+     * (exclusive) parameter sigma. (sigma > 1.0)
+     */
+    public Zipf(Random r, int min, int max, double sigma) {
+      this(r, min, max, sigma, DEFAULT_EPSILON);
+    }
+
+    /**
+     * Constructor. The random number generator. minimum integer (inclusvie) maximum integer
+     * (exclusive) parameter sigma. (sigma > 1.0) Allowable error percentage (0 < epsilon < 1.0).
+     */
+    public Zipf(Random r, int min, int max, double sigma, double epsilon) {
+      if ((max <= min) || (sigma <= 1) || (epsilon <= 0) || (epsilon >= 0.5)) {
+        throw new IllegalArgumentException("Invalid arguments");
+      }
+      random = r;
+      k = new ArrayList<>();
+      v = new ArrayList<>();
+
+      double sum = 0;
+      int last = -1;
+      for (int i = min; i < max; ++i) {
+        sum += Math.exp(-sigma * Math.log(i - min + 1));
+        if ((last == -1) || i * (1 - epsilon) > last) {
+          k.add(i);
+          v.add(sum);
+          last = i;
+        }
+      }
+
+      if (last != max - 1) {
+        k.add(max - 1);
+        v.add(sum);
+      }
+
+      v.set(v.size() - 1, 1.0);
+
+      for (int i = v.size() - 2; i >= 0; --i) {
+        v.set(i, v.get(i) / sum);
+      }
+    }
+
+    /**
+     * @see DiscreteRNG#nextInt()
+     */
+    @Override
+    public int nextInt() {
+      double d = random.nextDouble();
+      int idx = Collections.binarySearch(v, d);
+
+      if (idx > 0) {
+        ++idx;
+      } else {
+        idx = -(idx + 1);
+      }
+
+      if (idx >= v.size()) {
+        idx = v.size() - 1;
+      }
+
+      if (idx == 0) {
+        return k.get(0);
+      }
+
+      int ceiling = k.get(idx);
+      int lower = k.get(idx - 1);
+
+      return ceiling - random.nextInt(ceiling - lower);
+    }
+  }
+
+  /**
+   * Binomial distribution. P(k)=select(n, k)*p^k*(1-p)^(n-k) (k = 0, 1, ..., n)
+   * P(k)=select(max-min-1, k-min)*p^(k-min)*(1-p)^(k-min)*(1-p)^(max-k-1)
+   */
+  public static final class Binomial implements DiscreteRNG {
+    private final Random random;
+    private final int min;
+    private final int n;
+    private final double[] v;
+
+    private static double select(int n, int k) {
+      double ret = 1.0;
+      for (int i = k + 1; i <= n; ++i) {
+        ret *= (double) i / (i - k);
+      }
+      return ret;
+    }
+
+    private static double power(double p, int k) {
+      return Math.exp(k * Math.log(p));
+    }
+
+    /**
+     * Generate random integers from min (inclusive) to max (exclusive) following Binomial
+     * distribution. The basic random number generator. Minimum integer maximum integer (exclusive).
+     * parameter.
+     */
+    public Binomial(Random random, int min, int max, double p) {
+      if (min >= max) {
+        throw new IllegalArgumentException("Invalid range");
+      }
+      this.random = random;
+      this.min = min;
+      this.n = max - min - 1;
+      if (n > 0) {
+        v = new double[n + 1];
+        double sum = 0.0;
+        for (int i = 0; i <= n; ++i) {
+          sum += select(n, i) * power(p, i) * power(1 - p, n - i);
+          v[i] = sum;
+        }
+        for (int i = 0; i <= n; ++i) {
+          v[i] /= sum;
+        }
+      } else {
+        v = null;
+      }
+    }
+
+    /**
+     * @see DiscreteRNG#nextInt()
+     */
+    @Override
+    public int nextInt() {
+      if (v == null) {
+        return min;
+      }
+      double d = random.nextDouble();
+      int idx = Arrays.binarySearch(v, d);
+      if (idx > 0) {
+        ++idx;
+      } else {
+        idx = -(idx + 1);
+      }
+
+      if (idx >= v.length) {
+        idx = v.length - 1;
+      }
+      return idx + min;
+    }
+  }
+}

hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestZstdDictionary.java

@@ -30,7 +30,7 @@
 import org.apache.hadoop.hbase.io.compress.CompressionTestBase;
 import org.apache.hadoop.hbase.io.compress.DictionaryCache;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.RandomDistribution;
+import org.apache.hadoop.hbase.util.RandomDistributionCopy;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
@@ -82,8 +82,8 @@ public static void main(String[] args) throws IOException {
       System.err.println("Usage: TestZstdCodec <outFile>");
       System.exit(-1);
     }
-    final RandomDistribution.DiscreteRNG rng =
-      new RandomDistribution.Zipf(new Random(), 0, Byte.MAX_VALUE, 2);
+    final RandomDistributionCopy.DiscreteRNG rng =
+      new RandomDistributionCopy.Zipf(new Random(), 0, Byte.MAX_VALUE, 2);
     final File outFile = new File(args[0]);
     final byte[] buffer = new byte[1024];
     System.out.println("Generating " + outFile);