Skip to content

Commit 84f4fb3

Browse files
NihalJainNihalJain
authored
HBASE-28432 Refactor tools which are under test packaging to a new module hbase-diagnostics (#6258)
- Move PerformanceEvaluation, LoadTestTool, HFilePerformanceEvaluation, ScanPerformanceEvaluation, LoadBalancerPerformanceEvaluation, and WALPerformanceEvaluation to a new module: hbase-diagnostics Signed-off-by: Istvan Toth <[email protected]> Signed-off-by: Nick Dimiduk <[email protected]>
1 parent cac58ba commit 84f4fb3

File tree

72 files changed

+1180
-347
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+1180
-347
lines changed

‎hbase-assembly/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,10 @@
188188
<groupId>org.apache.hbase</groupId>
189189
<artifactId>hbase-compression-zstd</artifactId>
190190
</dependency>
191+
<dependency>
192+
<groupId>org.apache.hbase</groupId>
193+
<artifactId>hbase-diagnostics</artifactId>
194+
</dependency>
191195
<dependency>
192196
<groupId>jline</groupId>
193197
<artifactId>jline</artifactId>

‎hbase-assembly/src/main/assembly/hadoop-three-compat.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
<include>org.apache.hbase:hbase-it</include>
4747
<include>org.apache.hbase:hbase-logging</include>
4848
<include>org.apache.hbase:hbase-mapreduce</include>
49+
<include>org.apache.hbase:hbase-diagnostics</include>
4950
<include>org.apache.hbase:hbase-metrics</include>
5051
<include>org.apache.hbase:hbase-metrics-api</include>
5152
<include>org.apache.hbase:hbase-procedure</include>

‎hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
package org.apache.hadoop.hbase.security;
1919

2020
import java.io.File;
21-
import java.io.IOException;
22-
import java.net.InetAddress;
2321
import org.apache.hadoop.conf.Configuration;
2422
import org.apache.hadoop.fs.CommonConfigurationKeys;
2523
import org.apache.hadoop.hbase.AuthUtil;
@@ -172,23 +170,6 @@ public static void setSSLConfiguration(HBaseCommonTestingUtil utility, Class<?>
172170
KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
173171
}
174172

175-
public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
176-
throws IOException {
177-
String hostname = InetAddress.getLocalHost().getHostName();
178-
String keyTabFileConfKey = "hbase." + username + ".keytab.file";
179-
String keyTabFileLocation = conf.get(keyTabFileConfKey);
180-
String principalConfKey = "hbase." + username + ".kerberos.principal";
181-
String principal = org.apache.hadoop.security.SecurityUtil
182-
.getServerPrincipal(conf.get(principalConfKey), hostname);
183-
if (keyTabFileLocation == null || principal == null) {
184-
LOG.warn(
185-
"Principal or key tab file null for : " + principalConfKey + ", " + keyTabFileConfKey);
186-
}
187-
UserGroupInformation ugi =
188-
UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
189-
return ugi;
190-
}
191-
192173
public static UserGroupInformation loginKerberosPrincipal(String krbKeytab, String krbPrincipal)
193174
throws Exception {
194175
Configuration conf = new Configuration();

‎hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTesting.java‎ renamed to ‎hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/MockAesKeyProvider.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@
1919

2020
import java.security.Key;
2121
import javax.crypto.spec.SecretKeySpec;
22+
import org.apache.yetus.audience.InterfaceAudience;
2223

2324
/**
2425
* Return a fixed secret key for AES for testing.
2526
*/
26-
public class KeyProviderForTesting implements KeyProvider {
27+
@InterfaceAudience.Private
28+
public class MockAesKeyProvider implements KeyProvider {
2729

2830
@Override
2931
public void init(String parameters) {

‎hbase-common/src/test/java/org/apache/hadoop/hbase/util/RandomDistribution.java‎ renamed to ‎hbase-common/src/main/java/org/apache/hadoop/hbase/util/RandomDistribution.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.Arrays;
2222
import java.util.Collections;
2323
import java.util.Random;
24+
import org.apache.yetus.audience.InterfaceAudience;
2425

2526
/**
2627
* A class that generates random numbers that follow some distribution.
@@ -29,6 +30,7 @@
2930
* Remove after tfile is committed and use the tfile version of this class instead.
3031
* </p>
3132
*/
33+
@InterfaceAudience.Private
3234
public class RandomDistribution {
3335
/**
3436
* Interface for discrete (integer) random distributions.

‎hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ public class TestKeyProvider {
4343
@Test
4444
public void testTestProvider() {
4545
Configuration conf = HBaseConfiguration.create();
46-
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
46+
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
4747
KeyProvider provider = Encryption.getKeyProvider(conf);
4848
assertNotNull("Null returned for provider", provider);
49-
assertTrue("Provider is not the expected type", provider instanceof KeyProviderForTesting);
49+
assertTrue("Provider is not the expected type", provider instanceof MockAesKeyProvider);
5050

5151
Key key = provider.getKey("foo");
5252
assertNotNull("Test provider did not return a key as expected", key);

‎hbase-diagnostics/pom.xml‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<!--
4+
/**
5+
* Licensed to the Apache Software Foundation (ASF) under one
6+
* or more contributor license agreements. See the NOTICE file
7+
* distributed with this work for additional information
8+
* regarding copyright ownership. The ASF licenses this file
9+
* to you under the Apache License, Version 2.0 (the
10+
* "License"); you may not use this file except in compliance
11+
* with the License. You may obtain a copy of the License at
12+
*
13+
* http://www.apache.org/licenses/LICENSE-2.0
14+
*
15+
* Unless required by applicable law or agreed to in writing, software
16+
* distributed under the License is distributed on an "AS IS" BASIS,
17+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18+
* See the License for the specific language governing permissions and
19+
* limitations under the License.
20+
*/
21+
-->
22+
<modelVersion>4.0.0</modelVersion>
23+
<parent>
24+
<groupId>org.apache.hbase</groupId>
25+
<artifactId>hbase-build-configuration</artifactId>
26+
<version>${revision}</version>
27+
<relativePath>../hbase-build-configuration</relativePath>
28+
</parent>
29+
<artifactId>hbase-diagnostics</artifactId>
30+
<name>Apache HBase - Diagnostics Tools</name>
31+
<description>Diagnostics Tools for HBase</description>
32+
33+
<dependencies>
34+
<!-- Intra-project dependencies -->
35+
<dependency>
36+
<groupId>org.apache.hbase</groupId>
37+
<artifactId>hbase-mapreduce</artifactId>
38+
</dependency>
39+
<dependency>
40+
<groupId>org.apache.hbase</groupId>
41+
<artifactId>hbase-logging</artifactId>
42+
</dependency>
43+
<dependency>
44+
<groupId>org.apache.hbase.thirdparty</groupId>
45+
<artifactId>hbase-shaded-miscellaneous</artifactId>
46+
</dependency>
47+
<dependency>
48+
<groupId>org.apache.hbase</groupId>
49+
<artifactId>hbase-common</artifactId>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.apache.hbase</groupId>
53+
<artifactId>hbase-protocol-shaded</artifactId>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.apache.hbase</groupId>
57+
<artifactId>hbase-client</artifactId>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.apache.hbase</groupId>
61+
<artifactId>hbase-zookeeper</artifactId>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.apache.hbase</groupId>
65+
<artifactId>hbase-balancer</artifactId>
66+
</dependency>
67+
<dependency>
68+
<groupId>org.apache.hbase.thirdparty</groupId>
69+
<artifactId>hbase-shaded-gson</artifactId>
70+
</dependency>
71+
<!-- General dependencies -->
72+
<dependency>
73+
<groupId>io.opentelemetry</groupId>
74+
<artifactId>opentelemetry-context</artifactId>
75+
</dependency>
76+
<dependency>
77+
<groupId>io.opentelemetry</groupId>
78+
<artifactId>opentelemetry-api</artifactId>
79+
</dependency>
80+
<dependency>
81+
<groupId>io.dropwizard.metrics</groupId>
82+
<artifactId>metrics-core</artifactId>
83+
</dependency>
84+
<dependency>
85+
<groupId>commons-io</groupId>
86+
<artifactId>commons-io</artifactId>
87+
</dependency>
88+
<dependency>
89+
<groupId>org.apache.commons</groupId>
90+
<artifactId>commons-lang3</artifactId>
91+
</dependency>
92+
<dependency>
93+
<groupId>org.slf4j</groupId>
94+
<artifactId>slf4j-api</artifactId>
95+
</dependency>
96+
<dependency>
97+
<groupId>org.apache.commons</groupId>
98+
<artifactId>commons-math3</artifactId>
99+
</dependency>
100+
<dependency>
101+
<groupId>org.apache.zookeeper</groupId>
102+
<artifactId>zookeeper</artifactId>
103+
</dependency>
104+
<dependency>
105+
<groupId>org.apache.hadoop</groupId>
106+
<artifactId>hadoop-common</artifactId>
107+
<version>${hadoop-three.version}</version>
108+
</dependency>
109+
<dependency>
110+
<groupId>org.apache.hadoop</groupId>
111+
<artifactId>hadoop-mapreduce-client-core</artifactId>
112+
<version>${hadoop-three.version}</version>
113+
</dependency>
114+
<!-- Intra-project test dependencies -->
115+
<dependency>
116+
<groupId>org.apache.hbase</groupId>
117+
<artifactId>hbase-common</artifactId>
118+
<type>test-jar</type>
119+
<scope>test</scope>
120+
</dependency>
121+
<dependency>
122+
<groupId>org.apache.hbase</groupId>
123+
<artifactId>hbase-server</artifactId>
124+
<type>test-jar</type>
125+
<scope>test</scope>
126+
</dependency>
127+
<dependency>
128+
<groupId>org.apache.hbase</groupId>
129+
<artifactId>hbase-annotations</artifactId>
130+
<type>test-jar</type>
131+
<scope>test</scope>
132+
</dependency>
133+
<dependency>
134+
<groupId>org.apache.hbase</groupId>
135+
<artifactId>hbase-zookeeper</artifactId>
136+
<type>test-jar</type>
137+
<scope>test</scope>
138+
</dependency>
139+
<dependency>
140+
<groupId>org.apache.hbase</groupId>
141+
<artifactId>hbase-hadoop-compat</artifactId>
142+
<type>test-jar</type>
143+
<scope>test</scope>
144+
</dependency>
145+
<dependency>
146+
<groupId>org.apache.hadoop</groupId>
147+
<artifactId>hadoop-hdfs</artifactId>
148+
<version>${hadoop-three.version}</version>
149+
<type>test-jar</type>
150+
<scope>test</scope>
151+
</dependency>
152+
<dependency>
153+
<groupId>org.apache.hadoop</groupId>
154+
<artifactId>hadoop-minicluster</artifactId>
155+
<version>${hadoop-three.version}</version>
156+
<scope>test</scope>
157+
<exclusions>
158+
<exclusion>
159+
<groupId>javax.xml.bind</groupId>
160+
<artifactId>jaxb-api</artifactId>
161+
</exclusion>
162+
<exclusion>
163+
<groupId>javax.ws.rs</groupId>
164+
<artifactId>jsr311-api</artifactId>
165+
</exclusion>
166+
</exclusions>
167+
</dependency>
168+
<!-- General test dependencies -->
169+
<dependency>
170+
<groupId>junit</groupId>
171+
<artifactId>junit</artifactId>
172+
<scope>test</scope>
173+
</dependency>
174+
<dependency>
175+
<groupId>org.mockito</groupId>
176+
<artifactId>mockito-core</artifactId>
177+
<scope>test</scope>
178+
</dependency>
179+
<dependency>
180+
<groupId>org.slf4j</groupId>
181+
<artifactId>jul-to-slf4j</artifactId>
182+
<scope>test</scope>
183+
</dependency>
184+
<dependency>
185+
<groupId>org.apache.logging.log4j</groupId>
186+
<artifactId>log4j-1.2-api</artifactId>
187+
<scope>test</scope>
188+
</dependency>
189+
</dependencies>
190+
</project>

‎hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java‎ renamed to ‎hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
import org.apache.hadoop.hbase.io.crypto.CryptoCipherProvider;
2929
import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider;
3030
import org.apache.hadoop.hbase.io.crypto.Encryption;
31-
import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
31+
import org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider;
3232
import org.apache.hadoop.hbase.io.crypto.aes.AES;
3333
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
3434
import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -121,7 +121,7 @@ private void runBenchmarks() throws Exception {
121121

122122
// Add configuration for AES cipher
123123
final Configuration aesconf = new Configuration();
124-
aesconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
124+
aesconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
125125
aesconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
126126
aesconf.setInt("hfile.format.version", 3);
127127
final FileSystem aesfs = FileSystem.get(aesconf);
@@ -137,7 +137,7 @@ private void runBenchmarks() throws Exception {
137137

138138
// Add configuration for Commons cipher
139139
final Configuration cryptoconf = new Configuration();
140-
cryptoconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
140+
cryptoconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
141141
cryptoconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
142142
cryptoconf.setInt("hfile.format.version", 3);
143143
cryptoconf.set(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY, CryptoCipherProvider.class.getName());

‎hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java‎ renamed to ‎hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java‎

File renamed without changes.

‎hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java‎ renamed to ‎hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@
2121
import java.util.ArrayList;
2222
import java.util.List;
2323
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
24+
import org.apache.yetus.audience.InterfaceAudience;
2425
import org.slf4j.Logger;
2526
import org.slf4j.LoggerFactory;
2627

2728
/**
2829
* Code shared by PE tests.
2930
*/
31+
@InterfaceAudience.Private
3032
public class PerformanceEvaluationCommons {
3133
private static final Logger LOG =
3234
LoggerFactory.getLogger(PerformanceEvaluationCommons.class.getName());

0 commit comments

Comments
 (0)