Skip to content

Commit 6dd1b58

Browse files
Apache9Apache9
authored
HBASE-27148 Move minimum hadoop 3 support version to 3.2.3 (#4561) (#4599)
Signed-off-by: Xin Sun <[email protected]> (cherry picked from commit 41972cb)
1 parent 92525fb commit 6dd1b58

File tree

3 files changed

+112
-83
lines changed

3 files changed

+112
-83
lines changed

‎hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@
144144
import org.apache.hadoop.hdfs.DFSClient;
145145
import org.apache.hadoop.hdfs.DistributedFileSystem;
146146
import org.apache.hadoop.hdfs.MiniDFSCluster;
147+
import org.apache.hadoop.hdfs.server.datanode.DataNode;
148+
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
147149
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
148150
import org.apache.hadoop.mapred.JobConf;
149151
import org.apache.hadoop.mapred.MiniMRCluster;
@@ -196,6 +198,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
196198
public static final boolean PRESPLIT_TEST_TABLE = true;
197199

198200
private MiniDFSCluster dfsCluster = null;
201+
private FsDatasetAsyncDiskServiceFixer dfsClusterFixer = null;
199202

200203
private volatile HBaseCluster hbaseCluster = null;
201204
private MiniMRCluster mrCluster = null;
@@ -574,6 +577,56 @@ public boolean cleanupDataTestDirOnTestFS(String subdirName) throws IOException
574577
return getTestFileSystem().delete(cpath, true);
575578
}
576579

580+
// Workaround to avoid IllegalThreadStateException
581+
// See HBASE-27148 for more details
582+
private static final class FsDatasetAsyncDiskServiceFixer extends Thread {
583+
584+
private volatile boolean stopped = false;
585+
586+
private final MiniDFSCluster cluster;
587+
588+
FsDatasetAsyncDiskServiceFixer(MiniDFSCluster cluster) {
589+
super("FsDatasetAsyncDiskServiceFixer");
590+
setDaemon(true);
591+
this.cluster = cluster;
592+
}
593+
594+
@Override
595+
public void run() {
596+
while (!stopped) {
597+
try {
598+
Thread.sleep(30000);
599+
} catch (InterruptedException e) {
600+
Thread.currentThread().interrupt();
601+
continue;
602+
}
603+
// we could add new datanodes during tests, so here we will check every 30 seconds, as the
604+
// timeout of the thread pool executor is 60 seconds by default.
605+
try {
606+
for (DataNode dn : cluster.getDataNodes()) {
607+
FsDatasetSpi<?> dataset = dn.getFSDataset();
608+
Field service = dataset.getClass().getDeclaredField("asyncDiskService");
609+
service.setAccessible(true);
610+
Object asyncDiskService = service.get(dataset);
611+
Field group = asyncDiskService.getClass().getDeclaredField("threadGroup");
612+
group.setAccessible(true);
613+
ThreadGroup threadGroup = (ThreadGroup) group.get(asyncDiskService);
614+
if (threadGroup.isDaemon()) {
615+
threadGroup.setDaemon(false);
616+
}
617+
}
618+
} catch (Exception e) {
619+
LOG.warn("failed to reset thread pool timeout for FsDatasetAsyncDiskService", e);
620+
}
621+
}
622+
}
623+
624+
void shutdown() {
625+
stopped = true;
626+
interrupt();
627+
}
628+
}
629+
577630
/**
578631
* Start a minidfscluster.
579632
* @param servers How many DNs to start. n * @see #shutdownMiniDFSCluster()
@@ -632,7 +685,8 @@ public MiniDFSCluster startMiniDFSCluster(int servers, final String racks[], Str
632685

633686
this.dfsCluster =
634687
new MiniDFSCluster(0, this.conf, servers, true, true, true, null, racks, hosts, null);
635-
688+
this.dfsClusterFixer = new FsDatasetAsyncDiskServiceFixer(dfsCluster);
689+
this.dfsClusterFixer.start();
636690
// Set this just-started cluster as our filesystem.
637691
setFs();
638692

@@ -656,6 +710,8 @@ public MiniDFSCluster startMiniDFSClusterForTestWAL(int namenodePort) throws IOE
656710
"ERROR");
657711
dfsCluster =
658712
new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null);
713+
this.dfsClusterFixer = new FsDatasetAsyncDiskServiceFixer(dfsCluster);
714+
this.dfsClusterFixer.start();
659715
return dfsCluster;
660716
}
661717

@@ -778,6 +834,12 @@ public void shutdownMiniDFSCluster() throws IOException {
778834
// The below throws an exception per dn, AsynchronousCloseException.
779835
this.dfsCluster.shutdown();
780836
dfsCluster = null;
837+
// It is possible that the dfs cluster is set through setDFSCluster method, where we will not
838+
// have a fixer
839+
if (dfsClusterFixer != null) {
840+
this.dfsClusterFixer.shutdown();
841+
dfsClusterFixer = null;
842+
}
781843
dataTestDirOnTestFS = null;
782844
CommonFSUtils.setFsDefault(this.conf, new Path("file:///"));
783845
}

‎hbase-testing-util/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,16 @@
128128
<type>test-jar</type>
129129
<scope>compile</scope>
130130
</dependency>
131+
<dependency>
132+
<groupId>org.mockito</groupId>
133+
<artifactId>mockito-core</artifactId>
134+
<scope>compile</scope>
135+
</dependency>
136+
<dependency>
137+
<groupId>com.github.stephenc.findbugs</groupId>
138+
<artifactId>findbugs-annotations</artifactId>
139+
<scope>compile</scope>
140+
</dependency>
131141
<dependency>
132142
<groupId>org.slf4j</groupId>
133143
<artifactId>jcl-over-slf4j</artifactId>

‎pom.xml‎

Lines changed: 39 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@
544544
<java.min.version>${compileSource}</java.min.version>
545545
<!-- Dependencies -->
546546
<hadoop-two.version>2.10.0</hadoop-two.version>
547-
<hadoop-three.version>3.1.2</hadoop-three.version>
547+
<hadoop-three.version>3.2.3</hadoop-three.version>
548548
<!-- These must be defined here for downstream build tools that don't look at profiles.
549549
They ought to match the values found in our default hadoop profile, which is
550550
currently "hadoop-2.0". See HBASE-15925 for more info. -->
@@ -557,7 +557,7 @@
557557
<netty.hadoop.version>3.6.2.Final</netty.hadoop.version>
558558
<!-- end HBASE-15925 default hadoop compatibility values -->
559559
<audience-annotations.version>0.5.0</audience-annotations.version>
560-
<avro.version>1.7.7</avro.version>
560+
<avro.version>1.11.0</avro.version>
561561
<caffeine.version>2.8.1</caffeine.version>
562562
<commons-codec.version>1.13</commons-codec.version>
563563
<commons-io.version>2.11.0</commons-io.version>
@@ -2816,8 +2816,6 @@
28162816
--add-opens java.base/java.lang.reflect=ALL-UNNAMED
28172817
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
28182818
${hbase-surefire.argLine}</argLine>
2819-
<!-- We need a minimum HDFS version of 3.2.0 for HADOOP-12760 -->
2820-
<hadoop-three.version>3.2.0</hadoop-three.version>
28212819
<!--
28222820
Value to use for surefire when running jdk11.
28232821
TODO: replicate logic for windows
@@ -3643,12 +3641,16 @@
36433641
<artifactId>jersey-core</artifactId>
36443642
</exclusion>
36453643
<exclusion>
3646-
<groupId>org.codehaus.jackson</groupId>
3647-
<artifactId>jackson-jaxrs</artifactId>
3644+
<groupId>javax.xml.bind</groupId>
3645+
<artifactId>jaxb-api</artifactId>
3646+
</exclusion>
3647+
<exclusion>
3648+
<groupId>javax.ws.rs</groupId>
3649+
<artifactId>jsr311-api</artifactId>
36483650
</exclusion>
36493651
<exclusion>
36503652
<groupId>org.codehaus.jackson</groupId>
3651-
<artifactId>jackson-xc</artifactId>
3653+
<artifactId>*</artifactId>
36523654
</exclusion>
36533655
<exclusion>
36543656
<groupId>io.netty</groupId>
@@ -3666,14 +3668,6 @@
36663668
<groupId>javax.inject</groupId>
36673669
<artifactId>javax.inject</artifactId>
36683670
</exclusion>
3669-
<exclusion>
3670-
<groupId>org.codehaus.jackson</groupId>
3671-
<artifactId>jackson-core-asl</artifactId>
3672-
</exclusion>
3673-
<exclusion>
3674-
<groupId>org.codehaus.jackson</groupId>
3675-
<artifactId>jackson-mapper-asl</artifactId>
3676-
</exclusion>
36773671
<exclusion>
36783672
<groupId>com.google.guava</groupId>
36793673
<artifactId>guava</artifactId>
@@ -3702,12 +3696,8 @@
37023696
<version>${hadoop-three.version}</version>
37033697
<exclusions>
37043698
<exclusion>
3705-
<groupId>com.sun.jersey</groupId>
3706-
<artifactId>jersey-core</artifactId>
3707-
</exclusion>
3708-
<exclusion>
3709-
<groupId>io.netty</groupId>
3710-
<artifactId>netty</artifactId>
3699+
<groupId>org.codehaus.jackson</groupId>
3700+
<artifactId>*</artifactId>
37113701
</exclusion>
37123702
<!--HERE-->
37133703
<exclusion>
@@ -3777,19 +3767,7 @@
37773767
<exclusions>
37783768
<exclusion>
37793769
<groupId>org.codehaus.jackson</groupId>
3780-
<artifactId>jackson-mapper-asl</artifactId>
3781-
</exclusion>
3782-
<exclusion>
3783-
<groupId>org.codehaus.jackson</groupId>
3784-
<artifactId>jackson-core-asl</artifactId>
3785-
</exclusion>
3786-
<exclusion>
3787-
<groupId>org.codehaus.jackson</groupId>
3788-
<artifactId>jackson-jaxrs</artifactId>
3789-
</exclusion>
3790-
<exclusion>
3791-
<groupId>org.codehaus.jackson</groupId>
3792-
<artifactId>jackson-xc</artifactId>
3770+
<artifactId>*</artifactId>
37933771
</exclusion>
37943772
<exclusion>
37953773
<groupId>javax.xml.bind</groupId>
@@ -3860,11 +3838,7 @@
38603838
</exclusion>
38613839
<exclusion>
38623840
<groupId>org.codehaus.jackson</groupId>
3863-
<artifactId>jackson-core-asl</artifactId>
3864-
</exclusion>
3865-
<exclusion>
3866-
<groupId>org.codehaus.jackson</groupId>
3867-
<artifactId>jackson-mapper-asl</artifactId>
3841+
<artifactId>*</artifactId>
38683842
</exclusion>
38693843
<exclusion>
38703844
<groupId>com.google.guava</groupId>
@@ -3929,11 +3903,7 @@
39293903
</exclusion>
39303904
<exclusion>
39313905
<groupId>org.codehaus.jackson</groupId>
3932-
<artifactId>jackson-core-asl</artifactId>
3933-
</exclusion>
3934-
<exclusion>
3935-
<groupId>org.codehaus.jackson</groupId>
3936-
<artifactId>jackson-mapper-asl</artifactId>
3906+
<artifactId>*</artifactId>
39373907
</exclusion>
39383908
<exclusion>
39393909
<groupId>com.google.guava</groupId>
@@ -3968,38 +3938,6 @@
39683938
-->
39693939
</exclusions>
39703940
</dependency>
3971-
<dependency>
3972-
<!-- Is this needed? Seems a duplicate of the above dependency but for the
3973-
classifier-->
3974-
<groupId>org.apache.hadoop</groupId>
3975-
<artifactId>hadoop-hdfs</artifactId>
3976-
<version>${hadoop-three.version}</version>
3977-
<classifier>tests</classifier>
3978-
<type>test-jar</type>
3979-
<scope>test</scope>
3980-
<exclusions>
3981-
<exclusion>
3982-
<groupId>com.sun.jersey</groupId>
3983-
<artifactId>jersey-core</artifactId>
3984-
</exclusion>
3985-
<exclusion>
3986-
<groupId>org.slf4j</groupId>
3987-
<artifactId>slf4j-log4j12</artifactId>
3988-
</exclusion>
3989-
<exclusion>
3990-
<groupId>log4j</groupId>
3991-
<artifactId>log4j</artifactId>
3992-
</exclusion>
3993-
<exclusion>
3994-
<groupId>ch.qos.reload4j</groupId>
3995-
<artifactId>reload4j</artifactId>
3996-
</exclusion>
3997-
<exclusion>
3998-
<groupId>org.slf4j</groupId>
3999-
<artifactId>slf4j-reload4j</artifactId>
4000-
</exclusion>
4001-
</exclusions>
4002-
</dependency>
40033941
<dependency>
40043942
<groupId>org.apache.hadoop</groupId>
40053943
<artifactId>hadoop-auth</artifactId>
@@ -4081,12 +4019,8 @@
40814019
<artifactId>junit</artifactId>
40824020
</exclusion>
40834021
<exclusion>
4084-
<groupId>org.codehause.jackson</groupId>
4085-
<artifactId>jackson-core-asl</artifactId>
4086-
</exclusion>
4087-
<exclusion>
4088-
<groupId>org.codehause.jackson</groupId>
4089-
<artifactId>jackson-mapper-asl</artifactId>
4022+
<groupId>org.codehaus.jackson</groupId>
4023+
<artifactId>*</artifactId>
40904024
</exclusion>
40914025
<exclusion>
40924026
<groupId>org.slf4j</groupId>
@@ -4182,6 +4116,29 @@
41824116
<groupId>org.slf4j</groupId>
41834117
<artifactId>slf4j-reload4j</artifactId>
41844118
</exclusion>
4119+
<!--
4120+
Needed in test context when hadoop-3.3 runs.
4121+
<exclusion>
4122+
<groupId>io.netty</groupId>
4123+
<artifactId>netty-all</artifactId>
4124+
</exclusion>
4125+
-->
4126+
<exclusion>
4127+
<groupId>org.codehaus.jackson</groupId>
4128+
<artifactId>*</artifactId>
4129+
</exclusion>
4130+
<exclusion>
4131+
<groupId>javax.servlet.jsp</groupId>
4132+
<artifactId>jsp-api</artifactId>
4133+
</exclusion>
4134+
<exclusion>
4135+
<groupId>javax.xml.bind</groupId>
4136+
<artifactId>jaxb-api</artifactId>
4137+
</exclusion>
4138+
<exclusion>
4139+
<groupId>javax.ws.rs</groupId>
4140+
<artifactId>jsr311-api</artifactId>
4141+
</exclusion>
41854142
</exclusions>
41864143
</dependency>
41874144
<dependency>

0 commit comments

Comments
 (0)