From 6a4e80be5799b115702e458855b6165c2b971f63 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 16 Apr 2025 16:13:59 +0800 Subject: [PATCH 1/2] HADOOP-19422. Upgrade JUnit from 4 to 5 in hadoop-archive-logs. --- .../hadoop/tools/TestHadoopArchiveLogs.java | 161 +++++++++--------- .../tools/TestHadoopArchiveLogsRunner.java | 34 ++-- 2 files changed, 101 insertions(+), 94 deletions(-) diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java index b475cc25e8ee6..4dfd6f9d56d9a 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java @@ -37,8 +37,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.File; import java.io.IOException; @@ -55,7 +56,8 @@ public class TestHadoopArchiveLogs { new Random().nextBytes(DUMMY_DATA); } - @Test(timeout = 10000) + @Test + @Timeout(value = 10) public void testCheckFilesAndSeedApps() throws Exception { Configuration conf = new Configuration(); HadoopArchiveLogs hal = new HadoopArchiveLogs(conf); @@ -64,7 +66,7 @@ public void testCheckFilesAndSeedApps() throws Exception { String suffix = "logs"; Path logDir = new Path(rootLogDir, new Path(USER, suffix)); fs.delete(logDir, true); - Assert.assertFalse(fs.exists(logDir)); + Assertions.assertFalse(fs.exists(logDir)); fs.mkdirs(logDir); // no files found @@ -96,15 +98,16 @@ public void testCheckFilesAndSeedApps() throws Exception { createFile(fs, new Path(app5Path, "file1"), 2); createFile(fs, new Path(app5Path, "file2"), 3); - Assert.assertEquals(0, hal.eligibleApplications.size()); + Assertions.assertEquals(0, hal.eligibleApplications.size()); hal.checkFilesAndSeedApps(fs, rootLogDir, suffix, new Path(rootLogDir, "archive-logs-work")); - Assert.assertEquals(1, hal.eligibleApplications.size()); - Assert.assertEquals(appId5.toString(), + Assertions.assertEquals(1, hal.eligibleApplications.size()); + Assertions.assertEquals(appId5.toString(), hal.eligibleApplications.iterator().next().getAppId()); } - @Test(timeout = 10000) + @Test + @Timeout(value = 10) public void testCheckMaxEligible() throws Exception { Configuration conf = new Configuration(); HadoopArchiveLogs.AppInfo app1 = new HadoopArchiveLogs.AppInfo( @@ -129,7 +132,7 @@ public void testCheckMaxEligible() throws Exception { ApplicationId.newInstance(CLUSTER_TIMESTAMP, 7).toString(), USER); app7.setFinishTime(CLUSTER_TIMESTAMP); HadoopArchiveLogs hal = new HadoopArchiveLogs(conf); - Assert.assertEquals(0, hal.eligibleApplications.size()); + Assertions.assertEquals(0, hal.eligibleApplications.size()); hal.eligibleApplications.add(app1); hal.eligibleApplications.add(app2); hal.eligibleApplications.add(app3); @@ -137,38 +140,39 @@ public void testCheckMaxEligible() throws Exception { hal.eligibleApplications.add(app5); hal.eligibleApplications.add(app6); hal.eligibleApplications.add(app7); - Assert.assertEquals(7, hal.eligibleApplications.size()); + Assertions.assertEquals(7, hal.eligibleApplications.size()); hal.maxEligible = -1; hal.checkMaxEligible(); - Assert.assertEquals(7, hal.eligibleApplications.size()); + Assertions.assertEquals(7, hal.eligibleApplications.size()); hal.maxEligible = 6; hal.checkMaxEligible(); - Assert.assertEquals(6, hal.eligibleApplications.size()); - Assert.assertFalse(hal.eligibleApplications.contains(app5)); + Assertions.assertEquals(6, hal.eligibleApplications.size()); + Assertions.assertFalse(hal.eligibleApplications.contains(app5)); hal.maxEligible = 5; hal.checkMaxEligible(); - Assert.assertEquals(5, hal.eligibleApplications.size()); - Assert.assertFalse(hal.eligibleApplications.contains(app4)); + Assertions.assertEquals(5, hal.eligibleApplications.size()); + Assertions.assertFalse(hal.eligibleApplications.contains(app4)); hal.maxEligible = 4; hal.checkMaxEligible(); - Assert.assertEquals(4, hal.eligibleApplications.size()); - Assert.assertFalse(hal.eligibleApplications.contains(app7)); + Assertions.assertEquals(4, hal.eligibleApplications.size()); + Assertions.assertFalse(hal.eligibleApplications.contains(app7)); hal.maxEligible = 3; hal.checkMaxEligible(); - Assert.assertEquals(3, hal.eligibleApplications.size()); - Assert.assertFalse(hal.eligibleApplications.contains(app1)); + Assertions.assertEquals(3, hal.eligibleApplications.size()); + Assertions.assertFalse(hal.eligibleApplications.contains(app1)); hal.maxEligible = 2; hal.checkMaxEligible(); - Assert.assertEquals(2, hal.eligibleApplications.size()); - Assert.assertFalse(hal.eligibleApplications.contains(app2)); + Assertions.assertEquals(2, hal.eligibleApplications.size()); + Assertions.assertFalse(hal.eligibleApplications.contains(app2)); hal.maxEligible = 1; hal.checkMaxEligible(); - Assert.assertEquals(1, hal.eligibleApplications.size()); - Assert.assertFalse(hal.eligibleApplications.contains(app6)); - Assert.assertTrue(hal.eligibleApplications.contains(app3)); + Assertions.assertEquals(1, hal.eligibleApplications.size()); + Assertions.assertFalse(hal.eligibleApplications.contains(app6)); + Assertions.assertTrue(hal.eligibleApplications.contains(app3)); } - @Test(timeout = 30000) + @Test + @Timeout(value = 30) public void testFilterAppsByAggregatedStatus() throws Exception { try (MiniYARNCluster yarnCluster = new MiniYARNCluster(TestHadoopArchiveLogs.class.getSimpleName(), @@ -206,7 +210,7 @@ public void testFilterAppsByAggregatedStatus() throws Exception { // appImpl8 is not in the RM HadoopArchiveLogs hal = new HadoopArchiveLogs(conf); - Assert.assertEquals(0, hal.eligibleApplications.size()); + Assertions.assertEquals(0, hal.eligibleApplications.size()); hal.eligibleApplications.add( new HadoopArchiveLogs.AppInfo(appImpl1.getApplicationId().toString(), USER)); @@ -234,16 +238,17 @@ public void testFilterAppsByAggregatedStatus() throws Exception { new HadoopArchiveLogs.AppInfo(appImpl8.getApplicationId().toString(), USER); hal.eligibleApplications.add(app8); - Assert.assertEquals(8, hal.eligibleApplications.size()); + Assertions.assertEquals(8, hal.eligibleApplications.size()); hal.filterAppsByAggregatedStatus(); - Assert.assertEquals(3, hal.eligibleApplications.size()); - Assert.assertTrue(hal.eligibleApplications.contains(app4)); - Assert.assertTrue(hal.eligibleApplications.contains(app7)); - Assert.assertTrue(hal.eligibleApplications.contains(app8)); + Assertions.assertEquals(3, hal.eligibleApplications.size()); + Assertions.assertTrue(hal.eligibleApplications.contains(app4)); + Assertions.assertTrue(hal.eligibleApplications.contains(app7)); + Assertions.assertTrue(hal.eligibleApplications.contains(app8)); } } - @Test(timeout = 10000) + @Test + @Timeout(value = 10) public void testGenerateScript() throws Exception { _testGenerateScript(false); _testGenerateScript(true); @@ -276,59 +281,59 @@ private void _testGenerateScript(boolean proxy) throws Exception { File localScript = new File("target", "script.sh"); localScript.delete(); - Assert.assertFalse(localScript.exists()); + Assertions.assertFalse(localScript.exists()); hal.generateScript(localScript); - Assert.assertTrue(localScript.exists()); + Assertions.assertTrue(localScript.exists()); String script = IOUtils.toString(localScript.toURI(), StandardCharsets.UTF_8); String[] lines = script.split("\n"); - Assert.assertEquals(22, lines.length); - Assert.assertEquals("#!/bin/bash", lines[0]); - Assert.assertEquals("set -e", lines[1]); - Assert.assertEquals("set -x", lines[2]); - Assert.assertEquals("if [ \"$YARN_SHELL_ID\" == \"1\" ]; then", lines[3]); + Assertions.assertEquals(22, lines.length); + Assertions.assertEquals("#!/bin/bash", lines[0]); + Assertions.assertEquals("set -e", lines[1]); + Assertions.assertEquals("set -x", lines[2]); + Assertions.assertEquals("if [ \"$YARN_SHELL_ID\" == \"1\" ]; then", lines[3]); boolean oneBefore = true; if (lines[4].contains(app1.toString())) { - Assert.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[4]); - Assert.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[10]); + Assertions.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[4]); + Assertions.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[10]); } else { oneBefore = false; - Assert.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[4]); - Assert.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[10]); + Assertions.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[4]); + Assertions.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[10]); } - Assert.assertEquals("\tuser=\"" + USER + "\"", lines[5]); - Assert.assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir.toString() + Assertions.assertEquals("\tuser=\"" + USER + "\"", lines[5]); + Assertions.assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir.toString() : workingDir2.toString()) + "\"", lines[6]); - Assert.assertEquals("\tremoteRootLogDir=\"" + (oneBefore + Assertions.assertEquals("\tremoteRootLogDir=\"" + (oneBefore ? remoteRootLogDir.toString() : remoteRootLogDir2.toString()) + "\"", lines[7]); - Assert.assertEquals("\tsuffix=\"" + (oneBefore ? suffix : suffix2) + Assertions.assertEquals("\tsuffix=\"" + (oneBefore ? suffix : suffix2) + "\"", lines[8]); - Assert.assertEquals("elif [ \"$YARN_SHELL_ID\" == \"2\" ]; then", + Assertions.assertEquals("elif [ \"$YARN_SHELL_ID\" == \"2\" ]; then", lines[9]); - Assert.assertEquals("\tuser=\"" + USER + "\"", lines[11]); - Assert.assertEquals("\tworkingDir=\"" + (oneBefore + Assertions.assertEquals("\tuser=\"" + USER + "\"", lines[11]); + Assertions.assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir2.toString() : workingDir.toString()) + "\"", lines[12]); - Assert.assertEquals("\tremoteRootLogDir=\"" + (oneBefore + Assertions.assertEquals("\tremoteRootLogDir=\"" + (oneBefore ? remoteRootLogDir2.toString() : remoteRootLogDir.toString()) + "\"", lines[13]); - Assert.assertEquals("\tsuffix=\"" + (oneBefore ? suffix2 : suffix) + Assertions.assertEquals("\tsuffix=\"" + (oneBefore ? suffix2 : suffix) + "\"", lines[14]); - Assert.assertEquals("else", lines[15]); - Assert.assertEquals("\techo \"Unknown Mapping!\"", lines[16]); - Assert.assertEquals("\texit 1", lines[17]); - Assert.assertEquals("fi", lines[18]); - Assert.assertEquals("export HADOOP_CLIENT_OPTS=\"-Xmx1024m\"", lines[19]); - Assert.assertTrue(lines[20].startsWith("export HADOOP_CLASSPATH=")); + Assertions.assertEquals("else", lines[15]); + Assertions.assertEquals("\techo \"Unknown Mapping!\"", lines[16]); + Assertions.assertEquals("\texit 1", lines[17]); + Assertions.assertEquals("fi", lines[18]); + Assertions.assertEquals("export HADOOP_CLIENT_OPTS=\"-Xmx1024m\"", lines[19]); + Assertions.assertTrue(lines[20].startsWith("export HADOOP_CLASSPATH=")); if (proxy) { - Assert.assertEquals( + Assertions.assertEquals( "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." + "HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " + "-workingDir \"$workingDir\" -remoteRootLogDir " + "\"$remoteRootLogDir\" -suffix \"$suffix\"", lines[21]); } else { - Assert.assertEquals( + Assertions.assertEquals( "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." + "HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " + "-workingDir \"$workingDir\" -remoteRootLogDir " + @@ -343,7 +348,8 @@ private void _testGenerateScript(boolean proxy) throws Exception { * are updated as well, if necessary. * @throws Exception */ - @Test(timeout = 5000) + @Test + @Timeout(value = 5) public void testStatuses() throws Exception { LogAggregationStatus[] statuses = new LogAggregationStatus[7]; statuses[0] = LogAggregationStatus.DISABLED; @@ -353,37 +359,38 @@ public void testStatuses() throws Exception { statuses[4] = LogAggregationStatus.SUCCEEDED; statuses[5] = LogAggregationStatus.FAILED; statuses[6] = LogAggregationStatus.TIME_OUT; - Assert.assertArrayEquals(statuses, LogAggregationStatus.values()); + Assertions.assertArrayEquals(statuses, LogAggregationStatus.values()); } - @Test(timeout = 5000) + @Test + @Timeout(value = 5) public void testPrepareWorkingDir() throws Exception { Configuration conf = new Configuration(); HadoopArchiveLogs hal = new HadoopArchiveLogs(conf); FileSystem fs = FileSystem.getLocal(conf); Path workingDir = new Path("target", "testPrepareWorkingDir"); fs.delete(workingDir, true); - Assert.assertFalse(fs.exists(workingDir)); + Assertions.assertFalse(fs.exists(workingDir)); // -force is false and the dir doesn't exist so it will create one hal.force = false; boolean dirPrepared = hal.prepareWorkingDir(fs, workingDir); - Assert.assertTrue(dirPrepared); - Assert.assertTrue(fs.exists(workingDir)); - Assert.assertEquals( + Assertions.assertTrue(dirPrepared); + Assertions.assertTrue(fs.exists(workingDir)); + Assertions.assertEquals( new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, !Shell.WINDOWS), fs.getFileStatus(workingDir).getPermission()); // Throw a file in the dir Path dummyFile = new Path(workingDir, "dummy.txt"); fs.createNewFile(dummyFile); - Assert.assertTrue(fs.exists(dummyFile)); + Assertions.assertTrue(fs.exists(dummyFile)); // -force is false and the dir exists, so nothing will happen and the dummy // still exists dirPrepared = hal.prepareWorkingDir(fs, workingDir); - Assert.assertFalse(dirPrepared); - Assert.assertTrue(fs.exists(workingDir)); - Assert.assertTrue(fs.exists(dummyFile)); - Assert.assertEquals( + Assertions.assertFalse(dirPrepared); + Assertions.assertTrue(fs.exists(workingDir)); + Assertions.assertTrue(fs.exists(dummyFile)); + Assertions.assertEquals( new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, !Shell.WINDOWS), fs.getFileStatus(workingDir).getPermission()); @@ -391,13 +398,13 @@ public void testPrepareWorkingDir() throws Exception { // won't exist anymore hal.force = true; dirPrepared = hal.prepareWorkingDir(fs, workingDir); - Assert.assertTrue(dirPrepared); - Assert.assertTrue(fs.exists(workingDir)); - Assert.assertEquals( + Assertions.assertTrue(dirPrepared); + Assertions.assertTrue(fs.exists(workingDir)); + Assertions.assertEquals( new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, !Shell.WINDOWS), fs.getFileStatus(workingDir).getPermission()); - Assert.assertFalse(fs.exists(dummyFile)); + Assertions.assertFalse(fs.exists(dummyFile)); } private static void createFile(FileSystem fs, Path p, long sizeMultiple) @@ -413,7 +420,7 @@ private static void createFile(FileSystem fs, Path p, long sizeMultiple) out.close(); } } - Assert.assertTrue(fs.exists(p)); + Assertions.assertTrue(fs.exists(p)); } private static RMApp createRMApp(int id, Configuration conf, RMContext rmContext, diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java index 5369338d48638..59b7bcbfff29d 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java @@ -32,11 +32,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.junit.rules.Timeout; import org.mockito.Mockito; @@ -45,7 +45,7 @@ import java.util.Comparator; import java.util.Random; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; public class TestHadoopArchiveLogsRunner { @@ -71,7 +71,7 @@ public class TestHadoopArchiveLogsRunner { @Rule public Timeout globalTimeout = new Timeout(50000); - @Before + @BeforeEach public void setup() throws Exception { yarnCluster = new MiniYARNCluster( TestHadoopArchiveLogsRunner.class.getSimpleName(), 1, 2, 1, 1); @@ -100,10 +100,10 @@ public void setup() throws Exception { createFile(fs, new Path(app1Path, "log" + (i + 1)), FILE_SIZES[i]); } FileStatus[] app1Files = fs.listStatus(app1Path); - Assert.assertEquals(FILE_COUNT, app1Files.length); + Assertions.assertEquals(FILE_COUNT, app1Files.length); } - @After + @AfterEach public void teardown() throws IOException { if (fs != null) { fs.close(); @@ -124,12 +124,12 @@ public void testHadoopArchiveLogs() throws Exception { fs = FileSystem.get(conf); FileStatus[] app1Files = fs.listStatus(app1Path); - Assert.assertEquals(1, app1Files.length); + Assertions.assertEquals(1, app1Files.length); FileStatus harFile = app1Files[0]; - Assert.assertEquals(app1.toString() + ".har", harFile.getPath().getName()); + Assertions.assertEquals(app1.toString() + ".har", harFile.getPath().getName()); Path harPath = new Path("har:///" + harFile.getPath().toUri().getRawPath()); FileStatus[] harLogs = HarFs.get(harPath.toUri(), conf).listStatus(harPath); - Assert.assertEquals(FILE_COUNT, harLogs.length); + Assertions.assertEquals(FILE_COUNT, harLogs.length); Arrays.sort(harLogs, new Comparator() { @Override public int compare(FileStatus o1, FileStatus o2) { @@ -138,15 +138,15 @@ public int compare(FileStatus o1, FileStatus o2) { }); for (int i = 0; i < FILE_COUNT; i++) { FileStatus harLog = harLogs[i]; - Assert.assertEquals("log" + (i + 1), harLog.getPath().getName()); - Assert.assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, harLog.getLen()); - Assert.assertEquals( + Assertions.assertEquals("log" + (i + 1), harLog.getPath().getName()); + Assertions.assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, harLog.getLen()); + Assertions.assertEquals( new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE), harLog.getPermission()); - Assert.assertEquals(System.getProperty("user.name"), + Assertions.assertEquals(System.getProperty("user.name"), harLog.getOwner()); } - Assert.assertEquals(0, fs.listStatus(workingDir).length); + Assertions.assertEquals(0, fs.listStatus(workingDir).length); } @Test @@ -162,7 +162,7 @@ public void testHadoopArchiveLogsWithArchiveError() throws Exception { FileStatus[] app1Files = fs.listStatus(app1Path); assertEquals(FILE_COUNT, app1Files.length); for (int i = 0; i < FILE_COUNT; i++) { - Assert.assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, + Assertions.assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, app1Files[i].getLen()); } } From a44b5fe4aa2252852cd385a223777d28723d0980 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 16 Apr 2025 23:21:47 +0800 Subject: [PATCH 2/2] HADOOP-19422. Fix CheckStyle Issue. --- .../hadoop/tools/TestHadoopArchiveLogs.java | 144 +++++++++--------- .../tools/TestHadoopArchiveLogsRunner.java | 34 ++--- 2 files changed, 90 insertions(+), 88 deletions(-) diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java index 4dfd6f9d56d9a..eb99066fffdf6 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java @@ -18,6 +18,11 @@ package org.apache.hadoop.tools; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -37,7 +42,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -66,7 +70,7 @@ public void testCheckFilesAndSeedApps() throws Exception { String suffix = "logs"; Path logDir = new Path(rootLogDir, new Path(USER, suffix)); fs.delete(logDir, true); - Assertions.assertFalse(fs.exists(logDir)); + assertFalse(fs.exists(logDir)); fs.mkdirs(logDir); // no files found @@ -98,11 +102,11 @@ public void testCheckFilesAndSeedApps() throws Exception { createFile(fs, new Path(app5Path, "file1"), 2); createFile(fs, new Path(app5Path, "file2"), 3); - Assertions.assertEquals(0, hal.eligibleApplications.size()); + assertEquals(0, hal.eligibleApplications.size()); hal.checkFilesAndSeedApps(fs, rootLogDir, suffix, new Path(rootLogDir, "archive-logs-work")); - Assertions.assertEquals(1, hal.eligibleApplications.size()); - Assertions.assertEquals(appId5.toString(), + assertEquals(1, hal.eligibleApplications.size()); + assertEquals(appId5.toString(), hal.eligibleApplications.iterator().next().getAppId()); } @@ -132,7 +136,7 @@ public void testCheckMaxEligible() throws Exception { ApplicationId.newInstance(CLUSTER_TIMESTAMP, 7).toString(), USER); app7.setFinishTime(CLUSTER_TIMESTAMP); HadoopArchiveLogs hal = new HadoopArchiveLogs(conf); - Assertions.assertEquals(0, hal.eligibleApplications.size()); + assertEquals(0, hal.eligibleApplications.size()); hal.eligibleApplications.add(app1); hal.eligibleApplications.add(app2); hal.eligibleApplications.add(app3); @@ -140,35 +144,35 @@ public void testCheckMaxEligible() throws Exception { hal.eligibleApplications.add(app5); hal.eligibleApplications.add(app6); hal.eligibleApplications.add(app7); - Assertions.assertEquals(7, hal.eligibleApplications.size()); + assertEquals(7, hal.eligibleApplications.size()); hal.maxEligible = -1; hal.checkMaxEligible(); - Assertions.assertEquals(7, hal.eligibleApplications.size()); + assertEquals(7, hal.eligibleApplications.size()); hal.maxEligible = 6; hal.checkMaxEligible(); - Assertions.assertEquals(6, hal.eligibleApplications.size()); - Assertions.assertFalse(hal.eligibleApplications.contains(app5)); + assertEquals(6, hal.eligibleApplications.size()); + assertFalse(hal.eligibleApplications.contains(app5)); hal.maxEligible = 5; hal.checkMaxEligible(); - Assertions.assertEquals(5, hal.eligibleApplications.size()); - Assertions.assertFalse(hal.eligibleApplications.contains(app4)); + assertEquals(5, hal.eligibleApplications.size()); + assertFalse(hal.eligibleApplications.contains(app4)); hal.maxEligible = 4; hal.checkMaxEligible(); - Assertions.assertEquals(4, hal.eligibleApplications.size()); - Assertions.assertFalse(hal.eligibleApplications.contains(app7)); + assertEquals(4, hal.eligibleApplications.size()); + assertFalse(hal.eligibleApplications.contains(app7)); hal.maxEligible = 3; hal.checkMaxEligible(); - Assertions.assertEquals(3, hal.eligibleApplications.size()); - Assertions.assertFalse(hal.eligibleApplications.contains(app1)); + assertEquals(3, hal.eligibleApplications.size()); + assertFalse(hal.eligibleApplications.contains(app1)); hal.maxEligible = 2; hal.checkMaxEligible(); - Assertions.assertEquals(2, hal.eligibleApplications.size()); - Assertions.assertFalse(hal.eligibleApplications.contains(app2)); + assertEquals(2, hal.eligibleApplications.size()); + assertFalse(hal.eligibleApplications.contains(app2)); hal.maxEligible = 1; hal.checkMaxEligible(); - Assertions.assertEquals(1, hal.eligibleApplications.size()); - Assertions.assertFalse(hal.eligibleApplications.contains(app6)); - Assertions.assertTrue(hal.eligibleApplications.contains(app3)); + assertEquals(1, hal.eligibleApplications.size()); + assertFalse(hal.eligibleApplications.contains(app6)); + assertTrue(hal.eligibleApplications.contains(app3)); } @Test @@ -210,7 +214,7 @@ public void testFilterAppsByAggregatedStatus() throws Exception { // appImpl8 is not in the RM HadoopArchiveLogs hal = new HadoopArchiveLogs(conf); - Assertions.assertEquals(0, hal.eligibleApplications.size()); + assertEquals(0, hal.eligibleApplications.size()); hal.eligibleApplications.add( new HadoopArchiveLogs.AppInfo(appImpl1.getApplicationId().toString(), USER)); @@ -238,12 +242,12 @@ public void testFilterAppsByAggregatedStatus() throws Exception { new HadoopArchiveLogs.AppInfo(appImpl8.getApplicationId().toString(), USER); hal.eligibleApplications.add(app8); - Assertions.assertEquals(8, hal.eligibleApplications.size()); + assertEquals(8, hal.eligibleApplications.size()); hal.filterAppsByAggregatedStatus(); - Assertions.assertEquals(3, hal.eligibleApplications.size()); - Assertions.assertTrue(hal.eligibleApplications.contains(app4)); - Assertions.assertTrue(hal.eligibleApplications.contains(app7)); - Assertions.assertTrue(hal.eligibleApplications.contains(app8)); + assertEquals(3, hal.eligibleApplications.size()); + assertTrue(hal.eligibleApplications.contains(app4)); + assertTrue(hal.eligibleApplications.contains(app7)); + assertTrue(hal.eligibleApplications.contains(app8)); } } @@ -281,59 +285,59 @@ private void _testGenerateScript(boolean proxy) throws Exception { File localScript = new File("target", "script.sh"); localScript.delete(); - Assertions.assertFalse(localScript.exists()); + assertFalse(localScript.exists()); hal.generateScript(localScript); - Assertions.assertTrue(localScript.exists()); + assertTrue(localScript.exists()); String script = IOUtils.toString(localScript.toURI(), StandardCharsets.UTF_8); String[] lines = script.split("\n"); - Assertions.assertEquals(22, lines.length); - Assertions.assertEquals("#!/bin/bash", lines[0]); - Assertions.assertEquals("set -e", lines[1]); - Assertions.assertEquals("set -x", lines[2]); - Assertions.assertEquals("if [ \"$YARN_SHELL_ID\" == \"1\" ]; then", lines[3]); + assertEquals(22, lines.length); + assertEquals("#!/bin/bash", lines[0]); + assertEquals("set -e", lines[1]); + assertEquals("set -x", lines[2]); + assertEquals("if [ \"$YARN_SHELL_ID\" == \"1\" ]; then", lines[3]); boolean oneBefore = true; if (lines[4].contains(app1.toString())) { - Assertions.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[4]); - Assertions.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[10]); + assertEquals("\tappId=\"" + app1.toString() + "\"", lines[4]); + assertEquals("\tappId=\"" + app2.toString() + "\"", lines[10]); } else { oneBefore = false; - Assertions.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[4]); - Assertions.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[10]); + assertEquals("\tappId=\"" + app2.toString() + "\"", lines[4]); + assertEquals("\tappId=\"" + app1.toString() + "\"", lines[10]); } - Assertions.assertEquals("\tuser=\"" + USER + "\"", lines[5]); - Assertions.assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir.toString() + assertEquals("\tuser=\"" + USER + "\"", lines[5]); + assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir.toString() : workingDir2.toString()) + "\"", lines[6]); - Assertions.assertEquals("\tremoteRootLogDir=\"" + (oneBefore + assertEquals("\tremoteRootLogDir=\"" + (oneBefore ? remoteRootLogDir.toString() : remoteRootLogDir2.toString()) + "\"", lines[7]); - Assertions.assertEquals("\tsuffix=\"" + (oneBefore ? suffix : suffix2) + assertEquals("\tsuffix=\"" + (oneBefore ? suffix : suffix2) + "\"", lines[8]); - Assertions.assertEquals("elif [ \"$YARN_SHELL_ID\" == \"2\" ]; then", + assertEquals("elif [ \"$YARN_SHELL_ID\" == \"2\" ]; then", lines[9]); - Assertions.assertEquals("\tuser=\"" + USER + "\"", lines[11]); - Assertions.assertEquals("\tworkingDir=\"" + (oneBefore + assertEquals("\tuser=\"" + USER + "\"", lines[11]); + assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir2.toString() : workingDir.toString()) + "\"", lines[12]); - Assertions.assertEquals("\tremoteRootLogDir=\"" + (oneBefore + assertEquals("\tremoteRootLogDir=\"" + (oneBefore ? remoteRootLogDir2.toString() : remoteRootLogDir.toString()) + "\"", lines[13]); - Assertions.assertEquals("\tsuffix=\"" + (oneBefore ? suffix2 : suffix) + assertEquals("\tsuffix=\"" + (oneBefore ? suffix2 : suffix) + "\"", lines[14]); - Assertions.assertEquals("else", lines[15]); - Assertions.assertEquals("\techo \"Unknown Mapping!\"", lines[16]); - Assertions.assertEquals("\texit 1", lines[17]); - Assertions.assertEquals("fi", lines[18]); - Assertions.assertEquals("export HADOOP_CLIENT_OPTS=\"-Xmx1024m\"", lines[19]); - Assertions.assertTrue(lines[20].startsWith("export HADOOP_CLASSPATH=")); + assertEquals("else", lines[15]); + assertEquals("\techo \"Unknown Mapping!\"", lines[16]); + assertEquals("\texit 1", lines[17]); + assertEquals("fi", lines[18]); + assertEquals("export HADOOP_CLIENT_OPTS=\"-Xmx1024m\"", lines[19]); + assertTrue(lines[20].startsWith("export HADOOP_CLASSPATH=")); if (proxy) { - Assertions.assertEquals( + assertEquals( "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." + "HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " + "-workingDir \"$workingDir\" -remoteRootLogDir " + "\"$remoteRootLogDir\" -suffix \"$suffix\"", lines[21]); } else { - Assertions.assertEquals( + assertEquals( "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." + "HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " + "-workingDir \"$workingDir\" -remoteRootLogDir " + @@ -359,7 +363,7 @@ public void testStatuses() throws Exception { statuses[4] = LogAggregationStatus.SUCCEEDED; statuses[5] = LogAggregationStatus.FAILED; statuses[6] = LogAggregationStatus.TIME_OUT; - Assertions.assertArrayEquals(statuses, LogAggregationStatus.values()); + assertArrayEquals(statuses, LogAggregationStatus.values()); } @Test @@ -370,27 +374,27 @@ public void testPrepareWorkingDir() throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path workingDir = new Path("target", "testPrepareWorkingDir"); fs.delete(workingDir, true); - Assertions.assertFalse(fs.exists(workingDir)); + assertFalse(fs.exists(workingDir)); // -force is false and the dir doesn't exist so it will create one hal.force = false; boolean dirPrepared = hal.prepareWorkingDir(fs, workingDir); - Assertions.assertTrue(dirPrepared); - Assertions.assertTrue(fs.exists(workingDir)); - Assertions.assertEquals( + assertTrue(dirPrepared); + assertTrue(fs.exists(workingDir)); + assertEquals( new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, !Shell.WINDOWS), fs.getFileStatus(workingDir).getPermission()); // Throw a file in the dir Path dummyFile = new Path(workingDir, "dummy.txt"); fs.createNewFile(dummyFile); - Assertions.assertTrue(fs.exists(dummyFile)); + assertTrue(fs.exists(dummyFile)); // -force is false and the dir exists, so nothing will happen and the dummy // still exists dirPrepared = hal.prepareWorkingDir(fs, workingDir); - Assertions.assertFalse(dirPrepared); - Assertions.assertTrue(fs.exists(workingDir)); - Assertions.assertTrue(fs.exists(dummyFile)); - Assertions.assertEquals( + assertFalse(dirPrepared); + assertTrue(fs.exists(workingDir)); + assertTrue(fs.exists(dummyFile)); + assertEquals( new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, !Shell.WINDOWS), fs.getFileStatus(workingDir).getPermission()); @@ -398,13 +402,13 @@ public void testPrepareWorkingDir() throws Exception { // won't exist anymore hal.force = true; dirPrepared = hal.prepareWorkingDir(fs, workingDir); - Assertions.assertTrue(dirPrepared); - Assertions.assertTrue(fs.exists(workingDir)); - Assertions.assertEquals( + assertTrue(dirPrepared); + assertTrue(fs.exists(workingDir)); + assertEquals( new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, !Shell.WINDOWS), fs.getFileStatus(workingDir).getPermission()); - Assertions.assertFalse(fs.exists(dummyFile)); + assertFalse(fs.exists(dummyFile)); } private static void createFile(FileSystem fs, Path p, long sizeMultiple) @@ -420,7 +424,7 @@ private static void createFile(FileSystem fs, Path p, long sizeMultiple) out.close(); } } - Assertions.assertTrue(fs.exists(p)); + assertTrue(fs.exists(p)); } private static RMApp createRMApp(int id, Configuration conf, RMContext rmContext, diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java index 59b7bcbfff29d..5cc3972d51771 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java @@ -33,11 +33,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.Rule; import org.junit.jupiter.api.Test; -import org.junit.rules.Timeout; +import org.junit.jupiter.api.Timeout; import org.mockito.Mockito; import java.io.IOException; @@ -45,9 +43,12 @@ import java.util.Comparator; import java.util.Random; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +@Timeout(50) public class TestHadoopArchiveLogsRunner { private static final int FILE_SIZE_INCREMENT = 4096; @@ -68,9 +69,6 @@ public class TestHadoopArchiveLogsRunner { private Path remoteRootLogDir; private String suffix; - @Rule - public Timeout globalTimeout = new Timeout(50000); - @BeforeEach public void setup() throws Exception { yarnCluster = new MiniYARNCluster( @@ -100,7 +98,7 @@ public void setup() throws Exception { createFile(fs, new Path(app1Path, "log" + (i + 1)), FILE_SIZES[i]); } FileStatus[] app1Files = fs.listStatus(app1Path); - Assertions.assertEquals(FILE_COUNT, app1Files.length); + assertEquals(FILE_COUNT, app1Files.length); } @AfterEach @@ -124,12 +122,12 @@ public void testHadoopArchiveLogs() throws Exception { fs = FileSystem.get(conf); FileStatus[] app1Files = fs.listStatus(app1Path); - Assertions.assertEquals(1, app1Files.length); + assertEquals(1, app1Files.length); FileStatus harFile = app1Files[0]; - Assertions.assertEquals(app1.toString() + ".har", harFile.getPath().getName()); + assertEquals(app1.toString() + ".har", harFile.getPath().getName()); Path harPath = new Path("har:///" + harFile.getPath().toUri().getRawPath()); FileStatus[] harLogs = HarFs.get(harPath.toUri(), conf).listStatus(harPath); - Assertions.assertEquals(FILE_COUNT, harLogs.length); + assertEquals(FILE_COUNT, harLogs.length); Arrays.sort(harLogs, new Comparator() { @Override public int compare(FileStatus o1, FileStatus o2) { @@ -138,15 +136,15 @@ public int compare(FileStatus o1, FileStatus o2) { }); for (int i = 0; i < FILE_COUNT; i++) { FileStatus harLog = harLogs[i]; - Assertions.assertEquals("log" + (i + 1), harLog.getPath().getName()); - Assertions.assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, harLog.getLen()); - Assertions.assertEquals( + assertEquals("log" + (i + 1), harLog.getPath().getName()); + assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, harLog.getLen()); + assertEquals( new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE), harLog.getPermission()); - Assertions.assertEquals(System.getProperty("user.name"), + assertEquals(System.getProperty("user.name"), harLog.getOwner()); } - Assertions.assertEquals(0, fs.listStatus(workingDir).length); + assertEquals(0, fs.listStatus(workingDir).length); } @Test @@ -162,7 +160,7 @@ public void testHadoopArchiveLogsWithArchiveError() throws Exception { FileStatus[] app1Files = fs.listStatus(app1Path); assertEquals(FILE_COUNT, app1Files.length); for (int i = 0; i < FILE_COUNT; i++) { - Assertions.assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, + assertEquals(FILE_SIZES[i] * FILE_SIZE_INCREMENT, app1Files[i].getLen()); } }