From 3f9d8f46f9e0bdf75343e595aff65c9a2f289c3e Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Thu, 26 Sep 2019 15:38:06 +0530 Subject: [PATCH 1/7] HDFS-14869 Copy renamed files which are not excluded by filter --- .../java/org/apache/hadoop/tools/DistCp.java | 2 +- .../org/apache/hadoop/tools/DistCpSync.java | 33 +++++-- .../apache/hadoop/tools/TestDistCpSync.java | 87 +++++++++++++++++++ 3 files changed, 112 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index a1a2075831b6f..c36335afc161c 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -84,7 +84,7 @@ private void prepareFileListing(Job job) throws Exception { if (context.shouldUseSnapshotDiff()) { // When "-diff" or "-rdiff" is passed, do sync() first, then // create copyListing based on snapshot diff. - DistCpSync distCpSync = new DistCpSync(context, getConf()); + DistCpSync distCpSync = new DistCpSync(context, job.getConfiguration()); if (distCpSync.sync()) { createInputFileListingWithDiff(job, distCpSync); } else { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index a78320b05bf5e..fb8b4a42935e9 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -57,10 +57,13 @@ class DistCpSync { // private EnumMap> diffMap; private DiffInfo[] renameDiffs; + private CopyFilter copyFilter; DistCpSync(DistCpContext context, Configuration conf) { this.context = context; this.conf = conf; + this.copyFilter = CopyFilter.getCopyFilter(conf); + this.copyFilter.initialize(); } private boolean isRdiff() { @@ -213,18 +216,30 @@ private boolean getAllDiffs() throws IOException { } SnapshotDiffReport.DiffType dt = entry.getType(); List list = diffMap.get(dt); + final Path source = + new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); + final Path relativeSource = new Path(Path.SEPARATOR + source); if (dt == SnapshotDiffReport.DiffType.MODIFY || - dt == SnapshotDiffReport.DiffType.CREATE || - dt == SnapshotDiffReport.DiffType.DELETE) { - final Path source = - new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); - list.add(new DiffInfo(source, null, dt)); + dt == SnapshotDiffReport.DiffType.CREATE || + dt == SnapshotDiffReport.DiffType.DELETE) { + if (copyFilter.shouldCopy(relativeSource)) { + list.add(new DiffInfo(source, null, dt)); + } } else if (dt == SnapshotDiffReport.DiffType.RENAME) { - final Path source = - new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); final Path target = - new Path(DFSUtilClient.bytes2String(entry.getTargetPath())); - list.add(new DiffInfo(source, target, dt)); + new Path(DFSUtilClient.bytes2String(entry.getTargetPath())); + final Path relativeTarget = new Path(Path.SEPARATOR + target); + if (copyFilter.shouldCopy(relativeSource)) { + if (copyFilter.shouldCopy(relativeTarget)) { + list.add(new DiffInfo(source, target, dt)); + } else { + list = diffMap.get(SnapshotDiffReport.DiffType.DELETE); + list.add(new DiffInfo(source, target, SnapshotDiffReport.DiffType.DELETE)); + } + } else if (copyFilter.shouldCopy(relativeTarget)) { + list = diffMap.get(SnapshotDiffReport.DiffType.CREATE); + list.add(new DiffInfo(target, null, SnapshotDiffReport.DiffType.CREATE)); + } } } return true; diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 717b2f08436c1..e138ba1bcad3e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -39,6 +39,13 @@ import org.junit.Before; import org.junit.Test; + +import java.io.IOException; +import java.io.FileWriter; +import java.io.File; +import java.io.BufferedWriter; +import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -747,4 +754,84 @@ public void testSyncSnapshotTimeStampChecking() throws Exception { } Assert.assertTrue(threwException); } + + private void initData10(Path dir) throws Exception { + final Path staging = new Path(dir, ".staging"); + final Path staging_f1 = new Path(staging, "f1"); + final Path data = new Path(dir, "data"); + final Path data_f1 = new Path(data, "f1"); + + DFSTestUtil.createFile(dfs, staging_f1, BLOCK_SIZE, DATA_NUM, 0L); + DFSTestUtil.createFile(dfs, data_f1, BLOCK_SIZE, DATA_NUM, 0L); + } + + private void changeData10(Path dir) throws Exception { + final Path staging = new Path(dir, ".staging"); + final Path prod = new Path(dir, "prod"); + dfs.rename(staging, prod); + } + + private void generateFilterFile(String directory, String fileName){ + File theDir = new File(directory); + boolean threwException = false; + if (!theDir.exists()) { + theDir.mkdir(); + } + String str = ".*\\.staging.*"; + BufferedWriter writer = null; + try { + writer = new BufferedWriter(new FileWriter(directory + "/" + fileName)); + writer.write(str); + writer.close(); + } catch (IOException e) { + threwException = true; + } + Assert.assertFalse(threwException); + } + + private void deleteFilterFile(String directory, String fileName) { + File theDir = new File(directory); + if (!theDir.exists()) { + return; + } + File file = new File(directory + "/" + fileName); + if (file.exists()) { + file.delete(); + } + theDir.delete(); + } + + @Test + public void testSync10() throws Exception { + try { + Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); + initData10(sourcePath); + dfs.allowSnapshot(sourcePath); + dfs.createSnapshot(sourcePath, "s1"); + generateFilterFile("/tmp", "filters.txt"); + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new ArrayList<>(Arrays.asList(sourcePath)), + target) + .withFiltersFile("/tmp/filters.txt") + .withSyncFolder(true); + new DistCp(conf, builder.build()).execute(); + + dfs.allowSnapshot(target); + dfs.createSnapshot(target, "s1"); + changeData10(sourcePath); + dfs.createSnapshot(sourcePath, "s2"); + + final DistCpOptions.Builder diffBuilder = new DistCpOptions.Builder( + new ArrayList<>(Arrays.asList(sourcePath)), + target) + .withUseDiff("s1", "s2") + .withFiltersFile("/tmp/filters.txt") + .withSyncFolder(true); + new DistCp(conf, diffBuilder.build()).execute(); + verifyCopy(dfs.getFileStatus(sourcePath), + dfs.getFileStatus(target), false); + } finally { + deleteFilterFile("/tmp", "filters.txt"); + } + } } From a601ad82af7b017e41d79c7902b8704ab373ffdb Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Tue, 22 Oct 2019 14:48:42 +0530 Subject: [PATCH 2/7] Resolve review comments --- .../org/apache/hadoop/tools/DistCpSync.java | 4 +- .../apache/hadoop/tools/TestDistCpSync.java | 40 ++++++------------- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index fb8b4a42935e9..a8927d7c8f9d3 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -220,8 +220,8 @@ private boolean getAllDiffs() throws IOException { new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); final Path relativeSource = new Path(Path.SEPARATOR + source); if (dt == SnapshotDiffReport.DiffType.MODIFY || - dt == SnapshotDiffReport.DiffType.CREATE || - dt == SnapshotDiffReport.DiffType.DELETE) { + dt == SnapshotDiffReport.DiffType.CREATE || + dt == SnapshotDiffReport.DiffType.DELETE) { if (copyFilter.shouldCopy(relativeSource)) { list.add(new DiffInfo(source, null, dt)); } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index e138ba1bcad3e..b745223f4f5c3 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -42,8 +42,8 @@ import java.io.IOException; import java.io.FileWriter; -import java.io.File; import java.io.BufferedWriter; +import java.nio.file.Files; import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; @@ -771,48 +771,32 @@ private void changeData10(Path dir) throws Exception { dfs.rename(staging, prod); } - private void generateFilterFile(String directory, String fileName){ - File theDir = new File(directory); - boolean threwException = false; - if (!theDir.exists()) { - theDir.mkdir(); - } + private java.nio.file.Path generateFilterFile(String fileName) throws IOException { + java.nio.file.Path tmpFile = Files.createTempFile(fileName, "txt"); String str = ".*\\.staging.*"; - BufferedWriter writer = null; - try { - writer = new BufferedWriter(new FileWriter(directory + "/" + fileName)); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(tmpFile.toString()))) { writer.write(str); - writer.close(); - } catch (IOException e) { - threwException = true; } - Assert.assertFalse(threwException); + return tmpFile; } - private void deleteFilterFile(String directory, String fileName) { - File theDir = new File(directory); - if (!theDir.exists()) { - return; - } - File file = new File(directory + "/" + fileName); - if (file.exists()) { - file.delete(); - } - theDir.delete(); + private void deleteFilterFile(java.nio.file.Path filePath) throws IOException { + Files.delete(filePath); } @Test public void testSync10() throws Exception { + java.nio.file.Path filterFile = null; try { Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); initData10(sourcePath); dfs.allowSnapshot(sourcePath); dfs.createSnapshot(sourcePath, "s1"); - generateFilterFile("/tmp", "filters.txt"); + filterFile = generateFilterFile("filters"); final DistCpOptions.Builder builder = new DistCpOptions.Builder( new ArrayList<>(Arrays.asList(sourcePath)), target) - .withFiltersFile("/tmp/filters.txt") + .withFiltersFile(filterFile.toString()) .withSyncFolder(true); new DistCp(conf, builder.build()).execute(); @@ -825,13 +809,13 @@ public void testSync10() throws Exception { new ArrayList<>(Arrays.asList(sourcePath)), target) .withUseDiff("s1", "s2") - .withFiltersFile("/tmp/filters.txt") + .withFiltersFile(filterFile.toString()) .withSyncFolder(true); new DistCp(conf, diffBuilder.build()).execute(); verifyCopy(dfs.getFileStatus(sourcePath), dfs.getFileStatus(target), false); } finally { - deleteFilterFile("/tmp", "filters.txt"); + deleteFilterFile(filterFile); } } } From 713b08c1ac9b9f90f6da9fa7ba1d25f567c9b6db Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Tue, 22 Oct 2019 14:48:42 +0530 Subject: [PATCH 3/7] Resolve review comments --- .../org/apache/hadoop/tools/DistCpSync.java | 4 +- .../apache/hadoop/tools/TestDistCpSync.java | 125 ++++++++++++++---- 2 files changed, 99 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index fb8b4a42935e9..a8927d7c8f9d3 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -220,8 +220,8 @@ private boolean getAllDiffs() throws IOException { new Path(DFSUtilClient.bytes2String(entry.getSourcePath())); final Path relativeSource = new Path(Path.SEPARATOR + source); if (dt == SnapshotDiffReport.DiffType.MODIFY || - dt == SnapshotDiffReport.DiffType.CREATE || - dt == SnapshotDiffReport.DiffType.DELETE) { + dt == SnapshotDiffReport.DiffType.CREATE || + dt == SnapshotDiffReport.DiffType.DELETE) { if (copyFilter.shouldCopy(relativeSource)) { list.add(new DiffInfo(source, null, dt)); } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index e138ba1bcad3e..7affdea4a1084 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -42,8 +42,8 @@ import java.io.IOException; import java.io.FileWriter; -import java.io.File; import java.io.BufferedWriter; +import java.nio.file.Files; import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; @@ -771,50 +771,36 @@ private void changeData10(Path dir) throws Exception { dfs.rename(staging, prod); } - private void generateFilterFile(String directory, String fileName){ - File theDir = new File(directory); - boolean threwException = false; - if (!theDir.exists()) { - theDir.mkdir(); - } + private java.nio.file.Path generateFilterFile(String fileName) throws IOException { + java.nio.file.Path tmpFile = Files.createTempFile(fileName, "txt"); String str = ".*\\.staging.*"; - BufferedWriter writer = null; - try { - writer = new BufferedWriter(new FileWriter(directory + "/" + fileName)); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(tmpFile.toString()))) { writer.write(str); - writer.close(); - } catch (IOException e) { - threwException = true; } - Assert.assertFalse(threwException); + return tmpFile; } - private void deleteFilterFile(String directory, String fileName) { - File theDir = new File(directory); - if (!theDir.exists()) { - return; - } - File file = new File(directory + "/" + fileName); - if (file.exists()) { - file.delete(); - } - theDir.delete(); + private void deleteFilterFile(java.nio.file.Path filePath) throws IOException { + Files.delete(filePath); } @Test public void testSync10() throws Exception { + java.nio.file.Path filterFile = null; try { Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); initData10(sourcePath); dfs.allowSnapshot(sourcePath); dfs.createSnapshot(sourcePath, "s1"); - generateFilterFile("/tmp", "filters.txt"); + filterFile = generateFilterFile("filters"); final DistCpOptions.Builder builder = new DistCpOptions.Builder( new ArrayList<>(Arrays.asList(sourcePath)), target) - .withFiltersFile("/tmp/filters.txt") + .withFiltersFile(filterFile.toString()) .withSyncFolder(true); new DistCp(conf, builder.build()).execute(); + verifySync(dfs.getFileStatus(sourcePath), + dfs.getFileStatus(target), false, ".staging"); dfs.allowSnapshot(target); dfs.createSnapshot(target, "s1"); @@ -825,13 +811,96 @@ public void testSync10() throws Exception { new ArrayList<>(Arrays.asList(sourcePath)), target) .withUseDiff("s1", "s2") - .withFiltersFile("/tmp/filters.txt") + .withFiltersFile(filterFile.toString()) .withSyncFolder(true); new DistCp(conf, diffBuilder.build()).execute(); verifyCopy(dfs.getFileStatus(sourcePath), dfs.getFileStatus(target), false); } finally { - deleteFilterFile("/tmp", "filters.txt"); + deleteFilterFile(filterFile); + } + } + + private void initData11(Path dir) throws Exception { + final Path staging = new Path(dir, "prod"); + final Path staging_f1 = new Path(staging, "f1"); + final Path data = new Path(dir, "data"); + final Path data_f1 = new Path(data, "f1"); + + DFSTestUtil.createFile(dfs, staging_f1, BLOCK_SIZE, DATA_NUM, 0L); + DFSTestUtil.createFile(dfs, data_f1, BLOCK_SIZE, DATA_NUM, 0L); + } + + private void changeData11(Path dir) throws Exception { + final Path staging = new Path(dir, "prod"); + final Path prod = new Path(dir, ".staging"); + dfs.rename(staging, prod); + } + + private void verifySync(FileStatus s, FileStatus t, boolean compareName, String deletedName) + throws Exception { + Assert.assertEquals(s.isDirectory(), t.isDirectory()); + if (compareName) { + Assert.assertEquals(s.getPath().getName(), t.getPath().getName()); + } + if (!s.isDirectory()) { + // verify the file content is the same + byte[] sbytes = DFSTestUtil.readFileBuffer(dfs, s.getPath()); + byte[] tbytes = DFSTestUtil.readFileBuffer(dfs, t.getPath()); + Assert.assertArrayEquals(sbytes, tbytes); + } else { + FileStatus[] slist = dfs.listStatus(s.getPath()); + FileStatus[] tlist = dfs.listStatus(t.getPath()); + int minFiles = tlist.length; + if (slist.length < tlist.length) { + minFiles = slist.length; + } + for (int i = 0; i < minFiles; i++) { + if (slist[i].getPath().getName().contains(deletedName)) { + if (tlist[i].getPath().getName().contains(deletedName)) { + throw new Exception("Target file is not synced as per exclusion filter"); + } + continue; + } + verifySync(slist[i], tlist[i], true, deletedName); + } + } + } + + @Test + public void testSync11() throws Exception { + java.nio.file.Path filterFile = null; + try { + Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); + initData11(sourcePath); + dfs.allowSnapshot(sourcePath); + dfs.createSnapshot(sourcePath, "s1"); + filterFile = generateFilterFile("filters"); + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new ArrayList<>(Arrays.asList(sourcePath)), + target) + .withFiltersFile(filterFile.toString()) + .withSyncFolder(true); + new DistCp(conf, builder.build()).execute(); + verifyCopy(dfs.getFileStatus(sourcePath), + dfs.getFileStatus(target), false); + + dfs.allowSnapshot(target); + dfs.createSnapshot(target, "s1"); + changeData11(sourcePath); + dfs.createSnapshot(sourcePath, "s2"); + + final DistCpOptions.Builder diffBuilder = new DistCpOptions.Builder( + new ArrayList<>(Arrays.asList(sourcePath)), + target) + .withUseDiff("s1", "s2") + .withFiltersFile(filterFile.toString()) + .withSyncFolder(true); + new DistCp(conf, diffBuilder.build()).execute(); + verifySync(dfs.getFileStatus(sourcePath), + dfs.getFileStatus(target), false, ".staging"); + } finally { + deleteFilterFile(filterFile); } } } From 1e6a27aca487cb2effec3d1d34f14015eadbb2aa Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Tue, 19 Nov 2019 02:11:23 -0800 Subject: [PATCH 4/7] Empty commit --- .../src/test/java/org/apache/hadoop/tools/TestDistCpSync.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 2545b1ede5f2e..90b947eb42ce5 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -869,6 +869,7 @@ private void verifySync(FileStatus s, FileStatus t, boolean compareName, String @Test public void testSync11() throws Exception { + java.nio.file.Path filterFile = null; try { Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); From 5e9ddc84590c613e1e92c2880e1223a48cc2309d Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Thu, 5 Dec 2019 11:22:01 +0530 Subject: [PATCH 5/7] Fix whitespace --- .../src/test/java/org/apache/hadoop/tools/TestDistCpSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 90b947eb42ce5..92c6b52735fd2 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -869,7 +869,7 @@ private void verifySync(FileStatus s, FileStatus t, boolean compareName, String @Test public void testSync11() throws Exception { - + java.nio.file.Path filterFile = null; try { Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); From 445243e3d4183db0ae7d1e3aa3ff53625c009c23 Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Thu, 5 Dec 2019 11:28:40 +0530 Subject: [PATCH 6/7] Fix whitespace --- .../src/test/java/org/apache/hadoop/tools/TestDistCpSync.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 92c6b52735fd2..2545b1ede5f2e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -869,7 +869,6 @@ private void verifySync(FileStatus s, FileStatus t, boolean compareName, String @Test public void testSync11() throws Exception { - java.nio.file.Path filterFile = null; try { Path sourcePath = new Path(dfs.getWorkingDirectory(), "source"); From 4343c3874f49e34a9585aabee6947473c4d55ccc Mon Sep 17 00:00:00 2001 From: Aasha Medhi Date: Thu, 5 Dec 2019 23:15:12 +0530 Subject: [PATCH 7/7] Fix Checkstyle --- .../org/apache/hadoop/tools/DistCpSync.java | 6 ++-- .../apache/hadoop/tools/TestDistCpSync.java | 28 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index a8927d7c8f9d3..35ef3e4ab77e7 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -234,11 +234,13 @@ private boolean getAllDiffs() throws IOException { list.add(new DiffInfo(source, target, dt)); } else { list = diffMap.get(SnapshotDiffReport.DiffType.DELETE); - list.add(new DiffInfo(source, target, SnapshotDiffReport.DiffType.DELETE)); + list.add(new DiffInfo(source, target, + SnapshotDiffReport.DiffType.DELETE)); } } else if (copyFilter.shouldCopy(relativeTarget)) { list = diffMap.get(SnapshotDiffReport.DiffType.CREATE); - list.add(new DiffInfo(target, null, SnapshotDiffReport.DiffType.CREATE)); + list.add(new DiffInfo(target, null, + SnapshotDiffReport.DiffType.CREATE)); } } } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 2545b1ede5f2e..d6bbc25fdc7a8 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -757,12 +757,12 @@ public void testSyncSnapshotTimeStampChecking() throws Exception { private void initData10(Path dir) throws Exception { final Path staging = new Path(dir, ".staging"); - final Path staging_f1 = new Path(staging, "f1"); + final Path stagingF1 = new Path(staging, "f1"); final Path data = new Path(dir, "data"); - final Path data_f1 = new Path(data, "f1"); + final Path dataF1 = new Path(data, "f1"); - DFSTestUtil.createFile(dfs, staging_f1, BLOCK_SIZE, DATA_NUM, 0L); - DFSTestUtil.createFile(dfs, data_f1, BLOCK_SIZE, DATA_NUM, 0L); + DFSTestUtil.createFile(dfs, stagingF1, BLOCK_SIZE, DATA_NUM, 0L); + DFSTestUtil.createFile(dfs, dataF1, BLOCK_SIZE, DATA_NUM, 0L); } private void changeData10(Path dir) throws Exception { @@ -771,16 +771,19 @@ private void changeData10(Path dir) throws Exception { dfs.rename(staging, prod); } - private java.nio.file.Path generateFilterFile(String fileName) throws IOException { + private java.nio.file.Path generateFilterFile(String fileName) + throws IOException { java.nio.file.Path tmpFile = Files.createTempFile(fileName, "txt"); String str = ".*\\.staging.*"; - try (BufferedWriter writer = new BufferedWriter(new FileWriter(tmpFile.toString()))) { + try (BufferedWriter writer = new BufferedWriter( + new FileWriter(tmpFile.toString()))) { writer.write(str); } return tmpFile; } - private void deleteFilterFile(java.nio.file.Path filePath) throws IOException { + private void deleteFilterFile(java.nio.file.Path filePath) + throws IOException { Files.delete(filePath); } @@ -823,12 +826,12 @@ public void testSync10() throws Exception { private void initData11(Path dir) throws Exception { final Path staging = new Path(dir, "prod"); - final Path staging_f1 = new Path(staging, "f1"); + final Path stagingF1 = new Path(staging, "f1"); final Path data = new Path(dir, "data"); - final Path data_f1 = new Path(data, "f1"); + final Path dataF1 = new Path(data, "f1"); - DFSTestUtil.createFile(dfs, staging_f1, BLOCK_SIZE, DATA_NUM, 0L); - DFSTestUtil.createFile(dfs, data_f1, BLOCK_SIZE, DATA_NUM, 0L); + DFSTestUtil.createFile(dfs, stagingF1, BLOCK_SIZE, DATA_NUM, 0L); + DFSTestUtil.createFile(dfs, dataF1, BLOCK_SIZE, DATA_NUM, 0L); } private void changeData11(Path dir) throws Exception { @@ -837,7 +840,8 @@ private void changeData11(Path dir) throws Exception { dfs.rename(staging, prod); } - private void verifySync(FileStatus s, FileStatus t, boolean compareName, String deletedName) + private void verifySync(FileStatus s, FileStatus t, boolean compareName, + String deletedName) throws Exception { Assert.assertEquals(s.isDirectory(), t.isDirectory()); if (compareName) {