From 6ccd1c6cc29c70b999192dc130ddaf5bc0fe2ba6 Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Wed, 7 Feb 2018 14:01:22 +0100 Subject: [PATCH 1/7] HDFS-13118. SnapshotDiffReport should provide the INode type. --- .../impl/SnapshotDiffReportGenerator.java | 9 +- .../hdfs/protocol/SnapshotDiffReport.java | 29 +- .../protocol/SnapshotDiffReportListing.java | 34 +- .../hdfs/protocolPB/PBHelperClient.java | 59 +- .../hadoop/hdfs/web/JsonUtilClient.java | 6 +- .../src/main/proto/hdfs.proto | 3 + .../namenode/snapshot/SnapshotDiffInfo.java | 21 +- .../snapshot/SnapshotDiffListingInfo.java | 31 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 13 +- .../hadoop/hdfs/TestEncryptionZones.java | 9 +- .../snapshot/TestRenameWithSnapshots.java | 83 +- .../snapshot/TestSnapshotDiffReport.java | 826 +++++++++--------- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 16 +- 13 files changed, 644 insertions(+), 495 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java index 4dbe98858f5d5..327e0a5d9335f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java @@ -222,7 +222,8 @@ public SnapshotDiffReport generateReport() { generateReportList(); for (DiffReportListingEntry modified : mlist) { diffReportList.add( - new DiffReportEntry(DiffType.MODIFY, modified.getSourcePath(), null)); + new DiffReportEntry(modified.getINodeType().toSnapshotDiffReportINodeType(), + DiffType.MODIFY, modified.getSourcePath(), null)); if (modified.isReference() && dirDiffMap.get(modified.getDirId()) != null) { List subList = generateReport(modified); @@ -241,6 +242,7 @@ private List generateReport( RenameEntry entry = renameMap.get(created.getFileId()); if (entry == null || !entry.isRename()) { diffReportList.add(new DiffReportEntry( + created.getINodeType().toSnapshotDiffReportINodeType(), isFromEarlier ? DiffType.CREATE : DiffType.DELETE, created.getSourcePath())); } @@ -248,11 +250,14 @@ private List generateReport( for (DiffReportListingEntry deleted : list.getDeletedList()) { RenameEntry entry = renameMap.get(deleted.getFileId()); if (entry != null && entry.isRename()) { - diffReportList.add(new DiffReportEntry(DiffType.RENAME, + diffReportList.add(new DiffReportEntry( + deleted.getINodeType().toSnapshotDiffReportINodeType(), + DiffType.RENAME, isFromEarlier ? entry.getSourcePath() : entry.getTargetPath(), isFromEarlier ? entry.getTargetPath() : entry.getSourcePath())); } else { diffReportList.add(new DiffReportEntry( + deleted.getINodeType().toSnapshotDiffReportINodeType(), isFromEarlier ? DiffType.DELETE : DiffType.CREATE, deleted.getSourcePath())); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 7bc95c9350521..5438319ac4dec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -75,6 +75,16 @@ public static DiffType parseDiffType(String s){ } } + public enum INodeType { + FILE, + DIRECTORY, + SYMLINK; + + public static INodeType parseINodeType(String s) { + return INodeType.valueOf(s.toUpperCase()); + } + } + /** * Representing the full path and diff type of a file/directory where changes * have happened. @@ -82,6 +92,7 @@ public static DiffType parseDiffType(String s){ public static class DiffReportEntry { /** The type of the difference. */ private final DiffType type; + private final INodeType iNodeType; /** * The relative path (related to the snapshot root) of 1) the file/directory * where changes have happened, or 2) the source file/dir of a rename op. @@ -89,22 +100,24 @@ public static class DiffReportEntry { private final byte[] sourcePath; private final byte[] targetPath; - public DiffReportEntry(DiffType type, byte[] sourcePath) { - this(type, sourcePath, null); + public DiffReportEntry(INodeType iNodeType, DiffType type, byte[] sourcePath) { + this(iNodeType, type, sourcePath, null); } - public DiffReportEntry(DiffType type, byte[][] sourcePathComponents) { - this(type, sourcePathComponents, null); + public DiffReportEntry(INodeType iNodeType, DiffType type, byte[][] sourcePathComponents) { + this(iNodeType, type, sourcePathComponents, null); } - public DiffReportEntry(DiffType type, byte[] sourcePath, byte[] targetPath) { + public DiffReportEntry(INodeType iNodeType, DiffType type, byte[] sourcePath, byte[] targetPath) { + this.iNodeType = iNodeType; this.type = type; this.sourcePath = sourcePath; this.targetPath = targetPath; } - public DiffReportEntry(DiffType type, byte[][] sourcePathComponents, + public DiffReportEntry(INodeType iNodeType, DiffType type, byte[][] sourcePathComponents, byte[][] targetPathComponents) { + this.iNodeType = iNodeType; this.type = type; this.sourcePath = DFSUtilClient.byteArray2bytes(sourcePathComponents); this.targetPath = targetPathComponents == null ? null : DFSUtilClient @@ -124,6 +137,10 @@ public DiffType getType() { return type; } + public INodeType getINodeType() { + return iNodeType; + } + static String getPathString(byte[] path) { String pathStr = DFSUtilClient.bytes2String(path); if (pathStr.isEmpty()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java index a0e35f6c2a71e..9a189ddddb46d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java @@ -37,6 +37,26 @@ public class SnapshotDiffReportListing { * have happened. */ public static class DiffReportListingEntry { + + public enum INodeType { + + FILE(SnapshotDiffReport.INodeType.FILE), + DIRECTORY(SnapshotDiffReport.INodeType.DIRECTORY), + SYMLINK(SnapshotDiffReport.INodeType.SYMLINK); + + private SnapshotDiffReport.INodeType snapshotDiffReportINodeType; + + INodeType(SnapshotDiffReport.INodeType snapshotDiffReportINodeType) { + this.snapshotDiffReportINodeType = snapshotDiffReportINodeType; + } + + public SnapshotDiffReport.INodeType toSnapshotDiffReportINodeType() { + return snapshotDiffReportINodeType; + } + } + + private final INodeType inodeType; + /** * The type of the difference. */ @@ -51,9 +71,10 @@ public static class DiffReportListingEntry { private final byte[][] sourcePath; private final byte[][] targetPath; - public DiffReportListingEntry(long dirId, long fileId, byte[][] sourcePath, - boolean isReference, byte[][] targetPath) { + public DiffReportListingEntry(INodeType inodeType, long dirId, long fileId, + byte[][] sourcePath, boolean isReference, byte[][] targetPath) { Preconditions.checkNotNull(sourcePath); + this.inodeType = inodeType; this.dirId = dirId; this.fileId = fileId; this.sourcePath = sourcePath; @@ -61,9 +82,10 @@ public DiffReportListingEntry(long dirId, long fileId, byte[][] sourcePath, this.targetPath = targetPath; } - public DiffReportListingEntry(long dirId, long fileId, byte[] sourcePath, - boolean isReference, byte[] targetpath) { + public DiffReportListingEntry(INodeType inodeType, long dirId, long fileId, + byte[] sourcePath, boolean isReference, byte[] targetpath) { Preconditions.checkNotNull(sourcePath); + this.inodeType = inodeType; this.dirId = dirId; this.fileId = fileId; this.sourcePath = DFSUtilClient.bytes2byteArray(sourcePath); @@ -72,6 +94,10 @@ public DiffReportListingEntry(long dirId, long fileId, byte[] sourcePath, targetpath == null ? null : DFSUtilClient.bytes2byteArray(targetpath); } + public INodeType getINodeType() { + return inodeType; + } + public long getDirId() { return dirId; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 691ac54ff29ee..3a3cbee7651a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1599,7 +1599,21 @@ public static DiffReportEntry convert(SnapshotDiffReportEntryProto entry) { } DiffType type = DiffType.getTypeFromLabel(entry .getModificationLabel()); - return type == null ? null : new DiffReportEntry(type, entry.getFullpath() + SnapshotDiffReport.INodeType inodeType = null; + if (entry.hasFileType()) { + switch (entry.getFileType()) { + case IS_FILE: + inodeType = SnapshotDiffReport.INodeType.FILE; + break; + case IS_DIR: + inodeType = SnapshotDiffReport.INodeType.DIRECTORY; + break; + case IS_SYMLINK: + inodeType = SnapshotDiffReport.INodeType.SYMLINK; + break; + } + } + return type == null ? null : new DiffReportEntry(inodeType, type, entry.getFullpath() .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath() .toByteArray() : null); } @@ -1649,14 +1663,26 @@ public static DiffReportListingEntry convert( if (entry == null) { return null; } + DiffReportListingEntry.INodeType inodeType = null; + switch(entry.getFileType()) { + case IS_FILE: + inodeType = DiffReportListingEntry.INodeType.FILE; + break; + case IS_DIR: + inodeType = DiffReportListingEntry.INodeType.DIRECTORY; + break; + case IS_SYMLINK: + inodeType = DiffReportListingEntry.INodeType.SYMLINK; + break; + } long dirId = entry.getDirId(); long fileId = entry.getFileId(); boolean isReference = entry.getIsReference(); byte[] sourceName = entry.getFullpath().toByteArray(); byte[] targetName = entry.hasTargetPath() ? entry.getTargetPath().toByteArray() : null; - return new DiffReportListingEntry(dirId, fileId, sourceName, isReference, - targetName); + return new DiffReportListingEntry(inodeType, dirId, fileId, sourceName, + isReference, targetName); } public static SnapshottableDirectoryStatus[] convert( @@ -2679,6 +2705,17 @@ public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto .newBuilder().setFullpath(sourcePath) .setModificationLabel(modification); + switch(entry.getINodeType()){ + case FILE: + builder.setFileType(FileType.IS_FILE); + break; + case DIRECTORY: + builder.setFileType(FileType.IS_DIR); + break; + case SYMLINK: + builder.setFileType(FileType.IS_SYMLINK); + break; + } if (entry.getType() == DiffType.RENAME) { ByteString targetPath = getByteString(entry.getTargetPath() == null ? @@ -2696,6 +2733,18 @@ public static SnapshotDiffReportListingEntryProto convert( ByteString sourcePath = getByteString( entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES : DFSUtilClient.byteArray2bytes(entry.getSourcePath())); + HdfsFileStatusProto.FileType fileType = null; + switch(entry.getINodeType()){ + case FILE: + fileType = FileType.IS_FILE; + break; + case DIRECTORY: + fileType = FileType.IS_DIR; + break; + case SYMLINK: + fileType = FileType.IS_SYMLINK; + break; + } long dirId = entry.getDirId(); long fileId = entry.getFileId(); boolean isReference = entry.isReference(); @@ -2703,7 +2752,9 @@ public static SnapshotDiffReportListingEntryProto convert( entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES : DFSUtilClient.byteArray2bytes(entry.getTargetPath())); SnapshotDiffReportListingEntryProto.Builder builder = - SnapshotDiffReportListingEntryProto.newBuilder().setFullpath(sourcePath) + SnapshotDiffReportListingEntryProto.newBuilder() + .setFileType(fileType) + .setFullpath(sourcePath) .setDirId(dirId).setFileId(fileId).setIsReference(isReference) .setTargetPath(targetPath); return builder.build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index f9b847c12160a..4e7bfabb59506 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -803,11 +803,15 @@ private static SnapshotDiffReport.DiffReportEntry toDiffReportEntry( if (json == null) { return null; } + SnapshotDiffReport.INodeType iNodeType = + SnapshotDiffReport.INodeType.parseINodeType( + (String) json.get("iNodeType")); SnapshotDiffReport.DiffType type = SnapshotDiffReport.DiffType.parseDiffType((String) json.get("type")); byte[] sourcePath = toByteArray((String) json.get("sourcePath")); byte[] targetPath = toByteArray((String) json.get("targetPath")); - return new SnapshotDiffReport.DiffReportEntry(type, sourcePath, targetPath); + return new SnapshotDiffReport.DiffReportEntry(iNodeType, type, sourcePath, + targetPath); } private static byte[] toByteArray(String str) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 58a3d598f4b37..bdd7f427bf94a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -547,6 +547,7 @@ message SnapshottableDirectoryListingProto { repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1; } + /** * Snapshot diff report entry */ @@ -554,6 +555,7 @@ message SnapshotDiffReportEntryProto { required bytes fullpath = 1; required string modificationLabel = 2; optional bytes targetPath = 3; + optional HdfsFileStatusProto.FileType fileType = 4; } /** @@ -576,6 +578,7 @@ message SnapshotDiffReportListingEntryProto { required bool isReference = 3; optional bytes targetPath = 4; optional uint64 fileId = 5; + optional HdfsFileStatusProto.FileType fileType = 6; } message SnapshotDiffReportCursorProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java index ab6f4158c8417..10a94339bcf2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeFile; @@ -238,7 +239,7 @@ public SnapshotDiffReport generateReport() { for (Map.Entry drEntry : diffMap.entrySet()) { INode node = drEntry.getKey(); byte[][] path = drEntry.getValue(); - diffReportList.add(new DiffReportEntry(DiffType.MODIFY, path, null)); + diffReportList.add(new DiffReportEntry(determineINodeType(node),DiffType.MODIFY, path, null)); if (node.isDirectory()) { List subList = generateReport(dirDiffMap.get(node), path, isFromEarlier(), renameMap); @@ -274,22 +275,34 @@ private List generateReport(ChildrenDiff dirDiff, RenameEntry entry = renameMap.get(cnode.getId()); if (entry == null || !entry.isRename()) { fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE + list.add(new DiffReportEntry(determineINodeType(cnode),fromEarlier ? DiffType.CREATE : DiffType.DELETE, fullPath)); } } for (INode dnode : dirDiff.getDeletedUnmodifiable()) { RenameEntry entry = renameMap.get(dnode.getId()); if (entry != null && entry.isRename()) { - list.add(new DiffReportEntry(DiffType.RENAME, + list.add(new DiffReportEntry(determineINodeType(dnode), DiffType.RENAME, fromEarlier ? entry.getSourcePath() : entry.getTargetPath(), fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); } else { fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); - list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE + list.add(new DiffReportEntry(determineINodeType(dnode), fromEarlier ? DiffType.DELETE : DiffType.CREATE, fullPath)); } } return list; } + + private static INodeType determineINodeType(INode node) { + if (node.isDirectory()) { + return INodeType.DIRECTORY; + } else if (node.isFile()) { + return INodeType.FILE; + } else if (node.isSymlink()) { + return INodeType.SYMLINK; + } else { + throw new IllegalArgumentException("This type of INode is not supported"); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index a7960700e6811..a033f0117ac80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -32,6 +32,9 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.util.ChunkedArrayList; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.*; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.FILE; + /** * A class describing the difference between snapshots of a snapshottable * directory where the difference is limited by dfs.snapshotDiff-report.limit. @@ -86,8 +89,8 @@ boolean addDirDiff(long dirId, byte[][] parent, ChildrenDiff diff) { final Snapshot laterSnapshot = getLater(); if (lastIndex == -1) { if (getTotalEntries() < maxEntries) { - modifiedList.add(new DiffReportListingEntry( - dirId, dirId, parent, true, null)); + modifiedList.add(new DiffReportListingEntry(DIRECTORY, dirId, dirId, + parent, true, null)); } else { setLastPath(parent); setLastIndex(-1); @@ -103,8 +106,8 @@ boolean addDirDiff(long dirId, byte[][] parent, ChildrenDiff diff) { if (getTotalEntries() < maxEntries) { INode created = iterator.next(); byte[][] path = newPath(parent, created.getLocalNameBytes()); - createdList.add(new DiffReportListingEntry(dirId, created.getId(), - path, created.isReference(), null)); + createdList.add(new DiffReportListingEntry(fromINode(created), dirId, + created.getId(), path, created.isReference(), null)); } else { setLastPath(parent); setLastIndex(iterator.nextIndex()); @@ -124,9 +127,12 @@ boolean addDirDiff(long dirId, byte[][] parent, ChildrenDiff diff) { final INode d = iterator.next(); byte[][] path = newPath(parent, d.getLocalNameBytes()); byte[][] target = findRenameTargetPath(d, laterSnapshot); + DiffReportListingEntry.INodeType inodeType = fromINode(d); final DiffReportListingEntry e = target != null ? - new DiffReportListingEntry(dirId, d.getId(), path, true, target) : - new DiffReportListingEntry(dirId, d.getId(), path, false, null); + new DiffReportListingEntry(inodeType, dirId, d.getId(), path, + true, target) : + new DiffReportListingEntry(inodeType, dirId, d.getId(), path, + false, null); deletedList.add(e); } else { setLastPath(parent); @@ -156,6 +162,17 @@ private static byte[][] newPath(byte[][] parent, byte[] name) { return fullPath; } + private static DiffReportListingEntry.INodeType fromINode(INode inode) { + if (inode.isFile()) { + return FILE; + } else if (inode.isDirectory()) { + return DIRECTORY; + } else if (inode.isSymlink()) { + return SYMLINK; + } + throw new IllegalArgumentException("Unable to convert unknown INode type"); + } + Snapshot getEarlier() { return isFromEarlier()? from: to; } @@ -175,7 +192,7 @@ public void setLastIndex(int idx) { boolean addFileDiff(INodeFile file, byte[][] relativePath) { if (getTotalEntries() < maxEntries) { - modifiedList.add(new DiffReportListingEntry(file.getId(), + modifiedList.add(new DiffReportListingEntry(FILE, file.getId(), file.getId(), relativePath,false, null)); } else { setLastPath(relativePath); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 36f2eb2d8050f..b716f7101093d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.DIRECTORY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; @@ -2480,17 +2481,25 @@ public static void verifySnapshotDiffReport(DistributedFileSystem fs, assertEquals(entries.length, inverseReport.getDiffList().size()); for (DiffReportEntry entry : entries) { + DiffReportEntry reportEntry = report.getDiffList().stream() + .filter(e -> e.equals(entry)) + .findFirst() + .orElseThrow(() -> + new AssertionError("DiffReportEntry not found: " + + entry.getType() + " " + entry.getSourcePath())); if (entry.getType() == DiffType.MODIFY) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains(entry)); } else if (entry.getType() == DiffType.DELETE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); + new DiffReportEntry(DIRECTORY, + DiffType.CREATE, entry.getSourcePath()))); } else if (entry.getType() == DiffType.CREATE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); + new DiffReportEntry(DIRECTORY, + DiffType.DELETE, entry.getSourcePath()))); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 78fcc1506dd25..94f5f01c90aa9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -112,6 +112,7 @@ import org.mockito.Mockito; import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.FILE; import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -1540,12 +1541,12 @@ public void testSnapshotDiffOnEncryptionZones() throws Exception { fsWrapper.delete(zoneFile, true); fs.createSnapshot(zone, "snap2"); verifyDiffReport(zone, "snap1", "snap2", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); + new DiffReportEntry(FILE, DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); verifyDiffReport(rawZone, "snap1", "snap2", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); + new DiffReportEntry(FILE, DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index b8898eb3e8ea5..41441f695dfd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -33,6 +31,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.server.namenode.*; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; @@ -46,6 +45,8 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -53,6 +54,8 @@ import java.util.List; import java.util.Random; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.DIRECTORY; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.FILE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -165,9 +168,10 @@ public void testRenameFromSDir2NonSDir() throws Exception { } private static boolean existsInDiffReport(List entries, - DiffType type, String sourcePath, String targetPath) { + INodeType inodeType, DiffType type, String sourcePath, + String targetPath) { for (DiffReportEntry entry : entries) { - if (entry.equals(new DiffReportEntry(type, DFSUtil + if (entry.equals(new DiffReportEntry(inodeType, type, DFSUtil .string2Bytes(sourcePath), targetPath == null ? null : DFSUtil .string2Bytes(targetPath)))) { return true; @@ -192,8 +196,10 @@ public void testRenameFileNotInSnapshot() throws Exception { SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); List entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(), + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, "", + null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.CREATE, + file2.getName(), null)); } @@ -214,9 +220,10 @@ public void testRenameFileInSnapshot() throws Exception { System.out.println("DiffList is " + diffReport.toString()); List entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), - file2.getName())); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + "", null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + file1.getName(), file2.getName())); } @Test (timeout=60000) @@ -237,25 +244,28 @@ public void testRenameTwiceInSnapshot() throws Exception { LOG.info("DiffList is " + diffReport.toString()); List entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), - file2.getName())); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + "", null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + file1.getName(), file2.getName())); diffReport = hdfs.getSnapshotDiffReport(sub1, snap2, ""); LOG.info("DiffList is " + diffReport.toString()); entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, file2.getName(), - file3.getName())); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + "", null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + file2.getName(), file3.getName())); diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); LOG.info("DiffList is " + diffReport.toString()); entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), - file3.getName())); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + "", null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + file1.getName(), file3.getName())); } @Test (timeout=60000) @@ -278,10 +288,11 @@ public void testRenameFileInSubDirOfDirWithSnapshot() throws Exception { ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List entries = diffReport.getDiffList(); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(), - null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName() - + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName())); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + sub2.getName(), null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + sub2.getName() + "/" + sub2file1.getName(), + sub2.getName() + "/" + sub2file2.getName())); } @Test (timeout=60000) @@ -305,9 +316,10 @@ public void testRenameDirectoryInSnapshot() throws Exception { LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List entries = diffReport.getDiffList(); assertEquals(2, entries.size()); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(), - sub3.getName())); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + "", null)); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.RENAME, + sub2.getName(), sub3.getName())); } /** @@ -2366,12 +2378,19 @@ public void testRenameWithOverWrite() throws Exception { LOG.info("DiffList is \n\"" + report.toString() + "\""); List entries = report.getDiffList(); assertEquals(7, entries.size()); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName(), null)); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, bar.getName(), null)); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1", null)); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, "bar", "newDir")); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file2", "newDir/file2")); - assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file3", "newDir/file1")); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + "", null)); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + foo.getName(), null)); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.MODIFY, + bar.getName(), null)); + assertTrue(existsInDiffReport(entries, FILE, DiffType.DELETE, "foo/file1", + null)); + assertTrue(existsInDiffReport(entries, DIRECTORY, DiffType.RENAME, + "bar", "newDir")); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + "foo/file2", "newDir/file2")); + assertTrue(existsInDiffReport(entries, FILE, DiffType.RENAME, + "foo/file3", "newDir/file1")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java index 18ec3c581d502..03da38386b443 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; +import static org.apache.hadoop.hdfs.DFSUtil.string2Bytes; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.DIRECTORY; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.FILE; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.SYMLINK; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -27,6 +31,7 @@ import java.util.Date; import java.util.EnumSet; import java.util.HashMap; +import java.util.Optional; import java.util.Random; import java.util.List; import java.util.ArrayList; @@ -247,72 +252,72 @@ public void testDiffReport() throws Exception { assertEquals(0, report.getDiffList().size()); verifyDiffReport(sub1, "s0", "s2", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file15")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file12")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, string2Bytes("link13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes ("link13"))); verifyDiffReport(sub1, "s0", "s5", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file15")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file12")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, string2Bytes("link13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes("link13")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file15"))); verifyDiffReport(sub1, "s2", "s5", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file15"))); + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file10")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file15"))); verifyDiffReport(sub1, "s3", "", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file15")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file12")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1/file10")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file15")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/subsubsub1/file12")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/link13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, + string2Bytes("subsub1/subsubsub1/link13"))); } @Test(timeout = 60000) @@ -380,200 +385,189 @@ private void verifyDescendantDiffReports(final Path snapDir, final Path snapSubDir, final Path snapSubSubDir) throws IOException { verifyDiffReport(snapDir, "s0", "s2", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file15")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file12")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, string2Bytes("link13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes("link13"))); verifyDiffReport(snapSubDir, "s0", "s2", new DiffReportEntry[]{}); verifyDiffReport(snapSubSubDir, "s0", "s2", new DiffReportEntry[]{}); verifyDiffReport(snapDir, "s0", "s8", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file15")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file15")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file12")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, string2Bytes("link13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes("link13")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file15")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file15"))); verifyDiffReport(snapSubDir, "s0", "s8", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file15")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file15")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsubsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file15"))); verifyDiffReport(snapSubSubDir, "s0", "s8", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes("link13")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file15"))); verifyDiffReport(snapDir, "s2", "s5", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file15"))); + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file10")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file15"))); verifyDiffReport(snapSubDir, "s2", "s5", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes("link13")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file15"))); verifyDiffReport(snapSubSubDir, "s2", "s5", new DiffReportEntry[]{}); verifyDiffReport(snapDir, "s3", "", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file15")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/file12")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/file10")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/file11")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/link13")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/link13")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file15")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/file12")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("subsub1/file10")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("subsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/link13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, + string2Bytes("subsub1/link13")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file15"))); verifyDiffReport(snapSubDir, "s3", "", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file15")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("file12")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsubsub1/file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file15")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("file12")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("link13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, + string2Bytes("link13")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsubsub1/link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsubsub1/file15"))); verifyDiffReport(snapSubSubDir, "s3", "", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file10")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("link13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file15"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file10")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("link13")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("file15"))); } @Test @@ -602,33 +596,30 @@ public void testSnapRootDescendantDiffReportWithRename() throws Exception { // The snapshot diff for the snap root detects the change as file rename // as the file move happened within the snap root. verifyDiffReport(sub1, "s1", "s2", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.RENAME, - DFSUtil.string2Bytes("subsub1/subsubsub1/file20"), - DFSUtil.string2Bytes("subsub1/file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.RENAME, + string2Bytes("subsub1/subsubsub1/file20"), + string2Bytes("subsub1/file20"))); // The snapshot diff for the descendant dir still detects the // change as file rename as the file move happened under the snap root // descendant dir. verifyDiffReport(subSub, "s1", "s2", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsubsub1")), - new DiffReportEntry(DiffType.RENAME, - DFSUtil.string2Bytes("subsubsub1/file20"), - DFSUtil.string2Bytes("file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsubsub1")), + new DiffReportEntry(FILE, DiffType.RENAME, + string2Bytes("subsubsub1/file20"), + string2Bytes("file20"))); // The snapshot diff for the descendant dir detects the // change as file delete as the file got moved from its scope. verifyDiffReport(subSubSub, "s1", "s2", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file20"))); // Case 2: Move the file from the snap root descendant dir to any // non snap root dir. mv //file20 /file20. @@ -640,19 +631,18 @@ public void testSnapRootDescendantDiffReportWithRename() throws Exception { // as the file got moved away from the snap root dir to some non snap // root dir. verifyDiffReport(sub1, "s2", "s3", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/file20"))); // The snapshot diff for the snap root descendant detects the // change as file delete as the file was previously under its scope and // got moved away from its scope. verifyDiffReport(subSub, "s2", "s3", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("file20"))); // The file was already not under the descendant dir scope. // So, the snapshot diff report for the descendant dir doesn't @@ -668,10 +658,8 @@ public void testSnapRootDescendantDiffReportWithRename() throws Exception { // Snap root directory should show the file moved in as a new file. verifyDiffReport(sub1, "s3", "s4", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file20"))); // Snap descendant directories don't have visibility to the moved in file. verifyDiffReport(subSub, "s3", "s4", @@ -686,21 +674,18 @@ public void testSnapRootDescendantDiffReportWithRename() throws Exception { // Snap root directory now shows the rename as both source and // destination paths are under the snap root. verifyDiffReport(sub1, "s4", "s5", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.RENAME, - DFSUtil.string2Bytes("file20"), - DFSUtil.string2Bytes("subsub1/file20")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.RENAME, + string2Bytes("file20"), + string2Bytes("subsub1/file20")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1"))); // For the descendant directory under the snap root, the file // moved in shows up as a new file created. verifyDiffReport(subSub, "s4", "s5", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("file20"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file20"))); verifyDiffReport(subSubSub, "s4", "s5", new DiffReportEntry[]{}); @@ -714,10 +699,10 @@ public void testSnapRootDescendantDiffReportWithRename() throws Exception { hdfs.createSnapshot(sub1, genSnapshotName(sub1)); verifyDiffReport(sub1, "s5", "s6", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub2"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1")), + new DiffReportEntry(DIRECTORY, DiffType.CREATE, + string2Bytes("subsub1/subsubsub2"))); verifyDiffReport(subSubSub2, "s5", "s6", new DiffReportEntry[]{}); @@ -791,26 +776,27 @@ public void testDiffReport2() throws Exception { hdfs.delete(subsub1, true); // check diff report between s0 and s2 verifyDiffReport(sub1, "s0", "s2", - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file15")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file12")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/file11")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("subsub1/subsubsub1/file13")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("subsub1/subsubsub1/link13"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file15")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/subsubsub1/file12")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/file11")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes("subsub1/subsubsub1/file13")), + new DiffReportEntry(SYMLINK, DiffType.CREATE, + string2Bytes("subsub1/subsubsub1/link13")), + new DiffReportEntry(SYMLINK, DiffType.DELETE, + string2Bytes("subsub1/subsubsub1/link13"))); // check diff report between s0 and the current status verifyDiffReport(sub1, "s0", "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("subsub1"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.DELETE, + string2Bytes("subsub1"))); } /** @@ -842,16 +828,17 @@ public void testDiffReportWithRename() throws Exception { hdfs.delete(sdir2, true); verifyDiffReport(root, "s1", "s2", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/foo"), - DFSUtil.string2Bytes("dir2/bar/foo")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes("dir1/foo/bar")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), - new DiffReportEntry(DiffType.RENAME, DFSUtil - .string2Bytes("dir1/foo/bar"), DFSUtil.string2Bytes("dir2/bar"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir1")), + new DiffReportEntry(DIRECTORY, DiffType.RENAME, + string2Bytes("dir1/foo"), string2Bytes("dir2/bar/foo")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir2")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("dir1/foo/bar")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("dir1/foo")), + new DiffReportEntry(DIRECTORY, DiffType.RENAME, + string2Bytes("dir1/foo/bar"), string2Bytes("dir2/bar"))); } /** @@ -882,11 +869,11 @@ public void testDiffReportWithRenameOutside() throws Exception { SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1"); verifyDiffReport(dir1, "s0", "s1", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes(newBar + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.CREATE, string2Bytes(newBar .getName())), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes(foo.getName()))); + new DiffReportEntry(DIRECTORY, DiffType.DELETE, + string2Bytes(foo.getName()))); } /** @@ -909,25 +896,28 @@ public void testDiffReportWithRenameAndDelete() throws Exception { hdfs.rename(fileInFoo, fileInBar, Rename.OVERWRITE); SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); verifyDiffReport(root, "s0", "s1", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2/bar")), - new DiffReportEntry(DiffType.DELETE, DFSUtil - .string2Bytes("dir2/bar/file")), - new DiffReportEntry(DiffType.RENAME, - DFSUtil.string2Bytes("dir1/foo/file"), - DFSUtil.string2Bytes("dir2/bar/file"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("dir1/foo")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir2/bar")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("dir2/bar/file")), + new DiffReportEntry(FILE, DiffType.RENAME, + string2Bytes("dir1/foo/file"), + string2Bytes("dir2/bar/file"))); // delete bar hdfs.delete(bar, true); SnapshotTestHelper.createSnapshot(hdfs, root, "s2"); verifyDiffReport(root, "s0", "s2", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("dir2/bar")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("dir1/foo/file"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("dir1/foo")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir2")), + new DiffReportEntry(DIRECTORY, DiffType.DELETE, + string2Bytes("dir2/bar")), + new DiffReportEntry(FILE, DiffType.DELETE, + string2Bytes("dir1/foo/file"))); } @Test @@ -945,11 +935,11 @@ public void testDiffReportWithRenameToNewDir() throws Exception { SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); verifyDiffReport(root, "s0", "s1", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("bar")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo/file"), - DFSUtil.string2Bytes("bar/file"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("foo")), + new DiffReportEntry(DIRECTORY, DiffType.CREATE, string2Bytes("bar")), + new DiffReportEntry(DIRECTORY, DiffType.RENAME, + string2Bytes("foo/file"), string2Bytes("bar/file"))); } /** @@ -969,10 +959,10 @@ public void testDiffReportWithRenameAndAppend() throws Exception { // we always put modification on the file before rename verifyDiffReport(root, "s0", "s1", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"), - DFSUtil.string2Bytes("bar"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("foo")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("foo"), + string2Bytes("bar"))); } /** @@ -1005,10 +995,10 @@ public void testDiffReportWithRenameAndSnapshotDeletion() throws Exception { // we always put modification on the file before rename verifyDiffReport(root, "s1", "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo2/bar"), - DFSUtil.string2Bytes("foo2/bar-new"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("foo2")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("foo2/bar"), + string2Bytes("foo2/bar-new"))); } private void createFile(final Path filePath) throws IOException { @@ -1062,7 +1052,7 @@ public void testDiffReportWithOpenFiles() throws Exception { hdfs.getFileStatus(flumeS1Path).getLen()); verifyDiffReport(level0A, flumeSnap1Name, "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes(""))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes(""))); long flumeFileWrittenDataLength = flumeFileLengthAfterS1; int newWriteLength = (int) (BLOCKSIZE * 1.5); @@ -1087,17 +1077,17 @@ public void testDiffReportWithOpenFiles() throws Exception { hdfs.getFileStatus(flumeS2Path).getLen()); verifyDiffReport(level0A, flumeSnap1Name, "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes(flumeFileName))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes(flumeFileName))); verifyDiffReport(level0A, flumeSnap2Name, "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes(""))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes(""))); verifyDiffReport(level0A, flumeSnap1Name, flumeSnap2Name, - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes(flumeFileName))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes(flumeFileName))); // Write more data to flume file flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf); @@ -1123,26 +1113,26 @@ public void testDiffReportWithOpenFiles() throws Exception { hdfs.getFileStatus(flumeS2Path).getLen()); verifyDiffReport(level0A, flumeSnap1Name, "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes(flumeFileName))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes(flumeFileName))); verifyDiffReport(level0A, flumeSnap2Name, "", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes(flumeFileName))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes(flumeFileName))); verifyDiffReport(level0A, flumeSnap1Name, flumeSnap2Name, - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes(flumeFileName))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes(flumeFileName))); restartNameNode(); verifyDiffReport(level0A, flumeSnap1Name, flumeSnap2Name, - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, - DFSUtil.string2Bytes(flumeFileName))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(FILE, DiffType.MODIFY, + string2Bytes(flumeFileName))); } @@ -1309,28 +1299,19 @@ public void testDiffReportWithRpcLimit() throws Exception { SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); verifyDiffReport(root, "s0", "s1", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir1/file1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir1/file2")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir1/file3")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir2/file1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir2/file2")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir2/file3")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir3")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir3/file2")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir3/file3"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir1")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir1/file1")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir1/file2")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir1/file3")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir2")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir2/file1")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir2/file2")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir2/file3")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir3")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir3/file1")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir3/file2")), + new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("dir3/file3"))); } @Test @@ -1372,32 +1353,29 @@ public void testDiffReportWithRpcLimit2() throws Exception { hdfs.rename(pathToRename, targetDir); SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); verifyDiffReport(root, "s0", "s1", - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, - DFSUtil.string2Bytes("dir4")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2"), - DFSUtil.string2Bytes("dir3/dir2")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file1"), - DFSUtil.string2Bytes("dir4/file1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file2"), - DFSUtil.string2Bytes("dir4/file2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file3"), - DFSUtil.string2Bytes("dir4/file3")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file1"), - DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file2"), - DFSUtil.string2Bytes("dir3/file2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file3"), - DFSUtil.string2Bytes("dir3/file3")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir3")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.DELETE, - DFSUtil.string2Bytes("dir3/file3"))); + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.CREATE, + string2Bytes("dir4")), + new DiffReportEntry(DIRECTORY, DiffType.RENAME, string2Bytes("dir2"), + string2Bytes("dir3/dir2")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir1")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("dir1/file1"), + string2Bytes("dir4/file1")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("dir1/file2"), + string2Bytes("dir4/file2")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("dir1/file3"), + string2Bytes("dir4/file3")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir2")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("dir2/file1"), + string2Bytes("dir3/file1")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("dir2/file2"), + string2Bytes("dir3/file2")), + new DiffReportEntry(FILE, DiffType.RENAME, string2Bytes("dir2/file3"), + string2Bytes("dir3/file3")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir3")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("dir3/file1")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("dir3/file1")), + new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("dir3/file3"))); } private void verifyDiffReportForGivenReport(Path dirPath, String from, @@ -1419,11 +1397,13 @@ private void verifyDiffReportForGivenReport(Path dirPath, String from, } else if (entry.getType() == DiffType.DELETE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); + new DiffReportEntry(DIRECTORY, + DiffType.CREATE, entry.getSourcePath()))); } else if (entry.getType() == DiffType.CREATE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); + new DiffReportEntry(DIRECTORY, + DiffType.DELETE, entry.getSourcePath()))); } } } @@ -1496,30 +1476,30 @@ public void testSnapshotDiffReportRemoteIterator() throws Exception { report.getIsFromEarlier(), modifiedList, createdList, deletedList); verifyDiffReportForGivenReport(root, "s0", "s1", snapshotDiffReport.generateReport(), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("dir4")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2"), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.CREATE, DFSUtil.string2Bytes("dir4")), + new DiffReportEntry(DIRECTORY, DiffType.RENAME, DFSUtil.string2Bytes("dir2"), DFSUtil.string2Bytes("dir3/dir2")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file1"), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), + new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir1/file1"), DFSUtil.string2Bytes("dir4/file1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file2"), + new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir1/file2"), DFSUtil.string2Bytes("dir4/file2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file3"), + new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir1/file3"), DFSUtil.string2Bytes("dir4/file3")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file1"), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), + new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir2/file1"), DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file2"), + new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir2/file2"), DFSUtil.string2Bytes("dir3/file2")), - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file3"), + new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir2/file3"), DFSUtil.string2Bytes("dir3/file3")), - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir3")), - new DiffReportEntry(DiffType.DELETE, + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("dir3")), + new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.DELETE, + new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(DiffType.DELETE, + new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("dir3/file3"))); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index b77c8cbc6d513..92f1b2726c131 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.INodeType.FILE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -791,16 +792,19 @@ public void testWebHdfsSnapshotDiff() throws Exception { Assert.assertEquals("s1", diffReport.getFromSnapshot()); Assert.assertEquals("s2", diffReport.getLaterSnapshotName()); DiffReportEntry entry0 = - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")); + new DiffReportEntry(FILE, DiffType.MODIFY, DFSUtil.string2Bytes("")); DiffReportEntry entry1 = - new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file0")); + new DiffReportEntry(FILE, DiffType.MODIFY, + DFSUtil.string2Bytes("file0")); DiffReportEntry entry2 = - new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file1")); + new DiffReportEntry(FILE, DiffType.DELETE, + DFSUtil.string2Bytes("file1")); DiffReportEntry entry3 = - new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("file2"), - DFSUtil.string2Bytes("file4")); + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("file2"), DFSUtil.string2Bytes("file4")); DiffReportEntry entry4 = - new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file3")); + new DiffReportEntry(FILE, DiffType.CREATE, + DFSUtil.string2Bytes("file3")); Assert.assertTrue(diffReport.getDiffList().contains(entry0)); Assert.assertTrue(diffReport.getDiffList().contains(entry1)); Assert.assertTrue(diffReport.getDiffList().contains(entry2)); From 20739296fdc7b085822722f7f42f47ba7297a156 Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Mon, 19 Aug 2019 09:49:06 +0200 Subject: [PATCH 2/7] HDFS-13118. Fix findbugs issue with default case in INodeType switch statement. --- .../java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 3a3cbee7651a6..bebb7beaffe3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -2744,6 +2744,9 @@ public static SnapshotDiffReportListingEntryProto convert( case SYMLINK: fileType = FileType.IS_SYMLINK; break; + default: + throw new IllegalArgumentException("Unknown INodeType: " + + entry.getINodeType()); } long dirId = entry.getDirId(); long fileId = entry.getFileId(); From 1472c478dbe4539349a986549cf848374cef0730 Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Mon, 19 Aug 2019 10:51:09 +0200 Subject: [PATCH 3/7] HDFS-13118. Fix checkstyle issues for adding INodeType to snapshot diffreport. --- .../impl/SnapshotDiffReportGenerator.java | 3 +- .../hdfs/protocol/SnapshotDiffReport.java | 12 +++- .../protocol/SnapshotDiffReportListing.java | 3 + .../hdfs/protocolPB/PBHelperClient.java | 72 ++++++++++--------- .../namenode/snapshot/SnapshotDiffInfo.java | 11 +-- .../snapshot/SnapshotDiffListingInfo.java | 1 - .../hadoop/hdfs/TestEncryptionZones.java | 6 +- .../snapshot/TestSnapshotDiffReport.java | 42 +++++++---- 8 files changed, 90 insertions(+), 60 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java index 327e0a5d9335f..432cb5caf809b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/SnapshotDiffReportGenerator.java @@ -222,7 +222,8 @@ public SnapshotDiffReport generateReport() { generateReportList(); for (DiffReportListingEntry modified : mlist) { diffReportList.add( - new DiffReportEntry(modified.getINodeType().toSnapshotDiffReportINodeType(), + new DiffReportEntry(modified.getINodeType() + .toSnapshotDiffReportINodeType(), DiffType.MODIFY, modified.getSourcePath(), null)); if (modified.isReference() && dirDiffMap.get(modified.getDirId()) != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 5438319ac4dec..e367f787b0b31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -75,6 +75,9 @@ public static DiffType parseDiffType(String s){ } } + /** + * INodeType specifies the type of INode: FILE, DIRECTORY, or SYMLINK. + */ public enum INodeType { FILE, DIRECTORY, @@ -100,15 +103,18 @@ public static class DiffReportEntry { private final byte[] sourcePath; private final byte[] targetPath; - public DiffReportEntry(INodeType iNodeType, DiffType type, byte[] sourcePath) { + public DiffReportEntry(INodeType iNodeType, DiffType type, + byte[] sourcePath) { this(iNodeType, type, sourcePath, null); } - public DiffReportEntry(INodeType iNodeType, DiffType type, byte[][] sourcePathComponents) { + public DiffReportEntry(INodeType iNodeType, DiffType type, + byte[][] sourcePathComponents) { this(iNodeType, type, sourcePathComponents, null); } - public DiffReportEntry(INodeType iNodeType, DiffType type, byte[] sourcePath, byte[] targetPath) { + public DiffReportEntry(INodeType iNodeType, DiffType type, + byte[] sourcePath, byte[] targetPath) { this.iNodeType = iNodeType; this.type = type; this.sourcePath = sourcePath; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java index 9a189ddddb46d..9120395ca598e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java @@ -38,6 +38,9 @@ public class SnapshotDiffReportListing { */ public static class DiffReportListingEntry { + /** + * INodeType specifies the type of the INode. + */ public enum INodeType { FILE(SnapshotDiffReport.INodeType.FILE), diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index bebb7beaffe3e..f144028911359 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1611,11 +1611,14 @@ public static DiffReportEntry convert(SnapshotDiffReportEntryProto entry) { case IS_SYMLINK: inodeType = SnapshotDiffReport.INodeType.SYMLINK; break; + default: + throw new IllegalArgumentException("Unknown INodeType: " + + entry.getFileType()); } } - return type == null ? null : new DiffReportEntry(inodeType, type, entry.getFullpath() - .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath() - .toByteArray() : null); + return type == null ? null : new DiffReportEntry(inodeType, type, + entry.getFullpath().toByteArray(), entry.hasTargetPath() ? + entry.getTargetPath().toByteArray() : null); } public static SnapshotDiffReportListing convert( @@ -1665,15 +1668,15 @@ public static DiffReportListingEntry convert( } DiffReportListingEntry.INodeType inodeType = null; switch(entry.getFileType()) { - case IS_FILE: - inodeType = DiffReportListingEntry.INodeType.FILE; - break; - case IS_DIR: - inodeType = DiffReportListingEntry.INodeType.DIRECTORY; - break; - case IS_SYMLINK: - inodeType = DiffReportListingEntry.INodeType.SYMLINK; - break; + case IS_FILE: + inodeType = DiffReportListingEntry.INodeType.FILE; + break; + case IS_DIR: + inodeType = DiffReportListingEntry.INodeType.DIRECTORY; + break; + case IS_SYMLINK: + inodeType = DiffReportListingEntry.INodeType.SYMLINK; + break; } long dirId = entry.getDirId(); long fileId = entry.getFileId(); @@ -2706,15 +2709,18 @@ public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { .newBuilder().setFullpath(sourcePath) .setModificationLabel(modification); switch(entry.getINodeType()){ - case FILE: - builder.setFileType(FileType.IS_FILE); - break; - case DIRECTORY: - builder.setFileType(FileType.IS_DIR); - break; - case SYMLINK: - builder.setFileType(FileType.IS_SYMLINK); - break; + case FILE: + builder.setFileType(FileType.IS_FILE); + break; + case DIRECTORY: + builder.setFileType(FileType.IS_DIR); + break; + case SYMLINK: + builder.setFileType(FileType.IS_SYMLINK); + break; + default: + throw new IllegalArgumentException("Unknown INodeType: " + + entry.getINodeType()); } if (entry.getType() == DiffType.RENAME) { ByteString targetPath = @@ -2735,18 +2741,18 @@ public static SnapshotDiffReportListingEntryProto convert( DFSUtilClient.byteArray2bytes(entry.getSourcePath())); HdfsFileStatusProto.FileType fileType = null; switch(entry.getINodeType()){ - case FILE: - fileType = FileType.IS_FILE; - break; - case DIRECTORY: - fileType = FileType.IS_DIR; - break; - case SYMLINK: - fileType = FileType.IS_SYMLINK; - break; - default: - throw new IllegalArgumentException("Unknown INodeType: " + - entry.getINodeType()); + case FILE: + fileType = FileType.IS_FILE; + break; + case DIRECTORY: + fileType = FileType.IS_DIR; + break; + case SYMLINK: + fileType = FileType.IS_SYMLINK; + break; + default: + throw new IllegalArgumentException("Unknown INodeType: " + + entry.getINodeType()); } long dirId = entry.getDirId(); long fileId = entry.getFileId(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java index 10a94339bcf2f..c2486067e45bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -239,7 +239,8 @@ public SnapshotDiffReport generateReport() { for (Map.Entry drEntry : diffMap.entrySet()) { INode node = drEntry.getKey(); byte[][] path = drEntry.getValue(); - diffReportList.add(new DiffReportEntry(determineINodeType(node),DiffType.MODIFY, path, null)); + diffReportList.add(new DiffReportEntry(determineINodeType(node), + DiffType.MODIFY, path, null)); if (node.isDirectory()) { List subList = generateReport(dirDiffMap.get(node), path, isFromEarlier(), renameMap); @@ -275,8 +276,8 @@ private List generateReport(ChildrenDiff dirDiff, RenameEntry entry = renameMap.get(cnode.getId()); if (entry == null || !entry.isRename()) { fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - list.add(new DiffReportEntry(determineINodeType(cnode),fromEarlier ? DiffType.CREATE - : DiffType.DELETE, fullPath)); + list.add(new DiffReportEntry(determineINodeType(cnode), + fromEarlier ? DiffType.CREATE : DiffType.DELETE, fullPath)); } } for (INode dnode : dirDiff.getDeletedUnmodifiable()) { @@ -287,8 +288,8 @@ private List generateReport(ChildrenDiff dirDiff, fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); } else { fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); - list.add(new DiffReportEntry(determineINodeType(dnode), fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); + list.add(new DiffReportEntry(determineINodeType(dnode), + fromEarlier ? DiffType.DELETE : DiffType.CREATE, fullPath)); } } return list; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index a033f0117ac80..367739cef30a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -33,7 +33,6 @@ import org.apache.hadoop.util.ChunkedArrayList; import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.*; -import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.FILE; /** * A class describing the difference between snapshots of a snapshottable diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 94f5f01c90aa9..7275783c34f6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -1542,11 +1542,13 @@ public void testSnapshotDiffOnEncryptionZones() throws Exception { fs.createSnapshot(zone, "snap2"); verifyDiffReport(zone, "snap1", "snap2", new DiffReportEntry(FILE, DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); + new DiffReportEntry(FILE, DiffType.DELETE, + DFSUtil.string2Bytes("zoneFile"))); verifyDiffReport(rawZone, "snap1", "snap2", new DiffReportEntry(FILE, DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); + new DiffReportEntry(FILE, DiffType.DELETE, + DFSUtil.string2Bytes("zoneFile"))); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java index 03da38386b443..3c2864dba5e91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java @@ -31,7 +31,6 @@ import java.util.Date; import java.util.EnumSet; import java.util.HashMap; -import java.util.Optional; import java.util.Random; import java.util.List; import java.util.ArrayList; @@ -259,7 +258,7 @@ public void testDiffReport() throws Exception { new DiffReportEntry(FILE, DiffType.CREATE, string2Bytes("file11")), new DiffReportEntry(FILE, DiffType.MODIFY, string2Bytes("file13")), new DiffReportEntry(SYMLINK, DiffType.DELETE, string2Bytes("link13")), - new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes ("link13"))); + new DiffReportEntry(SYMLINK, DiffType.CREATE, string2Bytes("link13"))); verifyDiffReport(sub1, "s0", "s5", new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), @@ -899,7 +898,8 @@ public void testDiffReportWithRenameAndDelete() throws Exception { new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("")), new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir1/foo")), - new DiffReportEntry(DIRECTORY, DiffType.MODIFY, string2Bytes("dir2/bar")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + string2Bytes("dir2/bar")), new DiffReportEntry(FILE, DiffType.DELETE, string2Bytes("dir2/bar/file")), new DiffReportEntry(FILE, DiffType.RENAME, @@ -1476,25 +1476,37 @@ public void testSnapshotDiffReportRemoteIterator() throws Exception { report.getIsFromEarlier(), modifiedList, createdList, deletedList); verifyDiffReportForGivenReport(root, "s0", "s1", snapshotDiffReport.generateReport(), - new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("")), - new DiffReportEntry(DIRECTORY, DiffType.CREATE, DFSUtil.string2Bytes("dir4")), - new DiffReportEntry(DIRECTORY, DiffType.RENAME, DFSUtil.string2Bytes("dir2"), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + DFSUtil.string2Bytes("")), + new DiffReportEntry(DIRECTORY, DiffType.CREATE, + DFSUtil.string2Bytes("dir4")), + new DiffReportEntry(DIRECTORY, DiffType.RENAME, + DFSUtil.string2Bytes("dir2"), DFSUtil.string2Bytes("dir3/dir2")), - new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), - new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir1/file1"), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + DFSUtil.string2Bytes("dir1")), + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("dir1/file1"), DFSUtil.string2Bytes("dir4/file1")), - new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir1/file2"), + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("dir1/file2"), DFSUtil.string2Bytes("dir4/file2")), - new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir1/file3"), + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("dir1/file3"), DFSUtil.string2Bytes("dir4/file3")), - new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), - new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir2/file1"), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + DFSUtil.string2Bytes("dir2")), + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("dir2/file1"), DFSUtil.string2Bytes("dir3/file1")), - new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir2/file2"), + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("dir2/file2"), DFSUtil.string2Bytes("dir3/file2")), - new DiffReportEntry(FILE, DiffType.RENAME, DFSUtil.string2Bytes("dir2/file3"), + new DiffReportEntry(FILE, DiffType.RENAME, + DFSUtil.string2Bytes("dir2/file3"), DFSUtil.string2Bytes("dir3/file3")), - new DiffReportEntry(DIRECTORY, DiffType.MODIFY, DFSUtil.string2Bytes("dir3")), + new DiffReportEntry(DIRECTORY, DiffType.MODIFY, + DFSUtil.string2Bytes("dir3")), new DiffReportEntry(FILE, DiffType.DELETE, DFSUtil.string2Bytes("dir3/file1")), new DiffReportEntry(FILE, DiffType.DELETE, From abeddb10ec7c48a06b9fc7c60ec0cd266a58ff79 Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Wed, 21 Aug 2019 13:34:33 +0200 Subject: [PATCH 4/7] HDFS-13118. Remove use of wildcard import. --- .../server/namenode/snapshot/SnapshotDiffListingInfo.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index 367739cef30a0..da922d4925833 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -32,7 +32,9 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.util.ChunkedArrayList; -import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.*; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.DIRECTORY; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.FILE; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry.INodeType.SYMLINK; /** * A class describing the difference between snapshots of a snapshottable From dcb007578315c823bede80ea11bc7d9712dfd9ff Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Thu, 22 Aug 2019 13:13:56 +0200 Subject: [PATCH 5/7] HDFS-13118. Make INodeType variables consistent as inodeType (not iNodeType). --- .../hdfs/protocol/SnapshotDiffReport.java | 28 +++++++++---------- .../hdfs/protocolPB/PBHelperClient.java | 4 +-- .../snapshot/SnapshotDiffListingInfo.java | 6 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index e367f787b0b31..384d106d573a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -95,7 +95,7 @@ public static INodeType parseINodeType(String s) { public static class DiffReportEntry { /** The type of the difference. */ private final DiffType type; - private final INodeType iNodeType; + private final INodeType inodeType; /** * The relative path (related to the snapshot root) of 1) the file/directory * where changes have happened, or 2) the source file/dir of a rename op. @@ -103,27 +103,27 @@ public static class DiffReportEntry { private final byte[] sourcePath; private final byte[] targetPath; - public DiffReportEntry(INodeType iNodeType, DiffType type, - byte[] sourcePath) { - this(iNodeType, type, sourcePath, null); + public DiffReportEntry(INodeType inodeType, DiffType type, + byte[] sourcePath) { + this(inodeType, type, sourcePath, null); } - public DiffReportEntry(INodeType iNodeType, DiffType type, - byte[][] sourcePathComponents) { - this(iNodeType, type, sourcePathComponents, null); + public DiffReportEntry(INodeType inodeType, DiffType type, + byte[][] sourcePathComponents) { + this(inodeType, type, sourcePathComponents, null); } - public DiffReportEntry(INodeType iNodeType, DiffType type, + public DiffReportEntry(INodeType inodeType, DiffType type, byte[] sourcePath, byte[] targetPath) { - this.iNodeType = iNodeType; + this.inodeType = inodeType; this.type = type; this.sourcePath = sourcePath; this.targetPath = targetPath; } - public DiffReportEntry(INodeType iNodeType, DiffType type, byte[][] sourcePathComponents, - byte[][] targetPathComponents) { - this.iNodeType = iNodeType; + public DiffReportEntry(INodeType inodeType, DiffType type, byte[][] sourcePathComponents, + byte[][] targetPathComponents) { + this.inodeType = inodeType; this.type = type; this.sourcePath = DFSUtilClient.byteArray2bytes(sourcePathComponents); this.targetPath = targetPathComponents == null ? null : DFSUtilClient @@ -143,8 +143,8 @@ public DiffType getType() { return type; } - public INodeType getINodeType() { - return iNodeType; + public INodeType getInodeType() { + return inodeType; } static String getPathString(byte[] path) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index f144028911359..8e487e24ad281 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -2708,7 +2708,7 @@ public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto .newBuilder().setFullpath(sourcePath) .setModificationLabel(modification); - switch(entry.getINodeType()){ + switch(entry.getInodeType()){ case FILE: builder.setFileType(FileType.IS_FILE); break; @@ -2720,7 +2720,7 @@ public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { break; default: throw new IllegalArgumentException("Unknown INodeType: " + - entry.getINodeType()); + entry.getInodeType()); } if (entry.getType() == DiffType.RENAME) { ByteString targetPath = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index da922d4925833..f0e61d92f1b02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -107,7 +107,7 @@ boolean addDirDiff(long dirId, byte[][] parent, ChildrenDiff diff) { if (getTotalEntries() < maxEntries) { INode created = iterator.next(); byte[][] path = newPath(parent, created.getLocalNameBytes()); - createdList.add(new DiffReportListingEntry(fromINode(created), dirId, + createdList.add(new DiffReportListingEntry(getInodeType(created), dirId, created.getId(), path, created.isReference(), null)); } else { setLastPath(parent); @@ -128,7 +128,7 @@ boolean addDirDiff(long dirId, byte[][] parent, ChildrenDiff diff) { final INode d = iterator.next(); byte[][] path = newPath(parent, d.getLocalNameBytes()); byte[][] target = findRenameTargetPath(d, laterSnapshot); - DiffReportListingEntry.INodeType inodeType = fromINode(d); + DiffReportListingEntry.INodeType inodeType = getInodeType(d); final DiffReportListingEntry e = target != null ? new DiffReportListingEntry(inodeType, dirId, d.getId(), path, true, target) : @@ -163,7 +163,7 @@ private static byte[][] newPath(byte[][] parent, byte[] name) { return fullPath; } - private static DiffReportListingEntry.INodeType fromINode(INode inode) { + private static DiffReportListingEntry.INodeType getInodeType(INode inode) { if (inode.isFile()) { return FILE; } else if (inode.isDirectory()) { From e70c36d954cd0adfa398517005b8e58b4b1184ca Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Thu, 22 Aug 2019 13:50:13 +0200 Subject: [PATCH 6/7] HDFS-13118. Add inodeType to json output and parsed input. Update documentation/specification as well. --- .../java/org/apache/hadoop/hdfs/web/JsonUtilClient.java | 2 +- .../src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java | 1 + .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 4e7bfabb59506..e56845a697713 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -805,7 +805,7 @@ private static SnapshotDiffReport.DiffReportEntry toDiffReportEntry( } SnapshotDiffReport.INodeType iNodeType = SnapshotDiffReport.INodeType.parseINodeType( - (String) json.get("iNodeType")); + (String) json.get("inodeType")); SnapshotDiffReport.DiffType type = SnapshotDiffReport.DiffType.parseDiffType((String) json.get("type")); byte[] sourcePath = toByteArray((String) json.get("sourcePath")); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 0e8638d4ff910..3dac948d2dced 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -559,6 +559,7 @@ private static Object toJsonMap(SnapshotDiffReport diffReport) { private static Object toJsonMap( SnapshotDiffReport.DiffReportEntry diffReportEntry) { final Map m = new TreeMap(); + m.put("inodeType", diffReportEntry.getInodeType()); m.put("type", diffReportEntry.getType()); if (diffReportEntry.getSourcePath() != null) { m.put("sourcePath", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index 9e1b160d6f91f..9f6ef4076ad4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -2612,6 +2612,12 @@ var diffReportEntries = "enum" : ["CREATE", "MODIFY", "DELETE", "RENAME"], "required" : true } + "inodeType": + { + "description" : "Type of INode", + "enum" : ["FILE", "DIRECTORY", "SYMLINK"], + "required" : true + } } } ``` From 1f97057f880e3ccf65d6ffdabf39995e0246ab40 Mon Sep 17 00:00:00 2001 From: Ewan Higgs Date: Mon, 9 Sep 2019 16:10:43 +0200 Subject: [PATCH 7/7] HDFS-13118. Address checkstyle issues. --- .../org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java | 4 ++-- .../org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java | 3 +++ .../server/namenode/snapshot/SnapshotDiffListingInfo.java | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 384d106d573a3..b48762bcab8eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -121,8 +121,8 @@ public DiffReportEntry(INodeType inodeType, DiffType type, this.targetPath = targetPath; } - public DiffReportEntry(INodeType inodeType, DiffType type, byte[][] sourcePathComponents, - byte[][] targetPathComponents) { + public DiffReportEntry(INodeType inodeType, DiffType type, + byte[][] sourcePathComponents, byte[][] targetPathComponents) { this.inodeType = inodeType; this.type = type; this.sourcePath = DFSUtilClient.byteArray2bytes(sourcePathComponents); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 8e487e24ad281..a22aa971d4f11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1677,6 +1677,9 @@ public static DiffReportListingEntry convert( case IS_SYMLINK: inodeType = DiffReportListingEntry.INodeType.SYMLINK; break; + default: + throw new IllegalArgumentException("Unknown entry file type: " + + entry.getFileType()); } long dirId = entry.getDirId(); long fileId = entry.getFileId(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index f0e61d92f1b02..ea75e9469faf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -107,8 +107,8 @@ boolean addDirDiff(long dirId, byte[][] parent, ChildrenDiff diff) { if (getTotalEntries() < maxEntries) { INode created = iterator.next(); byte[][] path = newPath(parent, created.getLocalNameBytes()); - createdList.add(new DiffReportListingEntry(getInodeType(created), dirId, - created.getId(), path, created.isReference(), null)); + createdList.add(new DiffReportListingEntry(getInodeType(created), + dirId, created.getId(), path, created.isReference(), null)); } else { setLastPath(parent); setLastIndex(iterator.nextIndex());