diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 773a7b2def265..8b3ed5dea09ab 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -72,9 +72,9 @@ * */ -@InterfaceAudience.Private +@InterfaceAudience.Public @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ -class ChRootedFileSystem extends FilterFileSystem { +public class ChRootedFileSystem extends FilterFileSystem { private final URI myUri; // the base URI + the chRoot private final Path chRootPathPart; // the root below the root of the base private final String chRootPathPartString; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 37f1a16800e7d..78c036320440b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; /** @@ -85,4 +86,6 @@ public interface Constants { String CONFIG_VIEWFS_ENABLE_INNER_CACHE = "fs.viewfs.enable.inner.cache"; boolean CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT = true; + + Path ROOT_PATH = new Path(Path.SEPARATOR); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 69923438ecc20..e4e126deaee7d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -55,9 +55,9 @@ * {@link #resolve(String, boolean)} */ -@InterfaceAudience.Private +@InterfaceAudience.Public @InterfaceStability.Unstable -abstract class InodeTree { +public abstract class InodeTree { enum ResultKind { INTERNAL_DIR, EXTERNAL_DIR @@ -119,7 +119,7 @@ boolean isLink() { * Internal class to represent an internal dir of the mount table. * @param */ - static class INodeDir extends INode { + public static class INodeDir extends INode { private final Map> children = new HashMap<>(); private T internalDirFs = null; //filesystem of this internal directory private boolean isRoot = false; @@ -448,7 +448,7 @@ Configuration getConfig() { * @throws FileAlreadyExistsException * @throws IOException */ - protected InodeTree(final Configuration config, final String viewName) + public InodeTree(final Configuration config, final String viewName) throws UnsupportedFileSystemException, URISyntaxException, FileAlreadyExistsException, IOException { String mountTableName = viewName; @@ -600,7 +600,7 @@ protected InodeTree(final Configuration config, final String viewName) * If the input pathname leads to an internal mount-table entry then * the target file system is one that represents the internal inode. */ - static class ResolveResult { + public static class ResolveResult { final ResultKind kind; final T targetFileSystem; final String resolvedPath; @@ -614,8 +614,23 @@ static class ResolveResult { remainingPath = remainingP; } + // Gets the target filesystem the path is pointing to + public T getTargetFileSystem() { + return targetFileSystem; + } + + // Gets the resolved path from the result + public String getResolvedPath() { + return resolvedPath; + } + + // Gets the remaining path from the result + public Path getRemainingPath() { + return remainingPath; + } + // Internal dir path resolution completed within the mount table - boolean isInternalDir() { + public boolean isInternalDir() { return (kind == ResultKind.INTERNAL_DIR); } } @@ -627,7 +642,7 @@ boolean isInternalDir() { * @return ResolveResult which allows further resolution of the remaining path * @throws FileNotFoundException */ - ResolveResult resolve(final String p, final boolean resolveLastComponent) + public ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException { String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InternalDirOfViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InternalDirOfViewFs.java new file mode 100644 index 0000000000000..0836988e22bef --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InternalDirOfViewFs.java @@ -0,0 +1,421 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.ROOT_PATH; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStoragePolicySpi; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.QuotaUsage; +import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.fs.permission.AclUtil; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Progressable; + +import static org.apache.hadoop.fs.viewfs.Constants.*; + + +/** + * An instance of this class represents an internal dir of the viewFs + * that is internal dir of the mount table. + * It is a read only mount tables and create, mkdir or delete operations + * are not allowed. + * If called on create or mkdir then this target is the parent of the + * directory in which one is trying to create or mkdir; hence + * in this case the path name passed in is the last component. + * Otherwise this target is the end point of the path and hence + * the path name passed in is null. + */ +public class InternalDirOfViewFs extends FileSystem { + final InodeTree.INodeDir theInternalDir; + final long creationTime; // of the the mount table + final UserGroupInformation ugi; // the user/group of user who created mtable + final URI myUri; + + public InternalDirOfViewFs(final InodeTree.INodeDir dir, + final long cTime, final UserGroupInformation ugi, URI uri, + Configuration config) throws URISyntaxException { + myUri = uri; + try { + initialize(myUri, config); + } catch (IOException e) { + throw new RuntimeException("Cannot occur"); + } + theInternalDir = dir; + creationTime = cTime; + this.ugi = ugi; + } + + static private void checkPathIsSlash(final Path f) throws IOException { + if (f != InodeTree.SlashPath) { + throw new IOException( + "Internal implementation error: expected file name to be /"); + } + } + + @Override + public URI getUri() { + return myUri; + } + + @Override + public Path getWorkingDirectory() { + throw new RuntimeException( + "Internal impl error: getWorkingDir should not have been called"); + } + + @Override + public void setWorkingDirectory(final Path new_dir) { + throw new RuntimeException( + "Internal impl error: getWorkingDir should not have been called"); + } + + @Override + public FSDataOutputStream append(final Path f, final int bufferSize, + final Progressable progress) throws IOException { + throw ViewFileSystemUtil.readOnlyMountTable("append", f); + } + + @Override + public FSDataOutputStream create(final Path f, + final FsPermission permission, final boolean overwrite, + final int bufferSize, final short replication, final long blockSize, + final Progressable progress) throws AccessControlException { + throw ViewFileSystemUtil.readOnlyMountTable("create", f); + } + + @Override + public boolean delete(final Path f, final boolean recursive) + throws AccessControlException, IOException { + checkPathIsSlash(f); + throw ViewFileSystemUtil.readOnlyMountTable("delete", f); + } + + @Override + @SuppressWarnings("deprecation") + public boolean delete(final Path f) + throws AccessControlException, IOException { + return delete(f, true); + } + + @Override + public BlockLocation[] getFileBlockLocations(final FileStatus fs, + final long start, final long len) throws FileNotFoundException, IOException { + checkPathIsSlash(fs.getPath()); + throw new FileNotFoundException("Path points to dir not a file"); + } + + @Override + public FileChecksum getFileChecksum(final Path f) + throws FileNotFoundException, IOException { + checkPathIsSlash(f); + throw new FileNotFoundException("Path points to dir not a file"); + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + checkPathIsSlash(f); + return new FileStatus(0, true, 0, 0, creationTime, creationTime, + PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(), + + new Path(theInternalDir.fullPath).makeQualified( + myUri, ROOT_PATH)); + } + + + @Override + public FileStatus[] listStatus(Path f) throws AccessControlException, + FileNotFoundException, IOException { + checkPathIsSlash(f); + FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()]; + int i = 0; + for (Map.Entry> iEntry : + theInternalDir.getChildren().entrySet()) { + InodeTree.INode inode = iEntry.getValue(); + if (inode.isLink()) { + InodeTree.INodeLink link = (InodeTree.INodeLink) inode; + + result[i++] = new FileStatus(0, false, 0, 0, + creationTime, creationTime, PERMISSION_555, + ugi.getShortUserName(), ugi.getPrimaryGroupName(), + link.getTargetLink(), + new Path(inode.fullPath).makeQualified( + myUri, null)); + } else { + result[i++] = new FileStatus(0, true, 0, 0, + creationTime, creationTime, PERMISSION_555, + ugi.getShortUserName(), ugi.getGroupNames()[0], + new Path(inode.fullPath).makeQualified( + myUri, null)); + } + } + return result; + } + + @Override + public boolean mkdirs(Path dir, FsPermission permission) + throws AccessControlException, FileAlreadyExistsException { + if (theInternalDir.isRoot() && dir == null) { + throw new FileAlreadyExistsException("/ already exits"); + } + // Note dir starts with / + if (theInternalDir.getChildren().containsKey( + dir.toString().substring(1))) { + return true; // this is the stupid semantics of FileSystem + } + throw ViewFileSystemUtil.readOnlyMountTable("mkdirs", dir); + } + + @Override + public boolean mkdirs(Path dir) + throws AccessControlException, FileAlreadyExistsException { + return mkdirs(dir, null); + } + + @Override + public FSDataInputStream open(Path f, int bufferSize) + throws AccessControlException, FileNotFoundException, IOException { + checkPathIsSlash(f); + throw new FileNotFoundException("Path points to dir not a file"); + } + + @Override + public boolean rename(Path src, Path dst) throws AccessControlException, + IOException { + checkPathIsSlash(src); + checkPathIsSlash(dst); + throw ViewFileSystemUtil.readOnlyMountTable("rename", src); + } + + @Override + public boolean truncate(Path f, long newLength) throws IOException { + throw ViewFileSystemUtil.readOnlyMountTable("truncate", f); + } + + @Override + public void setOwner(Path f, String username, String groupname) + throws AccessControlException, IOException { + checkPathIsSlash(f); + throw ViewFileSystemUtil.readOnlyMountTable("setOwner", f); + } + + @Override + public void setPermission(Path f, FsPermission permission) + throws AccessControlException, IOException { + checkPathIsSlash(f); + throw ViewFileSystemUtil.readOnlyMountTable("setPermission", f); + } + + @Override + public boolean setReplication(Path f, short replication) + throws AccessControlException, IOException { + checkPathIsSlash(f); + throw ViewFileSystemUtil.readOnlyMountTable("setReplication", f); + } + + @Override + public void setTimes(Path f, long mtime, long atime) + throws AccessControlException, IOException { + checkPathIsSlash(f); + throw ViewFileSystemUtil.readOnlyMountTable("setTimes", f); + } + + @Override + public void setVerifyChecksum(boolean verifyChecksum) { + // Noop for viewfs + } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + throw new NotInMountpointException(f, "getServerDefaults"); + } + + @Override + public long getDefaultBlockSize(Path f) { + throw new NotInMountpointException(f, "getDefaultBlockSize"); + } + + @Override + public short getDefaultReplication(Path f) { + throw new NotInMountpointException(f, "getDefaultReplication"); + } + + @Override + public void modifyAclEntries(Path path, List aclSpec) + throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("modifyAclEntries", path); + } + + @Override + public void removeAclEntries(Path path, List aclSpec) + throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("removeAclEntries", path); + } + + @Override + public void removeDefaultAcl(Path path) throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("removeDefaultAcl", path); + } + + @Override + public void removeAcl(Path path) throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("removeAcl", path); + } + + @Override + public void setAcl(Path path, List aclSpec) throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("setAcl", path); + } + + @Override + public AclStatus getAclStatus(Path path) throws IOException { + checkPathIsSlash(path); + return new AclStatus.Builder().owner(ugi.getShortUserName()) + .group(ugi.getPrimaryGroupName()) + .addEntries(AclUtil.getMinimalAcl(PERMISSION_555)) + .stickyBit(false).build(); + } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("setXAttr", path); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + throw new NotInMountpointException(path, "getXAttr"); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + throw new NotInMountpointException(path, "getXAttrs"); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + throw new NotInMountpointException(path, "getXAttrs"); + } + + @Override + public List listXAttrs(Path path) throws IOException { + throw new NotInMountpointException(path, "listXAttrs"); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("removeXAttr", path); + } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("createSnapshot", path); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("renameSnapshot", path); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw ViewFileSystemUtil.readOnlyMountTable("deleteSnapshot", path); + } + + @Override + public QuotaUsage getQuotaUsage(Path f) throws IOException { + throw new NotInMountpointException(f, "getQuotaUsage"); + } + + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + checkPathIsSlash(src); + throw ViewFileSystemUtil.readOnlyMountTable("satisfyStoragePolicy", src); + } + + @Override + public void setStoragePolicy(Path src, String policyName) + throws IOException { + checkPathIsSlash(src); + throw ViewFileSystemUtil.readOnlyMountTable("setStoragePolicy", src); + } + + @Override + public void unsetStoragePolicy(Path src) throws IOException { + checkPathIsSlash(src); + throw ViewFileSystemUtil.readOnlyMountTable("unsetStoragePolicy", src); + } + + @Override + public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { + throw new NotInMountpointException(src, "getStoragePolicy"); + } + + @Override + public Collection getAllStoragePolicies() + throws IOException { + Collection allPolicies = new HashSet<>(); + for (FileSystem fs : getChildFileSystems()) { + try { + Collection policies = + fs.getAllStoragePolicies(); + allPolicies.addAll(policies); + } catch (UnsupportedOperationException e) { + // ignored + } + } + return allPolicies; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java index a406d77f2ef6c..924c27989ebce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java @@ -18,9 +18,10 @@ package org.apache.hadoop.fs.viewfs; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataInputStream; @@ -57,7 +58,8 @@ /** * Nfly is a multi filesystem mount point. */ -@Private +@InterfaceAudience.Public +@InterfaceStability.Unstable final class NflyFSystem extends FileSystem { private static final Logger LOG = LoggerFactory.getLogger(NflyFSystem.class); private static final String NFLY_TMP_PREFIX = "_nfly_tmp_"; @@ -920,7 +922,7 @@ private static void processThrowable(NflyNode nflyNode, String op, * @return an Nfly filesystem * @throws IOException */ - static FileSystem createFileSystem(URI[] uris, Configuration conf, + public static FileSystem createFileSystem(URI[] uris, Configuration conf, String settings) throws IOException { // assert settings != null int minRepl = DEFAULT_MIN_REPLICATION; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 115fc033f2702..2f9852f78f825 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; -import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import java.io.FileNotFoundException; import java.io.IOException; @@ -35,7 +34,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Set; @@ -49,7 +47,6 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -64,11 +61,8 @@ import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; -import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.viewfs.InodeTree.INode; -import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; @@ -83,23 +77,10 @@ @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ public class ViewFileSystem extends FileSystem { - private static final Path ROOT_PATH = new Path(Path.SEPARATOR); - - static AccessControlException readOnlyMountTable(final String operation, - final String p) { - return new AccessControlException( - "InternalDir of ViewFileSystem is readonly, operation " + operation + - " not permitted on path " + p + "."); - } - static AccessControlException readOnlyMountTable(final String operation, - final Path p) { - return readOnlyMountTable(operation, p.toString()); - } - /** * File system instance getter. */ - static class FsGetter { + public static class FsGetter { /** * Gets new file system instance of given uri. @@ -127,15 +108,15 @@ protected FsGetter fsGetter() { /** * Caching children filesystems. HADOOP-15565. */ - static class InnerCache { + public static class InnerCache { private Map map = new HashMap<>(); private FsGetter fsCreator; - InnerCache(FsGetter fsCreator) { + public InnerCache(FsGetter fsCreator) { this.fsCreator = fsCreator; } - FileSystem get(URI uri, Configuration config) throws IOException { + public FileSystem get(URI uri, Configuration config) throws IOException { Key key = new Key(uri); if (map.get(key) == null) { FileSystem fs = fsCreator.getNewInstance(uri, config); @@ -224,12 +205,12 @@ public URI[] getTargetFileSystemURIs() { } } - final long creationTime; // of the the mount table - final UserGroupInformation ugi; // the user/group of user who created mtable - private URI myUri; + protected final long creationTime; // of the the mount table + protected final UserGroupInformation ugi; // the user/group of user who created mtable + protected URI myUri; private Path workingDir; Configuration config; - InodeTree fsState; // the fs state; ie the mount table + protected InodeTree fsState; // the fs state; ie the mount table Path homeDir = null; private boolean enableInnerCache = false; private InnerCache cache; @@ -294,31 +275,7 @@ public void initialize(final URI theUri, final Configuration conf) final String authority = theUri.getAuthority(); try { myUri = new URI(getScheme(), authority, "/", null, null); - fsState = new InodeTree(conf, authority) { - @Override - protected FileSystem getTargetFileSystem(final URI uri) - throws URISyntaxException, IOException { - FileSystem fs; - if (enableInnerCache) { - fs = innerCache.get(uri, config); - } else { - fs = fsGetter.get(uri, config); - } - return new ChRootedFileSystem(fs, uri); - } - - @Override - protected FileSystem getTargetFileSystem(final INodeDir dir) - throws URISyntaxException { - return new InternalDirOfViewFs(dir, creationTime, ugi, myUri, config); - } - - @Override - protected FileSystem getTargetFileSystem(final String settings, - final URI[] uris) throws URISyntaxException, IOException { - return NflyFSystem.createFileSystem(uris, config, settings); - } - }; + buildMountTable(conf, authority, fsGetter, innerCache, enableInnerCache); workingDir = this.getHomeDirectory(); renameStrategy = RenameStrategy.valueOf( conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, @@ -335,6 +292,37 @@ protected FileSystem getTargetFileSystem(final String settings, } } + protected void buildMountTable(final Configuration conf, + final String authority, final ViewFileSystem.FsGetter fsGetter, + final ViewFileSystem.InnerCache innerCache, + final boolean enableInnerCache) throws URISyntaxException, IOException { + fsState = new InodeTree(conf, authority) { + @Override + protected FileSystem getTargetFileSystem(final URI uri) + throws URISyntaxException, IOException { + FileSystem fs; + if (enableInnerCache) { + fs = innerCache.get(uri, config); + } else { + fs = fsGetter.get(uri, config); + } + return new ChRootedFileSystem(fs, uri); + } + + @Override + protected FileSystem getTargetFileSystem(final INodeDir dir) + throws URISyntaxException { + return new InternalDirOfViewFs(dir, creationTime, ugi, myUri, config); + } + + @Override + protected FileSystem getTargetFileSystem(final String settings, + final URI[] uris) throws URISyntaxException, IOException { + return NflyFSystem.createFileSystem(uris, config, settings); + } + }; + } + /** * Convenience Constructor for apps to call directly * @param theUri which must be that of ViewFileSystem @@ -412,7 +400,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, try { res = fsState.resolve(getUriPath(f), false); } catch (FileNotFoundException e) { - throw readOnlyMountTable("create", f); + throw ViewFileSystemUtil.readOnlyMountTable("create", f); } assert(res.remainingPath != null); return res.targetFileSystem.createNonRecursive(res.remainingPath, @@ -427,7 +415,7 @@ public FSDataOutputStream create(final Path f, final FsPermission permission, try { res = fsState.resolve(getUriPath(f), false); } catch (FileNotFoundException e) { - throw readOnlyMountTable("create", f); + throw ViewFileSystemUtil.readOnlyMountTable("create", f); } assert(res.remainingPath != null); return res.targetFileSystem.create(res.remainingPath, permission, @@ -442,7 +430,7 @@ public boolean delete(final Path f, final boolean recursive) fsState.resolve(getUriPath(f), true); // If internal dir or target is a mount link (ie remainingPath is Slash) if (res.isInternalDir() || res.remainingPath == InodeTree.SlashPath) { - throw readOnlyMountTable("delete", f); + throw ViewFileSystemUtil.readOnlyMountTable("delete", f); } return res.targetFileSystem.delete(res.remainingPath, recursive); } @@ -616,13 +604,13 @@ public boolean rename(final Path src, final Path dst) throws IOException { fsState.resolve(getUriPath(src), false); if (resSrc.isInternalDir()) { - throw readOnlyMountTable("rename", src); + throw ViewFileSystemUtil.readOnlyMountTable("rename", src); } InodeTree.ResolveResult resDst = fsState.resolve(getUriPath(dst), false); if (resDst.isInternalDir()) { - throw readOnlyMountTable("rename", dst); + throw ViewFileSystemUtil.readOnlyMountTable("rename", dst); } URI srcUri = resSrc.targetFileSystem.getUri(); @@ -1091,374 +1079,6 @@ public boolean hasPathCapability(Path path, String capability) } } - /** - * An instance of this class represents an internal dir of the viewFs - * that is internal dir of the mount table. - * It is a read only mount tables and create, mkdir or delete operations - * are not allowed. - * If called on create or mkdir then this target is the parent of the - * directory in which one is trying to create or mkdir; hence - * in this case the path name passed in is the last component. - * Otherwise this target is the end point of the path and hence - * the path name passed in is null. - */ - static class InternalDirOfViewFs extends FileSystem { - final InodeTree.INodeDir theInternalDir; - final long creationTime; // of the the mount table - final UserGroupInformation ugi; // the user/group of user who created mtable - final URI myUri; - - public InternalDirOfViewFs(final InodeTree.INodeDir dir, - final long cTime, final UserGroupInformation ugi, URI uri, - Configuration config) throws URISyntaxException { - myUri = uri; - try { - initialize(myUri, config); - } catch (IOException e) { - throw new RuntimeException("Cannot occur"); - } - theInternalDir = dir; - creationTime = cTime; - this.ugi = ugi; - } - - static private void checkPathIsSlash(final Path f) throws IOException { - if (f != InodeTree.SlashPath) { - throw new IOException( - "Internal implementation error: expected file name to be /"); - } - } - - @Override - public URI getUri() { - return myUri; - } - - @Override - public Path getWorkingDirectory() { - throw new RuntimeException( - "Internal impl error: getWorkingDir should not have been called"); - } - - @Override - public void setWorkingDirectory(final Path new_dir) { - throw new RuntimeException( - "Internal impl error: getWorkingDir should not have been called"); - } - - @Override - public FSDataOutputStream append(final Path f, final int bufferSize, - final Progressable progress) throws IOException { - throw readOnlyMountTable("append", f); - } - - @Override - public FSDataOutputStream create(final Path f, - final FsPermission permission, final boolean overwrite, - final int bufferSize, final short replication, final long blockSize, - final Progressable progress) throws AccessControlException { - throw readOnlyMountTable("create", f); - } - - @Override - public boolean delete(final Path f, final boolean recursive) - throws AccessControlException, IOException { - checkPathIsSlash(f); - throw readOnlyMountTable("delete", f); - } - - @Override - @SuppressWarnings("deprecation") - public boolean delete(final Path f) - throws AccessControlException, IOException { - return delete(f, true); - } - - @Override - public BlockLocation[] getFileBlockLocations(final FileStatus fs, - final long start, final long len) throws - FileNotFoundException, IOException { - checkPathIsSlash(fs.getPath()); - throw new FileNotFoundException("Path points to dir not a file"); - } - - @Override - public FileChecksum getFileChecksum(final Path f) - throws FileNotFoundException, IOException { - checkPathIsSlash(f); - throw new FileNotFoundException("Path points to dir not a file"); - } - - @Override - public FileStatus getFileStatus(Path f) throws IOException { - checkPathIsSlash(f); - return new FileStatus(0, true, 0, 0, creationTime, creationTime, - PERMISSION_555, ugi.getShortUserName(), ugi.getPrimaryGroupName(), - - new Path(theInternalDir.fullPath).makeQualified( - myUri, ROOT_PATH)); - } - - - @Override - public FileStatus[] listStatus(Path f) throws AccessControlException, - FileNotFoundException, IOException { - checkPathIsSlash(f); - FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()]; - int i = 0; - for (Entry> iEntry : - theInternalDir.getChildren().entrySet()) { - INode inode = iEntry.getValue(); - if (inode.isLink()) { - INodeLink link = (INodeLink) inode; - - result[i++] = new FileStatus(0, false, 0, 0, - creationTime, creationTime, PERMISSION_555, - ugi.getShortUserName(), ugi.getPrimaryGroupName(), - link.getTargetLink(), - new Path(inode.fullPath).makeQualified( - myUri, null)); - } else { - result[i++] = new FileStatus(0, true, 0, 0, - creationTime, creationTime, PERMISSION_555, - ugi.getShortUserName(), ugi.getGroupNames()[0], - new Path(inode.fullPath).makeQualified( - myUri, null)); - } - } - return result; - } - - @Override - public boolean mkdirs(Path dir, FsPermission permission) - throws AccessControlException, FileAlreadyExistsException { - if (theInternalDir.isRoot() && dir == null) { - throw new FileAlreadyExistsException("/ already exits"); - } - // Note dir starts with / - if (theInternalDir.getChildren().containsKey( - dir.toString().substring(1))) { - return true; // this is the stupid semantics of FileSystem - } - throw readOnlyMountTable("mkdirs", dir); - } - - @Override - public boolean mkdirs(Path dir) - throws AccessControlException, FileAlreadyExistsException { - return mkdirs(dir, null); - } - - @Override - public FSDataInputStream open(Path f, int bufferSize) - throws AccessControlException, FileNotFoundException, IOException { - checkPathIsSlash(f); - throw new FileNotFoundException("Path points to dir not a file"); - } - - @Override - public boolean rename(Path src, Path dst) throws AccessControlException, - IOException { - checkPathIsSlash(src); - checkPathIsSlash(dst); - throw readOnlyMountTable("rename", src); - } - - @Override - public boolean truncate(Path f, long newLength) throws IOException { - throw readOnlyMountTable("truncate", f); - } - - @Override - public void setOwner(Path f, String username, String groupname) - throws AccessControlException, IOException { - checkPathIsSlash(f); - throw readOnlyMountTable("setOwner", f); - } - - @Override - public void setPermission(Path f, FsPermission permission) - throws AccessControlException, IOException { - checkPathIsSlash(f); - throw readOnlyMountTable("setPermission", f); - } - - @Override - public boolean setReplication(Path f, short replication) - throws AccessControlException, IOException { - checkPathIsSlash(f); - throw readOnlyMountTable("setReplication", f); - } - - @Override - public void setTimes(Path f, long mtime, long atime) - throws AccessControlException, IOException { - checkPathIsSlash(f); - throw readOnlyMountTable("setTimes", f); - } - - @Override - public void setVerifyChecksum(boolean verifyChecksum) { - // Noop for viewfs - } - - @Override - public FsServerDefaults getServerDefaults(Path f) throws IOException { - throw new NotInMountpointException(f, "getServerDefaults"); - } - - @Override - public long getDefaultBlockSize(Path f) { - throw new NotInMountpointException(f, "getDefaultBlockSize"); - } - - @Override - public short getDefaultReplication(Path f) { - throw new NotInMountpointException(f, "getDefaultReplication"); - } - - @Override - public void modifyAclEntries(Path path, List aclSpec) - throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("modifyAclEntries", path); - } - - @Override - public void removeAclEntries(Path path, List aclSpec) - throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("removeAclEntries", path); - } - - @Override - public void removeDefaultAcl(Path path) throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("removeDefaultAcl", path); - } - - @Override - public void removeAcl(Path path) throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("removeAcl", path); - } - - @Override - public void setAcl(Path path, List aclSpec) throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("setAcl", path); - } - - @Override - public AclStatus getAclStatus(Path path) throws IOException { - checkPathIsSlash(path); - return new AclStatus.Builder().owner(ugi.getShortUserName()) - .group(ugi.getPrimaryGroupName()) - .addEntries(AclUtil.getMinimalAcl(PERMISSION_555)) - .stickyBit(false).build(); - } - - @Override - public void setXAttr(Path path, String name, byte[] value, - EnumSet flag) throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("setXAttr", path); - } - - @Override - public byte[] getXAttr(Path path, String name) throws IOException { - throw new NotInMountpointException(path, "getXAttr"); - } - - @Override - public Map getXAttrs(Path path) throws IOException { - throw new NotInMountpointException(path, "getXAttrs"); - } - - @Override - public Map getXAttrs(Path path, List names) - throws IOException { - throw new NotInMountpointException(path, "getXAttrs"); - } - - @Override - public List listXAttrs(Path path) throws IOException { - throw new NotInMountpointException(path, "listXAttrs"); - } - - @Override - public void removeXAttr(Path path, String name) throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("removeXAttr", path); - } - - @Override - public Path createSnapshot(Path path, String snapshotName) - throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("createSnapshot", path); - } - - @Override - public void renameSnapshot(Path path, String snapshotOldName, - String snapshotNewName) throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("renameSnapshot", path); - } - - @Override - public void deleteSnapshot(Path path, String snapshotName) - throws IOException { - checkPathIsSlash(path); - throw readOnlyMountTable("deleteSnapshot", path); - } - - @Override - public QuotaUsage getQuotaUsage(Path f) throws IOException { - throw new NotInMountpointException(f, "getQuotaUsage"); - } - - @Override - public void satisfyStoragePolicy(Path src) throws IOException { - checkPathIsSlash(src); - throw readOnlyMountTable("satisfyStoragePolicy", src); - } - - @Override - public void setStoragePolicy(Path src, String policyName) - throws IOException { - checkPathIsSlash(src); - throw readOnlyMountTable("setStoragePolicy", src); - } - - @Override - public void unsetStoragePolicy(Path src) throws IOException { - checkPathIsSlash(src); - throw readOnlyMountTable("unsetStoragePolicy", src); - } - - @Override - public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { - throw new NotInMountpointException(src, "getStoragePolicy"); - } - - @Override - public Collection getAllStoragePolicies() - throws IOException { - Collection allPolicies = new HashSet<>(); - for (FileSystem fs : getChildFileSystems()) { - try { - Collection policies = - fs.getAllStoragePolicies(); - allPolicies.addAll(policies); - } catch (UnsupportedOperationException e) { - // ignored - } - } - return allPolicies; - } - } - enum RenameStrategy { SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT, SAME_FILESYSTEM_ACROSS_MOUNTPOINT diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java index c8a1d78cffd46..330a26fa07b54 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java @@ -29,6 +29,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint; +import org.apache.hadoop.security.AccessControlException; + /** * Utility APIs for ViewFileSystem. @@ -161,4 +163,15 @@ private static void updateMountPointFsStatus( mountPointMap.put(mountPoint, fsStatus); } + static AccessControlException readOnlyMountTable(final String operation, + final String p) { + return new AccessControlException( + "InternalDir of ViewFileSystem is readonly, operation " + operation + + " not permitted on path " + p + "."); + } + static AccessControlException readOnlyMountTable(final String operation, + final Path p) { + return readOnlyMountTable(operation, p.toString()); + } + }