make dirFS first class (#1201)

Author: martindurant

fsspec/implementations/dirfs.py

@@ -1,17 +1,35 @@
+from .. import filesystem
 from ..asyn import AsyncFileSystem
 
 
 class DirFileSystem(AsyncFileSystem):
-    def __init__(self, path, fs, *args, **storage_options):
+    def __init__(
+        self,
+        path=None,
+        fs=None,
+        fo=None,
+        target_protocol=None,
+        target_options=None,
+        **storage_options,
+    ):
         """
         Parameters
         ----------
         path: str
             Path to the directory.
         fs: AbstractFileSystem
             An instantiated filesystem to wrap.
+        target_protocol, target_options:
+            if fs is none, construct it from these
+        fo: str
+            Alternate for path; do not provide both
         """
-        super().__init__(*args, **storage_options)
+        super().__init__(**storage_options)
+        if fs is None:
+            fs = filesystem(protocol=target_protocol, **(target_options or {}))
+        if (path is not None) ^ (fo is not None) is False:
+            raise ValueError("Provide path or fo, not both")
+        path = path or fo
 
         if self.asynchronous and not fs.async_impl:
             raise ValueError("can't use asynchronous with non-async fs")
@@ -198,20 +216,26 @@ def info(self, path, **kwargs):
         return self.fs.info(self._join(path), **kwargs)
 
     async def _ls(self, path, detail=True, **kwargs):
-        ret = await self.fs._ls(self._join(path), detail=detail, **kwargs)
+        ret = (await self.fs._ls(self._join(path), detail=detail, **kwargs)).copy()
         if detail:
+            out = []
             for entry in ret:
+                entry = entry.copy()
                 entry["name"] = self._relpath(entry["name"])
-            return ret
+                out.append(entry)
+            return out
 
         return self._relpath(ret)
 
     def ls(self, path, detail=True, **kwargs):
-        ret = self.fs.ls(self._join(path), detail=detail, **kwargs)
+        ret = self.fs.ls(self._join(path), detail=detail, **kwargs).copy()
         if detail:
+            out = []
             for entry in ret:
+                entry = entry.copy()
                 entry["name"] = self._relpath(entry["name"])
-            return ret
+                out.append(entry)
+            return out
 
         return self._relpath(ret)
 

fsspec/implementations/tests/test_dirfs.py

@@ -559,3 +559,13 @@ def test_open(mocker, dirfs):
     dirfs.fs.open = mocker.Mock()
     assert dirfs.open("file", *ARGS, **KWARGS) == dirfs.fs.open.return_value
     dirfs.fs.open.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+def test_from_url(m):
+    from fsspec.core import url_to_fs
+
+    m.pipe("inner/file", b"data")
+    fs, _ = url_to_fs("dir::memory://inner")
+    assert fs.ls("", False) == ["file"]
+    assert fs.ls("", True)[0]["name"] == "file"
+    assert fs.cat("file") == b"data"

fsspec/registry.py

@@ -185,6 +185,7 @@ def register_implementation(name, cls, clobber=True, errtxt=None):
         + " Note: 'root' is the protocol name for xrootd storage systems,"
         + " not referring to root directories",
     },
+    "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"},
 }