From efd41c17606d3eef1c450681263db0fa6302da31 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 29 Feb 2020 22:50:41 -0600 Subject: [PATCH 1/6] Implementing a 'Multiplexer', suitable for a namespace package which may have several paths on which it is based. --- importlib_resources/abc.py | 49 +++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py index 28596a4a..ac85b69b 100644 --- a/importlib_resources/abc.py +++ b/importlib_resources/abc.py @@ -1,8 +1,9 @@ from __future__ import absolute_import import abc +import itertools -from ._compat import ABC, FileNotFoundError +from ._compat import ABC, FileNotFoundError, suppress # Use mypy's comment syntax for Python 2 compatibility try: @@ -116,6 +117,52 @@ def open(self, mode='r', *args, **kwargs): """ +class Multiplexed(Traversable): + """ + Given a series of Traversable objects, implement a merged + version of the interface across all objects. Useful for + namespace packages which may be multihomed at a single + name. + """ + + def __init__(self, *paths): + self._paths = paths + + def iterdir(self): + return itertools.chain.from_iterable( + path.iterdir() for path in self._paths) + + def read_bytes(self): + for path in self._paths[:-1]: + with suppress(Exception): + return path.read_bytes() + return self._paths[-1].read_bytes() + + def read_text(self, *args, **kwargs): + for path in self._paths[:-1]: + with suppress(Exception): + return path.read_text(*args, **kwargs) + return self._paths[-1].read_text() + + def is_dir(self): + return True + + def is_file(self): + return False + + def joinpath(self, child): + # todo: how to handle subpackages that are themselves multiplexed? + subpackages = () + children = (path.joinpath(child) for path in self._paths) + return next( + child + for child in itertools.chain(subpackages, children) + if child.is_dir() or child.is_file() + ) + + __truediv__ = joinpath + + class TraversableResources(ResourceReader): @abc.abstractmethod def files(self): From a20a0d63c6dc575a94e900a7c3a98f01abad1aae Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 19 Mar 2020 23:25:12 -0400 Subject: [PATCH 2/6] Implement open --- importlib_resources/abc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py index ac85b69b..8bb18349 100644 --- a/importlib_resources/abc.py +++ b/importlib_resources/abc.py @@ -162,6 +162,12 @@ def joinpath(self, child): __truediv__ = joinpath + def open(self, *args, **kwargs): + for path in self._paths[:-1]: + with suppress(Exception): + return path.open(*args, **kwargs) + return self._paths[-1].open(*args, **kwargs) + class TraversableResources(ResourceReader): @abc.abstractmethod From 61f64acc043aa8ef88b5e956891d5ba0c9705467 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 19 Mar 2020 23:27:55 -0400 Subject: [PATCH 3/6] Rely on open in read ops --- importlib_resources/abc.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py index 8bb18349..65af56b8 100644 --- a/importlib_resources/abc.py +++ b/importlib_resources/abc.py @@ -133,16 +133,10 @@ def iterdir(self): path.iterdir() for path in self._paths) def read_bytes(self): - for path in self._paths[:-1]: - with suppress(Exception): - return path.read_bytes() - return self._paths[-1].read_bytes() + return self.open(mode='rb').read() def read_text(self, *args, **kwargs): - for path in self._paths[:-1]: - with suppress(Exception): - return path.read_text(*args, **kwargs) - return self._paths[-1].read_text() + return self.open(mode='r', *args, **kwargs).read() def is_dir(self): return True From 6934f743ca8d0fa10b039d63e9ca082429da71f7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 24 Mar 2020 10:45:13 -0400 Subject: [PATCH 4/6] Update Multiplexed to return Multiplexed objects for subpackages. --- importlib_resources/abc.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py index 65af56b8..1774501f 100644 --- a/importlib_resources/abc.py +++ b/importlib_resources/abc.py @@ -139,20 +139,22 @@ def read_text(self, *args, **kwargs): return self.open(mode='r', *args, **kwargs).read() def is_dir(self): - return True + return any(path.is_dir() for path in self._paths) def is_file(self): - return False + return any(path.is_file() for path in self._paths) def joinpath(self, child): - # todo: how to handle subpackages that are themselves multiplexed? - subpackages = () - children = (path.joinpath(child) for path in self._paths) - return next( + children = ( + path.joinpath(child) + for path in self._paths + ) + existing = ( child - for child in itertools.chain(subpackages, children) + for child in children if child.is_dir() or child.is_file() ) + return Multiplexed(*existing) __truediv__ = joinpath From 4eb6788d9f854677791fe26d95ff40986db8d80c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Jun 2020 21:26:23 -0400 Subject: [PATCH 5/6] Move Multiplexed to its own module --- importlib_resources/abc.py | 51 +------------------------------- importlib_resources/namespace.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 50 deletions(-) create mode 100644 importlib_resources/namespace.py diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py index 1774501f..28596a4a 100644 --- a/importlib_resources/abc.py +++ b/importlib_resources/abc.py @@ -1,9 +1,8 @@ from __future__ import absolute_import import abc -import itertools -from ._compat import ABC, FileNotFoundError, suppress +from ._compat import ABC, FileNotFoundError # Use mypy's comment syntax for Python 2 compatibility try: @@ -117,54 +116,6 @@ def open(self, mode='r', *args, **kwargs): """ -class Multiplexed(Traversable): - """ - Given a series of Traversable objects, implement a merged - version of the interface across all objects. Useful for - namespace packages which may be multihomed at a single - name. - """ - - def __init__(self, *paths): - self._paths = paths - - def iterdir(self): - return itertools.chain.from_iterable( - path.iterdir() for path in self._paths) - - def read_bytes(self): - return self.open(mode='rb').read() - - def read_text(self, *args, **kwargs): - return self.open(mode='r', *args, **kwargs).read() - - def is_dir(self): - return any(path.is_dir() for path in self._paths) - - def is_file(self): - return any(path.is_file() for path in self._paths) - - def joinpath(self, child): - children = ( - path.joinpath(child) - for path in self._paths - ) - existing = ( - child - for child in children - if child.is_dir() or child.is_file() - ) - return Multiplexed(*existing) - - __truediv__ = joinpath - - def open(self, *args, **kwargs): - for path in self._paths[:-1]: - with suppress(Exception): - return path.open(*args, **kwargs) - return self._paths[-1].open(*args, **kwargs) - - class TraversableResources(ResourceReader): @abc.abstractmethod def files(self): diff --git a/importlib_resources/namespace.py b/importlib_resources/namespace.py new file mode 100644 index 00000000..55614bbf --- /dev/null +++ b/importlib_resources/namespace.py @@ -0,0 +1,51 @@ +import itertools +from ._compat import suppress +from .abc import Traversable + + +class Multiplexed(Traversable): + """ + Given a series of Traversable objects, implement a merged + version of the interface across all objects. Useful for + namespace packages which may be multihomed at a single + name. + """ + + def __init__(self, *paths): + self._paths = paths + + def iterdir(self): + return itertools.chain.from_iterable( + path.iterdir() for path in self._paths) + + def read_bytes(self): + return self.open(mode='rb').read() + + def read_text(self, *args, **kwargs): + return self.open(mode='r', *args, **kwargs).read() + + def is_dir(self): + return any(path.is_dir() for path in self._paths) + + def is_file(self): + return any(path.is_file() for path in self._paths) + + def joinpath(self, child): + children = ( + path.joinpath(child) + for path in self._paths + ) + existing = ( + child + for child in children + if child.is_dir() or child.is_file() + ) + return Multiplexed(*existing) + + __truediv__ = joinpath + + def open(self, *args, **kwargs): + for path in self._paths[:-1]: + with suppress(Exception): + return path.open(*args, **kwargs) + return self._paths[-1].open(*args, **kwargs) From 501190d20a33900ffb1d4fecfd505609ff863bd8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Jun 2020 22:20:55 -0400 Subject: [PATCH 6/6] Employ Multiplexer in _compat.LoaderAdapter.get_resource_reader --- importlib_resources/_compat.py | 7 +++++++ importlib_resources/namespace.py | 21 ++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/importlib_resources/_compat.py b/importlib_resources/_compat.py index 64773f73..becff6f7 100644 --- a/importlib_resources/_compat.py +++ b/importlib_resources/_compat.py @@ -106,9 +106,16 @@ def _native_reader(spec): reader = _available_reader(spec) return reader if hasattr(reader, 'files') else None + def _namespace_reader(spec): + from . import namespace + if 'NamespaceLoader' not in spec.loader.__class__.__name__: + return + return namespace.Multiplexed.load(spec.submodule_search_locations) + return ( # native reader if it supplies 'files' _native_reader(self.spec) or + _namespace_reader(self.spec) or # local ZipReader if a zip module _zip_reader(self.spec) or # local FileReader diff --git a/importlib_resources/namespace.py b/importlib_resources/namespace.py index 55614bbf..e664aa87 100644 --- a/importlib_resources/namespace.py +++ b/importlib_resources/namespace.py @@ -1,8 +1,24 @@ +import os import itertools -from ._compat import suppress +from ._compat import suppress, ZipPath, Path from .abc import Traversable +def infer_path(path): + return resolve_zip_path(path) or Path(path) + + +def resolve_zip_path(candidate, tail=''): + if not candidate: + return + try: + return ZipPath(candidate, at=tail) + except Exception: + new_tail = os.path.basename(candidate) + '/' + tail + new_base = os.path.dirname(candidate) + return resolve_zip_path(new_base, new_tail) + + class Multiplexed(Traversable): """ Given a series of Traversable objects, implement a merged @@ -10,6 +26,9 @@ class Multiplexed(Traversable): namespace packages which may be multihomed at a single name. """ + @classmethod + def load(cls, paths): + return cls(map(cls._infer_path, paths)) def __init__(self, *paths): self._paths = paths