Merge pull request #3594 from pytest-dev/interal-pathlib

[WIP] port cache plugin internals to pathlib

Author: nicoddemus

appveyor.yml

@@ -14,7 +14,7 @@ environment:
   - TOXENV: "py34"
   - TOXENV: "py35"
   - TOXENV: "py36"
-  - TOXENV: "pypy"
+#  - TOXENV: "pypy" reenable when we are able to provide a scandir wheel or build scandir
   - TOXENV: "py27-pexpect"
   - TOXENV: "py27-xdist"
   - TOXENV: "py27-trial"

setup.py

@@ -73,15 +73,19 @@ def main():
     environment_marker_support_level = get_environment_marker_support_level()
     if environment_marker_support_level >= 2:
         install_requires.append('funcsigs;python_version<"3.0"')
+        install_requires.append('pathlib2;python_version<"3.6"')
         install_requires.append('colorama;sys_platform=="win32"')
     elif environment_marker_support_level == 1:
         extras_require[':python_version<"3.0"'] = ["funcsigs"]
+        extras_require[':python_version<"3.6"'] = ["pathlib2"]
         extras_require[':sys_platform=="win32"'] = ["colorama"]
     else:
         if sys.platform == "win32":
             install_requires.append("colorama")
         if sys.version_info < (3, 0):
             install_requires.append("funcsigs")
+        if sys.version_info < (3, 6):
+            install_requires.append("pathlib2")
 
     setup(
         name="pytest",

src/_pytest/cacheprovider.py

@@ -5,40 +5,51 @@
 ignores the external pytest-cache
 """
 from __future__ import absolute_import, division, print_function
-
 from collections import OrderedDict
 
 import py
 import six
+import attr
 
 import pytest
 import json
-import os
-from os.path import sep as _sep, altsep as _altsep
-from textwrap import dedent
+import shutil
+
+from . import paths
+from .compat import _PY2 as PY2, Path
+
+README_CONTENT = u"""\
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/latest/cache.html) for more information.
+"""
 
 
+@attr.s
 class Cache(object):
 
-    def __init__(self, config):
-        self.config = config
-        self._cachedir = Cache.cache_dir_from_config(config)
-        self.trace = config.trace.root.get("cache")
-        if config.getoption("cacheclear"):
-            self.trace("clearing cachedir")
-            if self._cachedir.check():
-                self._cachedir.remove()
-            self._cachedir.mkdir()
+    _cachedir = attr.ib(repr=False)
+    _warn = attr.ib(repr=False)
+
+    @classmethod
+    def for_config(cls, config):
+        cachedir = cls.cache_dir_from_config(config)
+        if config.getoption("cacheclear") and cachedir.exists():
+            shutil.rmtree(str(cachedir))
+            cachedir.mkdir()
+        return cls(cachedir, config.warn)
 
     @staticmethod
     def cache_dir_from_config(config):
-        cache_dir = config.getini("cache_dir")
-        cache_dir = os.path.expanduser(cache_dir)
-        cache_dir = os.path.expandvars(cache_dir)
-        if os.path.isabs(cache_dir):
-            return py.path.local(cache_dir)
-        else:
-            return config.rootdir.join(cache_dir)
+        return paths.resolve_from_str(config.getini("cache_dir"), config.rootdir)
+
+    def warn(self, fmt, **args):
+        self._warn(code="I9", message=fmt.format(**args) if args else fmt)
 
     def makedir(self, name):
         """ return a directory path object with the given name.  If the
@@ -50,12 +61,15 @@ def makedir(self, name):
              Make sure the name contains your plugin or application
              identifiers to prevent clashes with other cache users.
         """
-        if _sep in name or _altsep is not None and _altsep in name:
+        name = Path(name)
+        if len(name.parts) > 1:
             raise ValueError("name is not allowed to contain path separators")
-        return self._cachedir.ensure_dir("d", name)
+        res = self._cachedir.joinpath("d", name)
+        res.mkdir(exist_ok=True, parents=True)
+        return py.path.local(res)
 
     def _getvaluepath(self, key):
-        return self._cachedir.join("v", *key.split("/"))
+        return self._cachedir.joinpath("v", Path(key))
 
     def get(self, key, default):
         """ return cached value for the given key.  If no value
@@ -69,13 +83,11 @@ def get(self, key, default):
 
         """
         path = self._getvaluepath(key)
-        if path.check():
-            try:
-                with path.open("r") as f:
-                    return json.load(f)
-            except ValueError:
-                self.trace("cache-invalid at %s" % (path,))
-        return default
+        try:
+            with path.open("r") as f:
+                return json.load(f)
+        except (ValueError, IOError, OSError):
+            return default
 
     def set(self, key, value):
         """ save value for the given key.
@@ -88,42 +100,25 @@ def set(self, key, value):
         """
         path = self._getvaluepath(key)
         try:
-            path.dirpath().ensure_dir()
-        except (py.error.EEXIST, py.error.EACCES):
-            self.config.warn(
-                code="I9", message="could not create cache path %s" % (path,)
-            )
+            path.parent.mkdir(exist_ok=True, parents=True)
+        except (IOError, OSError):
+            self.warn("could not create cache path {path}", path=path)
             return
         try:
-            f = path.open("w")
-        except py.error.ENOTDIR:
-            self.config.warn(
-                code="I9", message="cache could not write path %s" % (path,)
-            )
+            f = path.open("wb" if PY2 else "w")
+        except (IOError, OSError):
+            self.warn("cache could not write path {path}", path=path)
         else:
             with f:
-                self.trace("cache-write %s: %r" % (key, value))
                 json.dump(value, f, indent=2, sort_keys=True)
                 self._ensure_readme()
 
     def _ensure_readme(self):
 
-        content_readme = dedent(
-            """\
-        # pytest cache directory #
-
-        This directory contains data from the pytest's cache plugin,
-        which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
-
-        **Do not** commit this to version control.
-
-        See [the docs](https://docs.pytest.org/en/latest/cache.html) for more information.
-        """
-        )
-        if self._cachedir.check(dir=True):
-            readme_path = self._cachedir.join("README.md")
-            if not readme_path.check(file=True):
-                readme_path.write(content_readme)
+        if self._cachedir.is_dir():
+            readme_path = self._cachedir / "README.md"
+            if not readme_path.is_file():
+                readme_path.write_text(README_CONTENT)
 
 
 class LFPlugin(object):
@@ -297,7 +292,7 @@ def pytest_cmdline_main(config):
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_configure(config):
-    config.cache = Cache(config)
+    config.cache = Cache.for_config(config)
     config.pluginmanager.register(LFPlugin(config), "lfplugin")
     config.pluginmanager.register(NFPlugin(config), "nfplugin")
 
@@ -320,41 +315,40 @@ def cache(request):
 
 def pytest_report_header(config):
     if config.option.verbose:
-        relpath = py.path.local().bestrelpath(config.cache._cachedir)
-        return "cachedir: %s" % relpath
+        relpath = config.cache._cachedir.relative_to(config.rootdir)
+        return "cachedir: {}".format(relpath)
 
 
 def cacheshow(config, session):
-    from pprint import pprint
+    from pprint import pformat
 
     tw = py.io.TerminalWriter()
     tw.line("cachedir: " + str(config.cache._cachedir))
-    if not config.cache._cachedir.check():
+    if not config.cache._cachedir.is_dir():
         tw.line("cache is empty")
         return 0
     dummy = object()
     basedir = config.cache._cachedir
-    vdir = basedir.join("v")
+    vdir = basedir / "v"
     tw.sep("-", "cache values")
-    for valpath in sorted(vdir.visit(lambda x: x.isfile())):
-        key = valpath.relto(vdir).replace(valpath.sep, "/")
+    for valpath in sorted(x for x in vdir.rglob("*") if x.is_file()):
+        key = valpath.relative_to(vdir)
         val = config.cache.get(key, dummy)
         if val is dummy:
             tw.line("%s contains unreadable content, " "will be ignored" % key)
         else:
             tw.line("%s contains:" % key)
-            stream = py.io.TextIO()
-            pprint(val, stream=stream)
-            for line in stream.getvalue().splitlines():
+            for line in pformat(val).splitlines():
                 tw.line("  " + line)
 
-    ddir = basedir.join("d")
-    if ddir.isdir() and ddir.listdir():
+    ddir = basedir / "d"
+    if ddir.is_dir():
+        contents = sorted(ddir.rglob("*"))
         tw.sep("-", "cache directories")
-        for p in sorted(basedir.join("d").visit()):
+        for p in contents:
             # if p.check(dir=1):
             #    print("%s/" % p.relto(basedir))
-            if p.isfile():
-                key = p.relto(basedir)
-                tw.line("%s is a file of length %d" % (key, p.size()))
+            if p.is_file():
+                key = p.relative_to(basedir)
+                tw.line("{} is a file of length {:d}".format(key, p.stat().st_size))
     return 0

src/_pytest/compat.py

@@ -22,6 +22,7 @@
     # Only available in Python 3.4+ or as a backport
     enum = None
 
+__all__ = ["Path"]
 
 _PY3 = sys.version_info > (3, 0)
 _PY2 = not _PY3
@@ -32,14 +33,19 @@
 else:
     from funcsigs import signature, Parameter as Parameter
 
-
 NoneType = type(None)
 NOTSET = object()
 
 PY35 = sys.version_info[:2] >= (3, 5)
 PY36 = sys.version_info[:2] >= (3, 6)
 MODULE_NOT_FOUND_ERROR = "ModuleNotFoundError" if PY36 else "ImportError"
 
+if PY36:
+    from pathlib import Path
+else:
+    from pathlib2 import Path
+
+
 if _PY3:
     from collections.abc import MutableMapping as MappingMixin  # noqa
     from collections.abc import Mapping, Sequence  # noqa

src/_pytest/paths.py

@@ -0,0 +1,13 @@
+from .compat import Path
+from os.path import expanduser, expandvars, isabs
+
+
+def resolve_from_str(input, root):
+    assert not isinstance(input, Path), "would break on py2"
+    root = Path(root)
+    input = expanduser(input)
+    input = expandvars(input)
+    if isabs(input):
+        return Path(input)
+    else:
+        return root.joinpath(input)

testing/test_cacheprovider.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import, division, print_function
+
 import sys
+
 import py
 import _pytest
 import pytest
@@ -26,7 +28,7 @@ def test_config_cache_dataerror(self, testdir):
         cache = config.cache
         pytest.raises(TypeError, lambda: cache.set("key/name", cache))
         config.cache.set("key/name", 0)
-        config.cache._getvaluepath("key/name").write("123invalid")
+        config.cache._getvaluepath("key/name").write_bytes(b"123invalid")
         val = config.cache.get("key/name", -2)
         assert val == -2
 
@@ -824,8 +826,8 @@ class TestReadme(object):
 
     def check_readme(self, testdir):
         config = testdir.parseconfigure()
-        readme = config.cache._cachedir.join("README.md")
-        return readme.isfile()
+        readme = config.cache._cachedir.joinpath("README.md")
+        return readme.is_file()
 
     def test_readme_passed(self, testdir):
         testdir.makepyfile(