Port pip requirements.txt parser

.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     rev: "0.5.0"
     hooks:
       - id: tox-ini-fmt
-        args: [ "-p", "fix,flake8" ]
+        args: [ "-p", "fix" ]
   - repo: https://github.com/asottile/blacken-docs
     rev: v1.10.0
     hooks:
@@ -59,3 +59,13 @@ repos:
         entry: "changelog files must be named ####.(feature|bugfix|doc|removal|misc).rst"
         exclude: ^docs/changelog/(\d+\.(feature|bugfix|doc|removal|misc).rst|README.rst|template.jinja2)
         files: ^docs/changelog/
+  - repo: https://github.com/PyCQA/flake8
+    rev: 3.9.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+        - flake8-bugbear==21.4.3
+        - flake8-comprehensions==3.4
+        - flake8-pytest-style==1.4.1
+        - flake8-spellcheck==0.24
+        - flake8-unused-arguments==0.0.6

docs/changelog/1929.bugfix.rst

@@ -0,0 +1,2 @@
+Port pip requirements file parser to ``tox`` to achieve full equivalency (such as support for the per requirement
+``--install-option`` and ``--global-option`` flags) - by :user:`gaborbernat`.

setup.cfg

@@ -38,6 +38,7 @@ packages = find:
 install_requires =
     appdirs>=1.4.3
     cachetools
+    chardet>=4
     colorama>=0.4.3
     packaging>=20.3
     pluggy>=0.13.1

src/tox/config/loader/stringify.py

@@ -29,7 +29,7 @@ def stringify(value: Any) -> Tuple[str, bool]:
         env_var_keys = sorted(value)
         return stringify({k: value.load(k) for k in env_var_keys})
     if isinstance(value, PythonDeps):
-        return stringify([next(iter(v.keys())) if isinstance(v, dict) else v for v in value.validate_and_expand()])
+        return stringify(value.lines())
     return str(value), False
 
 

src/tox/tox_env/package.py

@@ -25,6 +25,9 @@ def __init__(self, path: Path) -> None:
         super().__init__()
         self.path = path
 
+    def __str__(self) -> str:
+        return str(self.path)
+
 
 class PackageToxEnv(ToxEnv, ABC):
     def __init__(

src/tox/tox_env/python/pip/pip_install.py

@@ -1,27 +1,20 @@
 import logging
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Sequence, Set, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
 from packaging.requirements import Requirement
 
 from tox.config.cli.parser import DEFAULT_VERBOSITY
 from tox.config.main import Config
 from tox.config.types import Command
 from tox.execute.request import StdinSource
+from tox.report import HandledError
 from tox.tox_env.errors import Recreate
 from tox.tox_env.installer import Installer
 from tox.tox_env.package import PathPackage
 from tox.tox_env.python.api import Python
 from tox.tox_env.python.package import DevLegacyPackage, SdistPackage, WheelPackage
-from tox.tox_env.python.pip.req_file import (
-    ConstraintFile,
-    EditablePathReq,
-    Flags,
-    PathReq,
-    PythonDeps,
-    RequirementsFile,
-    UrlReq,
-)
+from tox.tox_env.python.pip.req_file import PythonDeps
 
 
 class Pip(Installer[Python]):
@@ -89,64 +82,37 @@ def install(self, arguments: Any, section: str, of_type: str) -> None:
             raise SystemExit(1)
 
     def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None:
-        result = arguments.validate_and_expand()
-        new_set = arguments.unroll()
-        # content we can have here in a nested fashion
-        # the entire universe does not resolve anymore, therefore we only cache the first level
-        # root level -> Union[Flags, Requirement, PathReq, EditablePathReq, UrlReq, ConstraintFile, RequirementsFile]
-        # if the constraint file changes recreate
-        with self._env.cache.compare(new_set, section, of_type) as (eq, old):
-            if not eq:  # pick all options and constraint files, do not pick other equal requirements
-                new_deps: List[str] = []
-                found: Set[int] = set()
-                has_dep = False
-                for entry, as_cache in zip(result, new_set):
-                    entry_as_str = str(entry)
-                    found_pos = None
-                    for at_pos, value in enumerate(old or []):
-                        if (next(iter(value)) if isinstance(value, dict) else value) == entry_as_str:
-                            found_pos = at_pos
-                            break
-                    if found_pos is not None:
-                        found.add(found_pos)
-                    if isinstance(entry, Flags):
-                        if found_pos is None and old is not None:
-                            raise Recreate(f"new flag {entry}")
-                        new_deps.extend(entry.as_args())
-                    elif isinstance(entry, Requirement):
-                        if found_pos is None:
-                            has_dep = True
-                            new_deps.append(str(entry))
-                    elif isinstance(entry, (PathReq, EditablePathReq, UrlReq)):
-                        if found_pos is None:
-                            has_dep = True
-                            new_deps.extend(entry.as_args())
-                    elif isinstance(entry, ConstraintFile):
-                        if found_pos is None and old is not None:
-                            raise Recreate(f"new constraint file {entry}")
-                        if old is not None and old[found_pos] != as_cache:
-                            raise Recreate(f"constraint file {entry.rel_path} changed")
-                        new_deps.extend(entry.as_args())
-                    elif isinstance(entry, RequirementsFile):
-                        if found_pos is None:
-                            has_dep = True
-                            new_deps.extend(entry.as_args())
-                        elif old is not None and old[found_pos] != as_cache:
-                            raise Recreate(f"requirements file {entry.rel_path} changed")
-                    else:
-                        # can only happen when we introduce new content and we don't handle it in any of the branches
-                        logging.warning(f"pip cannot install {entry!r}")  # pragma: no cover
-                        raise SystemExit(1)  # pragma: no cover
-                if len(found) != len(old or []):
-                    missing = " ".join(
-                        (next(iter(o)) if isinstance(o, dict) else o) for i, o in enumerate(old or []) if i not in found
-                    )
-                    raise Recreate(f"dependencies removed: {missing}")
-                if new_deps:
-                    if not has_dep:
-                        logging.warning(f"no dependencies for tox env {self._env.name} within {of_type}")
-                        raise SystemExit(1)
-                    self._execute_installer(new_deps, of_type)
+        try:
+            new_options, new_reqs = arguments.unroll()
+        except ValueError as exception:
+            raise HandledError(f"{exception} for tox env py within deps")
+        new_requirements: List[str] = []
+        new_constraints: List[str] = []
+        for req in new_reqs:
+            (new_constraints if req.startswith("-c ") else new_requirements).append(req)
+        new = {"options": new_options, "requirements": new_requirements, "constraints": new_constraints}
+        # if option or constraint change in any way recreate, if the requirements change only if some are removed
+        with self._env.cache.compare(new, section, of_type) as (eq, old):
+            if not eq:
+                if old is not None:
+                    self._recreate_if_diff("install flag(s)", new_options, old["options"], lambda i: i)
+                    self._recreate_if_diff("constraint(s)", new_constraints, old["constraints"], lambda i: i[3:])
+                    missing_requirement = set(old["requirements"]) - set(new_requirements)
+                    if missing_requirement:
+                        raise Recreate(f"requirements removed: {' '.join(missing_requirement)}")
+                args = arguments.as_args()
+                if args:
+                    self._execute_installer(args, of_type)
+
+    @staticmethod
+    def _recreate_if_diff(of_type: str, new_opts: List[str], old_opts: List[str], fmt: Callable[[str], str]) -> None:
+        if old_opts == new_opts:
+            return
+        removed_opts = set(old_opts) - set(new_opts)
+        removed = f" removed {', '.join(sorted(fmt(i) for i in removed_opts))}" if removed_opts else ""
+        added_opts = set(new_opts) - set(old_opts)
+        added = f" added {', '.join(sorted(fmt(i) for i in added_opts))}" if added_opts else ""
+        raise Recreate(f"changed {of_type}{removed}{added}")
 
     def _install_list_of_deps(
         self,

src/tox/tox_env/python/pip/req/__init__.py

@@ -0,0 +1,5 @@
+"""
+Specification is defined within pip itself and documented under:
+- https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
+- https://github.com/pypa/pip/blob/master/src/pip/_internal/req/constructors.py#L291
+"""

src/tox/tox_env/python/pip/req/args.py

@@ -0,0 +1,88 @@
+import bisect
+import re
+from argparse import Action, ArgumentParser, ArgumentTypeError, Namespace
+from typing import IO, Any, NoReturn, Optional, Sequence, Union
+
+
+class _OurArgumentParser(ArgumentParser):
+    def print_usage(self, file: Optional[IO[str]] = None) -> None:  # noqa: U100
+        """"""
+
+    def exit(self, status: int = 0, message: Optional[str] = None) -> NoReturn:  # noqa: U100
+        message = "" if message is None else message
+        msg = message.lstrip(": ").rstrip()
+        if msg.startswith("error: "):
+            msg = msg[len("error: ") :]
+        raise ValueError(msg)
+
+
+def build_parser(cli_only: bool) -> ArgumentParser:
+    parser = _OurArgumentParser(add_help=False, prog="")
+    _global_options(parser)
+    _req_options(parser, cli_only)
+    return parser
+
+
+def _global_options(parser: ArgumentParser) -> None:
+    parser.add_argument("-i", "--index-url", "--pypi-url", dest="index_url", default=None)
+    parser.add_argument("--extra-index-url", action=AddUniqueAction)
+    parser.add_argument("--no-index", action="store_true", default=False)
+    parser.add_argument("-c", "--constraint", action=AddUniqueAction, dest="constraints")
+    parser.add_argument("-r", "--requirement", action=AddUniqueAction, dest="requirements")
+    parser.add_argument("-e", "--editable", action=AddUniqueAction, dest="editables")
+    parser.add_argument("-f", "--find-links", action=AddUniqueAction)
+    parser.add_argument("--no-binary", choices=[":all:", ":none:"])  # TODO: colon separated package names
+    parser.add_argument("--only-binary", choices=[":all:", ":none:"])  # TODO: colon separated package names
+    parser.add_argument("--prefer-binary", action="store_true", default=False)
+    parser.add_argument("--require-hashes", action="store_true", default=False)
+    parser.add_argument("--pre", action="store_true", default=False)
+    parser.add_argument("--trusted-host", action=AddSortedUniqueAction)
+    parser.add_argument(
+        "--use-feature", choices=["2020-resolver", "fast-deps"], action=AddSortedUniqueAction, dest="features_enabled"
+    )
+
+
+def _req_options(parser: ArgumentParser, cli_only: bool) -> None:
+    parser.add_argument("--install-option", action=AddSortedUniqueAction)
+    parser.add_argument("--global-option", action=AddSortedUniqueAction)
+    if not cli_only:
+        parser.add_argument("--hash", action=AddSortedUniqueAction, type=_validate_hash)
+
+
+_HASH = re.compile(r"sha(256:[a-z0-9]{64}|384:[a-z0-9]{96}|521:[a-z0-9]{128})")
+
+
+def _validate_hash(value: str) -> str:
+    if not _HASH.fullmatch(value):
+        raise ArgumentTypeError(value)
+    return value
+
+
+class AddSortedUniqueAction(Action):
+    def __call__(
+        self,
+        parser: ArgumentParser,  # noqa
+        namespace: Namespace,
+        values: Union[str, Sequence[Any], None],
+        option_string: Optional[str] = None,  # noqa: U100
+    ) -> None:
+        if getattr(namespace, self.dest, None) is None:
+            setattr(namespace, self.dest, [])
+        current = getattr(namespace, self.dest)
+        if values not in current:
+            bisect.insort(current, values)
+
+
+class AddUniqueAction(Action):
+    def __call__(
+        self,
+        parser: ArgumentParser,  # noqa
+        namespace: Namespace,
+        values: Union[str, Sequence[Any], None],
+        option_string: Optional[str] = None,  # noqa: U100
+    ) -> None:
+        if getattr(namespace, self.dest, None) is None:
+            setattr(namespace, self.dest, [])
+        current = getattr(namespace, self.dest)
+        if values not in current:
+            current.append(values)