From d9e413e3e99922c929f180ea111b603706f8ff4c Mon Sep 17 00:00:00 2001 From: Avasam Date: Sat, 8 Mar 2025 21:10:58 -0500 Subject: [PATCH 1/8] Generate `distutils-stubs` on install --- mypy.ini | 10 +--------- newsfragments/4861.feature.rst | 1 + pyrightconfig.json | 2 ++ setup.py | 21 +++++++++++++++++++++ setuptools/__init__.py | 9 ++------- setuptools/build_meta.py | 4 ++-- setuptools/command/build_ext.py | 2 +- setuptools/command/sdist.py | 2 +- setuptools/command/setopt.py | 2 +- setuptools/config/setupcfg.py | 5 ++--- setuptools/errors.py | 8 ++++---- setuptools/extension.py | 2 +- setuptools/logging.py | 2 +- setuptools/modified.py | 2 +- setuptools/monkey.py | 6 +++--- 15 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 newsfragments/4861.feature.rst diff --git a/mypy.ini b/mypy.ini index c1d01a42c3..b294995c25 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,7 +5,7 @@ strict = False # Early opt-in even when strict = False -# warn_unused_ignores = True # Disabled until we have distutils stubs for Python 3.12+ +# warn_unused_ignores = True # Disabled as long as there's inconsistent typing issues between pypa and stdlib's distutils warn_redundant_casts = True enable_error_code = ignore-without-code @@ -48,14 +48,6 @@ disable_error_code = [mypy-pkg_resources.tests.*] disable_error_code = import-not-found -# - distutils doesn't exist on Python 3.12, unfortunately, this means typing -# will be missing for subclasses of distutils on Python 3.12 until either: -# - support for `SETUPTOOLS_USE_DISTUTILS=stdlib` is dropped (#3625) -# for setuptools to import `_distutils` directly -# - or non-stdlib distutils typings are exposed -[mypy-distutils.*] -ignore_missing_imports = True - # - wheel: does not intend on exposing a programmatic API https://github.com/pypa/wheel/pull/610#issuecomment-2081687671 [mypy-wheel.*] follow_untyped_imports = True diff --git a/newsfragments/4861.feature.rst b/newsfragments/4861.feature.rst new file mode 100644 index 0000000000..37541b8599 --- /dev/null +++ b/newsfragments/4861.feature.rst @@ -0,0 +1 @@ +``setuptools`` now provide its own ``distutils-stubs`` instead of relying on typeshed -- by :user:`Avasam` diff --git a/pyrightconfig.json b/pyrightconfig.json index da3cd978ce..6123957847 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -12,6 +12,8 @@ ], // Our testing setup doesn't allow passing CLI arguments, so local devs have to set this manually. // "pythonVersion": "3.9", + // Allow using distutils-stubs on Python 3.12+ + "reportMissingModuleSource": false, // For now we don't mind if mypy's `type: ignore` comments accidentally suppresses pyright issues "enableTypeIgnoreComments": true, "typeCheckingMode": "basic", diff --git a/setup.py b/setup.py index c28a14e722..97406ac8c8 100755 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ import os import sys import textwrap +from pathlib import Path import setuptools from setuptools.command.install import install @@ -37,6 +38,22 @@ def pypi_link(pkg_filename): return '/'.join(parts) +vendored_distutils_path = Path(here) / "setuptools" / "_distutils" + + +def generate_distutils_stubs(destination: Path) -> None: + for path in vendored_distutils_path.rglob("*.py"): + relative_path = path.relative_to(vendored_distutils_path) + if relative_path.parts[0] == "tests": + continue + stub_path = (destination / relative_path).with_suffix(".pyi") + stub_path.parent.mkdir(parents=True, exist_ok=True) + module = "setuptools._distutils." + str(relative_path.with_suffix("")).replace( + os.sep, "." + ).removesuffix(".__init__") + stub_path.write_text(f"from {module} import *\n") + + class install_with_pth(install): """ Custom install command to install a .pth file for distutils patching. @@ -68,6 +85,10 @@ def initialize_options(self): install.initialize_options(self) self.extra_path = self._pth_name, self._pth_contents + def run(self): + install.run(self) + generate_distutils_stubs(Path(self.install_lib) / 'distutils-stubs') + def finalize_options(self): install.finalize_options(self) self._restore_install_lib() diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 64464dfaa3..cd4d5bb37f 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,9 +1,4 @@ """Extensions to the 'distutils' for large or complex distributions""" -# mypy: disable_error_code=override -# Command.reinitialize_command has an extra **kw param that distutils doesn't have -# Can't disable on the exact line because distutils doesn't exists on Python 3.12 -# and mypy isn't aware of distutils_hack, causing distutils.core.Command to be Any, -# and a [unused-ignore] to be raised on 3.12+ from __future__ import annotations @@ -211,7 +206,7 @@ def ensure_string_list(self, option: str) -> None: f"'{option}' must be a list of strings (got {val!r})" ) - @overload + @overload # type: ignore[override] # extra **kw param that distutils doesn't have def reinitialize_command( self, command: str, reinit_subcommands: bool = False, **kw ) -> _Command: ... @@ -224,7 +219,7 @@ def reinitialize_command( ) -> _Command: cmd = _Command.reinitialize_command(self, command, reinit_subcommands) vars(cmd).update(kw) - return cmd # pyright: ignore[reportReturnType] # pypa/distutils#307 + return cmd @abstractmethod def initialize_options(self) -> None: diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 00fa5e1f70..23471accb6 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -91,11 +91,11 @@ def patch(cls): for the duration of this context. """ orig = distutils.core.Distribution - distutils.core.Distribution = cls # type: ignore[misc] # monkeypatching + distutils.core.Distribution = cls try: yield finally: - distutils.core.Distribution = orig # type: ignore[misc] # monkeypatching + distutils.core.Distribution = orig @contextlib.contextmanager diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index be833a379c..8e75fb408d 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -248,7 +248,7 @@ def setup_shlib_compiler(self): compiler.set_link_objects(self.link_objects) # hack so distutils' build_extension() builds a library instead - compiler.link_shared_object = link_shared_object.__get__(compiler) # type: ignore[method-assign] + compiler.link_shared_object = link_shared_object.__get__(compiler) def get_export_symbols(self, ext): if isinstance(ext, Library): diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 9631cf3114..0de2d32843 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -50,7 +50,7 @@ class sdist(orig.sdist): ] distribution: Distribution # override distutils.dist.Distribution with setuptools.dist.Distribution - negative_opt: ClassVar[dict[str, str]] = {} + negative_opt: ClassVar[dict[str, str]] = {} # type: ignore[misc] # TODO: Fix upstream README_EXTENSIONS = ['', '.rst', '.txt', '.md'] READMES = tuple(f'README{ext}' for ext in README_EXTENSIONS) diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 678a0593d6..a9be2418e0 100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -37,7 +37,7 @@ def edit_config(filename, settings, dry_run=False): """ log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() - opts.optionxform = lambda optionstr: optionstr # type: ignore[method-assign] # overriding method + opts.optionxform = lambda optionstr: optionstr _cfg_read_utf8_with_fallback(opts, filename) for section, options in settings.items(): diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py index 633aa9d45d..9bf33de5d1 100644 --- a/setuptools/config/setupcfg.py +++ b/setuptools/config/setupcfg.py @@ -17,7 +17,7 @@ from collections import defaultdict from collections.abc import Iterable, Iterator from functools import partial, wraps -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar from packaging.markers import default_environment as marker_env from packaging.requirements import InvalidRequirement, Requirement @@ -101,8 +101,7 @@ def _apply( filenames = [*other_files, filepath] try: - # TODO: Temporary cast until mypy 1.12 is released with upstream fixes from typeshed - _Distribution.parse_config_files(dist, filenames=cast(list[str], filenames)) + _Distribution.parse_config_files(dist, filenames=filenames) # type: ignore[arg-type] # Vendored version of distutils supports PathLike handlers = parse_configuration( dist, dist.command_options, ignore_option_errors=ignore_option_errors ) diff --git a/setuptools/errors.py b/setuptools/errors.py index 990ecbf4e2..61b0a55c72 100644 --- a/setuptools/errors.py +++ b/setuptools/errors.py @@ -30,15 +30,15 @@ BaseError = _distutils_errors.DistutilsError -class InvalidConfigError(OptionError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class InvalidConfigError(OptionError): """Error used for invalid configurations.""" -class RemovedConfigError(OptionError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class RemovedConfigError(OptionError): """Error used for configurations that were deprecated and removed.""" -class RemovedCommandError(BaseError, RuntimeError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class RemovedCommandError(BaseError, RuntimeError): """Error used for commands that have been removed in setuptools. Since ``setuptools`` is built on ``distutils``, simply removing a command @@ -48,7 +48,7 @@ class RemovedCommandError(BaseError, RuntimeError): # type: ignore[valid-type, """ -class PackageDiscoveryError(BaseError, RuntimeError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class PackageDiscoveryError(BaseError, RuntimeError): """Impossible to perform automatic discovery of packages and/or modules. The current project layout or given discovery options can lead to problems when diff --git a/setuptools/extension.py b/setuptools/extension.py index 76e03d9d6b..96974a3f78 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -153,7 +153,7 @@ def __init__( self.py_limited_api = py_limited_api super().__init__( name, - sources, # type: ignore[arg-type] # Vendored version of setuptools supports PathLike + sources, # type: ignore[arg-type] # Vendored version of distutils supports PathLike *args, **kw, ) diff --git a/setuptools/logging.py b/setuptools/logging.py index 532da899f7..7601ded6e8 100644 --- a/setuptools/logging.py +++ b/setuptools/logging.py @@ -32,7 +32,7 @@ def configure() -> None: # and then loaded again when patched, # implying: id(distutils.log) != id(distutils.dist.log). # Make sure the same module object is used everywhere: - distutils.dist.log = distutils.log + distutils.dist.log = distutils.log # type: ignore[assignment] def set_threshold(level: int) -> int: diff --git a/setuptools/modified.py b/setuptools/modified.py index 6ba02fab68..36c548590b 100644 --- a/setuptools/modified.py +++ b/setuptools/modified.py @@ -1,6 +1,6 @@ try: # Ensure a DistutilsError raised by these methods is the same as distutils.errors.DistutilsError - from distutils._modified import ( + from distutils._modified import ( # type: ignore[import-not-found] newer, newer_group, newer_pairwise, diff --git a/setuptools/monkey.py b/setuptools/monkey.py index 6ad1abac29..177ec05e66 100644 --- a/setuptools/monkey.py +++ b/setuptools/monkey.py @@ -73,7 +73,7 @@ def patch_all(): import setuptools # we can't patch distutils.cmd, alas - distutils.core.Command = setuptools.Command # type: ignore[misc,assignment] # monkeypatching + distutils.core.Command = setuptools.Command _patch_distribution_metadata() @@ -82,8 +82,8 @@ def patch_all(): module.Distribution = setuptools.dist.Distribution # Install the patched Extension - distutils.core.Extension = setuptools.extension.Extension # type: ignore[misc,assignment] # monkeypatching - distutils.extension.Extension = setuptools.extension.Extension # type: ignore[misc,assignment] # monkeypatching + distutils.core.Extension = setuptools.extension.Extension + distutils.extension.Extension = setuptools.extension.Extension if 'distutils.command.build_ext' in sys.modules: sys.modules[ 'distutils.command.build_ext' From 1ac7ac8f1b05e6d8e649acc7904b81f3a4725409 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 9 Mar 2025 01:03:04 -0500 Subject: [PATCH 2/8] Use custom build backend for PEP 660 editable installs --- build_with_distutils_stubs.py | 70 +++++++++++++++++++++++++++++++ mypy.ini | 2 +- pyproject.toml | 3 +- setup.py | 21 ---------- setuptools/__init__.py | 2 +- setuptools/command/build_py.py | 4 +- setuptools/command/install_lib.py | 7 ++-- setuptools/config/setupcfg.py | 2 +- setuptools/dist.py | 2 +- setuptools/extension.py | 7 +--- setuptools/modified.py | 2 +- 11 files changed, 83 insertions(+), 39 deletions(-) create mode 100644 build_with_distutils_stubs.py diff --git a/build_with_distutils_stubs.py b/build_with_distutils_stubs.py new file mode 100644 index 0000000000..45befef0e4 --- /dev/null +++ b/build_with_distutils_stubs.py @@ -0,0 +1,70 @@ +"""Generate distutils stub files inside the source directory before packaging. +We have to do this as a custom build backend for PEP 660 editable installs. +Doing it this way also allows us to point local type-checkers to types/distutils, +overriding the stdlib types even on Python < 3.12.""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +from setuptools._path import StrPath +from setuptools.build_meta import * # noqa: F403 # expose everything +from setuptools.build_meta import ( + _ConfigSettings, + build_editable as _build_editable, + build_sdist as _build_sdist, + build_wheel as _build_wheel, +) + +_vendored_distutils_path = Path(__file__).parent / "setuptools" / "_distutils" +_distutils_stubs_path = Path(__file__).parent / "distutils-stubs" + + +def _regenerate_distutils_stubs() -> None: + shutil.rmtree(_distutils_stubs_path, ignore_errors=True) + _distutils_stubs_path.mkdir(parents=True) + (_distutils_stubs_path / ".gitignore").write_text("*") + (_distutils_stubs_path / "ruff.toml").write_text('[lint]\nignore = ["F403"]') + for path in _vendored_distutils_path.rglob("*.py"): + relative_path = path.relative_to(_vendored_distutils_path) + if relative_path.parts[0] == "tests": + continue + # if str(relative_path) == "__init__.py": + # # Work around a mypy issue with types/distutils/__init__.pyi: + # # error: Source file found twice under different module names: "setuptools._distutils.__init__" and "setuptools._distutils" + # (_distutils_stubs_path / "py.typed").write_text('partial') + # continue + stub_path = _distutils_stubs_path / relative_path.with_suffix(".pyi") + stub_path.parent.mkdir(parents=True, exist_ok=True) + module = "setuptools._distutils." + str(relative_path.with_suffix("")).replace( + os.sep, "." + ).removesuffix(".__init__") + stub_path.write_text(f"from {module} import *\n") + + +def build_wheel( # type: ignore[no-redef] + wheel_directory: StrPath, + config_settings: _ConfigSettings = None, + metadata_directory: StrPath | None = None, +) -> str: + _regenerate_distutils_stubs() + return _build_wheel(wheel_directory, config_settings, metadata_directory) + + +def build_sdist( # type: ignore[no-redef] + sdist_directory: StrPath, + config_settings: _ConfigSettings = None, +) -> str: + _regenerate_distutils_stubs() + return _build_sdist(sdist_directory, config_settings) + + +def build_editable( # type: ignore[no-redef] + wheel_directory: StrPath, + config_settings: _ConfigSettings = None, + metadata_directory: StrPath | None = None, +) -> str: + _regenerate_distutils_stubs() + return _build_editable(wheel_directory, config_settings, metadata_directory) diff --git a/mypy.ini b/mypy.ini index b294995c25..823c6b47f1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,7 +5,7 @@ strict = False # Early opt-in even when strict = False -# warn_unused_ignores = True # Disabled as long as there's inconsistent typing issues between pypa and stdlib's distutils +warn_unused_ignores = True warn_redundant_casts = True enable_error_code = ignore-without-code diff --git a/pyproject.toml b/pyproject.toml index 1df95f02ed..7effcdc89b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [] -build-backend = "setuptools.build_meta" +build-backend = "build_with_distutils_stubs" backend-path = ["."] [project] @@ -202,6 +202,7 @@ include-package-data = true include = [ "setuptools*", "pkg_resources*", + "distutils-stubs*", "_distutils_hack*", ] exclude = [ diff --git a/setup.py b/setup.py index 97406ac8c8..c28a14e722 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ import os import sys import textwrap -from pathlib import Path import setuptools from setuptools.command.install import install @@ -38,22 +37,6 @@ def pypi_link(pkg_filename): return '/'.join(parts) -vendored_distutils_path = Path(here) / "setuptools" / "_distutils" - - -def generate_distutils_stubs(destination: Path) -> None: - for path in vendored_distutils_path.rglob("*.py"): - relative_path = path.relative_to(vendored_distutils_path) - if relative_path.parts[0] == "tests": - continue - stub_path = (destination / relative_path).with_suffix(".pyi") - stub_path.parent.mkdir(parents=True, exist_ok=True) - module = "setuptools._distutils." + str(relative_path.with_suffix("")).replace( - os.sep, "." - ).removesuffix(".__init__") - stub_path.write_text(f"from {module} import *\n") - - class install_with_pth(install): """ Custom install command to install a .pth file for distutils patching. @@ -85,10 +68,6 @@ def initialize_options(self): install.initialize_options(self) self.extra_path = self._pth_name, self._pth_contents - def run(self): - install.run(self) - generate_distutils_stubs(Path(self.install_lib) / 'distutils-stubs') - def finalize_options(self): install.finalize_options(self) self._restore_install_lib() diff --git a/setuptools/__init__.py b/setuptools/__init__.py index cd4d5bb37f..befbff939e 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -206,7 +206,7 @@ def ensure_string_list(self, option: str) -> None: f"'{option}' must be a list of strings (got {val!r})" ) - @overload # type: ignore[override] # extra **kw param that distutils doesn't have + @overload def reinitialize_command( self, command: str, reinit_subcommands: bool = False, **kw ) -> _Command: ... diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 2f6fcb7cdc..35aa93da0b 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -48,7 +48,7 @@ def finalize_options(self): if 'data_files' in self.__dict__: del self.__dict__['data_files'] - def copy_file( # type: ignore[override] # No overload, no bytes support + def copy_file( self, infile: StrPath, outfile: StrPathT, @@ -135,7 +135,7 @@ def find_data_files(self, package, src_dir): ) return self.exclude_data_files(package, src_dir, files) - def get_outputs(self, include_bytecode: bool = True) -> list[str]: # type: ignore[override] # Using a real boolean instead of 0|1 + def get_outputs(self, include_bytecode: bool = True) -> list[str]: """See :class:`setuptools.commands.build.SubCommand`""" if self.editable_mode: return list(self.get_output_mapping().keys()) diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py index 8e1e072710..ef76286d4f 100644 --- a/setuptools/command/install_lib.py +++ b/setuptools/command/install_lib.py @@ -95,10 +95,9 @@ def copy_tree( self, infile: StrPath, outfile: str, - # override: Using actual booleans - preserve_mode: bool = True, # type: ignore[override] - preserve_times: bool = True, # type: ignore[override] - preserve_symlinks: bool = False, # type: ignore[override] + preserve_mode: bool = True, + preserve_times: bool = True, + preserve_symlinks: bool = False, level: object = 1, ) -> list[str]: assert preserve_mode diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py index 9bf33de5d1..16db3c1b9d 100644 --- a/setuptools/config/setupcfg.py +++ b/setuptools/config/setupcfg.py @@ -101,7 +101,7 @@ def _apply( filenames = [*other_files, filepath] try: - _Distribution.parse_config_files(dist, filenames=filenames) # type: ignore[arg-type] # Vendored version of distutils supports PathLike + _Distribution.parse_config_files(dist, filenames=filenames) handlers = parse_configuration( dist, dist.command_options, ignore_option_errors=ignore_option_errors ) diff --git a/setuptools/dist.py b/setuptools/dist.py index 0249651267..8af8a48f9f 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -727,7 +727,7 @@ def fetch_build_egg(self, req): return fetch_build_egg(self, req) - def get_command_class(self, command: str) -> type[distutils.cmd.Command]: # type: ignore[override] # Not doing complex overrides yet + def get_command_class(self, command: str) -> type[distutils.cmd.Command]: """Pluggable version of get_command_class()""" if command in self.cmdclass: return self.cmdclass[command] diff --git a/setuptools/extension.py b/setuptools/extension.py index 96974a3f78..e6acf93c31 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -151,12 +151,7 @@ def __init__( # The *args is needed for compatibility as calls may use positional # arguments. py_limited_api may be set only via keyword. self.py_limited_api = py_limited_api - super().__init__( - name, - sources, # type: ignore[arg-type] # Vendored version of distutils supports PathLike - *args, - **kw, - ) + super().__init__(name, sources, *args, **kw) def _convert_pyx_sources_to_lang(self): """ diff --git a/setuptools/modified.py b/setuptools/modified.py index 36c548590b..6ba02fab68 100644 --- a/setuptools/modified.py +++ b/setuptools/modified.py @@ -1,6 +1,6 @@ try: # Ensure a DistutilsError raised by these methods is the same as distutils.errors.DistutilsError - from distutils._modified import ( # type: ignore[import-not-found] + from distutils._modified import ( newer, newer_group, newer_pairwise, From 4e82d63d29789a81a325e9e57da65c97704add74 Mon Sep 17 00:00:00 2001 From: Avasam Date: Mon, 17 Mar 2025 01:07:39 -0400 Subject: [PATCH 3/8] Correctly work around mypy issue --- build_with_distutils_stubs.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/build_with_distutils_stubs.py b/build_with_distutils_stubs.py index 45befef0e4..aafc266d42 100644 --- a/build_with_distutils_stubs.py +++ b/build_with_distutils_stubs.py @@ -27,21 +27,25 @@ def _regenerate_distutils_stubs() -> None: _distutils_stubs_path.mkdir(parents=True) (_distutils_stubs_path / ".gitignore").write_text("*") (_distutils_stubs_path / "ruff.toml").write_text('[lint]\nignore = ["F403"]') + (_distutils_stubs_path / "py.typed").write_text("\n") for path in _vendored_distutils_path.rglob("*.py"): relative_path = path.relative_to(_vendored_distutils_path) - if relative_path.parts[0] == "tests": + if "tests" in relative_path.parts: continue - # if str(relative_path) == "__init__.py": - # # Work around a mypy issue with types/distutils/__init__.pyi: - # # error: Source file found twice under different module names: "setuptools._distutils.__init__" and "setuptools._distutils" - # (_distutils_stubs_path / "py.typed").write_text('partial') - # continue stub_path = _distutils_stubs_path / relative_path.with_suffix(".pyi") stub_path.parent.mkdir(parents=True, exist_ok=True) module = "setuptools._distutils." + str(relative_path.with_suffix("")).replace( os.sep, "." ).removesuffix(".__init__") - stub_path.write_text(f"from {module} import *\n") + if str(relative_path) == "__init__.py": + # Work around python/mypy#18775 + stub_path.write_text("""\ +from typing import Final + +__version__: Final[str] +""") + else: + stub_path.write_text(f"from {module} import *\n") def build_wheel( # type: ignore[no-redef] From 74e7ea88d8a427e3043869196283b03d688df445 Mon Sep 17 00:00:00 2001 From: Avasam Date: Mon, 21 Apr 2025 12:57:06 -0400 Subject: [PATCH 4/8] Fix all mypy type issues --- setuptools/_distutils/command/install.py | 10 +++--- setuptools/_distutils/dist.py | 11 +++--- setuptools/_path.py | 4 ++- setuptools/command/bdist_egg.py | 2 +- setuptools/command/build_py.py | 44 ++++++++++++++++++++---- setuptools/command/sdist.py | 2 +- setuptools/dist.py | 5 +++ 7 files changed, 58 insertions(+), 20 deletions(-) diff --git a/setuptools/_distutils/command/install.py b/setuptools/_distutils/command/install.py index dc17e56a80..50c04f1cb8 100644 --- a/setuptools/_distutils/command/install.py +++ b/setuptools/_distutils/command/install.py @@ -264,12 +264,12 @@ def initialize_options(self) -> None: # supplied by the user, they are filled in using the installation # scheme implied by prefix/exec-prefix/home and the contents of # that installation scheme. - self.install_purelib = None # for pure module distributions - self.install_platlib = None # non-pure (dists w/ extensions) - self.install_headers = None # for C/C++ headers + self.install_purelib: str | None = None # for pure module distributions + self.install_platlib: str | None = None # non-pure (dists w/ extensions) + self.install_headers: str | None = None # for C/C++ headers self.install_lib: str | None = None # set to either purelib or platlib - self.install_scripts = None - self.install_data = None + self.install_scripts: str | None = None + self.install_data: str | None = None self.install_userbase = USER_BASE self.install_usersite = USER_SITE diff --git a/setuptools/_distutils/dist.py b/setuptools/_distutils/dist.py index 37b788df92..f75b691910 100644 --- a/setuptools/_distutils/dist.py +++ b/setuptools/_distutils/dist.py @@ -45,6 +45,7 @@ # type-only import because of mutual dependence between these modules from .cmd import Command + from .extension import Extension _CommandT = TypeVar("_CommandT", bound="Command") _OptionsList: TypeAlias = list[ @@ -220,18 +221,18 @@ def __init__(self, attrs: MutableMapping[str, Any] | None = None) -> None: # no # These options are really the business of various commands, rather # than of the Distribution itself. We provide aliases for them in # Distribution as a convenience to the developer. - self.packages = None + self.packages: list[str] | None = None self.package_data: dict[str, list[str]] = {} - self.package_dir = None - self.py_modules = None + self.package_dir: dict[str, str] | None = None + self.py_modules: list[str] | None = None self.libraries = None self.headers = None - self.ext_modules = None + self.ext_modules: list[Extension] | None = None self.ext_package = None self.include_dirs = None self.extra_path = None self.scripts = None - self.data_files = None + self.data_files: list[str | tuple] | None = None self.password = '' # And now initialize bookkeeping stuff that can't be supplied by diff --git a/setuptools/_path.py b/setuptools/_path.py index 0d99b0f539..5a396709cc 100644 --- a/setuptools/_path.py +++ b/setuptools/_path.py @@ -11,7 +11,9 @@ from typing_extensions import TypeAlias StrPath: TypeAlias = Union[str, os.PathLike[str]] # Same as _typeshed.StrPath -StrPathT = TypeVar("StrPathT", bound=Union[str, os.PathLike[str]]) +StrPathT = TypeVar("StrPathT", bound=StrPath) +BytesPath: TypeAlias = Union[bytes, os.PathLike[bytes]] # Same as _typeshed.BytesPath +BytesPathT = TypeVar("BytesPathT", bound=BytesPath) def ensure_directory(path): diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 7f66c3ba6a..83f6a804a1 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -132,7 +132,7 @@ def do_install_data(self) -> None: site_packages = os.path.normcase(os.path.realpath(_get_purelib())) old, self.distribution.data_files = self.distribution.data_files, [] - for item in old: + for item in old or (): if isinstance(item, tuple) and len(item) == 2: if os.path.isabs(item[0]): realpath = os.path.realpath(item[0]) diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 35aa93da0b..76b65b95cc 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -9,10 +9,11 @@ from functools import partial from glob import glob from pathlib import Path +from typing import Literal, overload from more_itertools import unique_everseen -from .._path import StrPath, StrPathT +from .._path import BytesPath, BytesPathT, StrPath, StrPathT from ..dist import Distribution from ..warnings import SetuptoolsDeprecationWarning @@ -48,6 +49,7 @@ def finalize_options(self): if 'data_files' in self.__dict__: del self.__dict__['data_files'] + @overload # type: ignore[override] # Truthy link with bytes is not supported, unlike supertype def copy_file( self, infile: StrPath, @@ -55,14 +57,42 @@ def copy_file( preserve_mode: bool = True, preserve_times: bool = True, link: str | None = None, - level: object = 1, - ) -> tuple[StrPathT | str, bool]: + level: int = 1, + ) -> tuple[StrPathT | str, bool]: ... + @overload + def copy_file( + self, + infile: BytesPath, + outfile: BytesPathT, + preserve_mode: bool = True, + preserve_times: bool = True, + link: Literal[""] | None = None, + level: int = 1, + ) -> tuple[BytesPathT | bytes, bool]: ... + def copy_file( + self, + infile: StrPath | BytesPath, + outfile: StrPath | BytesPath, + preserve_mode: bool = True, + preserve_times: bool = True, + link: str | None = None, + level: int = 1, + ) -> tuple[StrPath | BytesPath, bool]: # Overwrite base class to allow using links if link: - infile = str(Path(infile).resolve()) - outfile = str(Path(outfile).resolve()) # type: ignore[assignment] # Re-assigning a str when outfile is StrPath is ok - return super().copy_file( # pyright: ignore[reportReturnType] # pypa/distutils#309 - infile, outfile, preserve_mode, preserve_times, link, level + # NOTE: Explanation for the type ignores: + # 1. If link is truthy, then we only allow infile and outfile to be StrPath + # 2. Re-assigning a str when outfile is StrPath is ok + # We can't easily check for PathLike[str], so ignoring instead of asserting. + infile = str(Path(infile).resolve()) # type: ignore[arg-type] + outfile = str(Path(outfile).resolve()) # type: ignore[arg-type] + return super().copy_file( # type: ignore[misc, type-var] # pyright: ignore[reportCallIssue] + infile, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + outfile, # pyright: ignore[reportArgumentType] + preserve_mode, + preserve_times, + link, + level, ) def run(self) -> None: diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 0de2d32843..9631cf3114 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -50,7 +50,7 @@ class sdist(orig.sdist): ] distribution: Distribution # override distutils.dist.Distribution with setuptools.dist.Distribution - negative_opt: ClassVar[dict[str, str]] = {} # type: ignore[misc] # TODO: Fix upstream + negative_opt: ClassVar[dict[str, str]] = {} README_EXTENSIONS = ['', '.rst', '.txt', '.md'] READMES = tuple(f'README{ext}' for ext in README_EXTENSIONS) diff --git a/setuptools/dist.py b/setuptools/dist.py index f47b97b34f..8593f31675 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -48,6 +48,8 @@ from pkg_resources import Distribution as _pkg_resources_Distribution + from .extension import Extension + __all__ = ['Distribution'] @@ -301,6 +303,9 @@ class Distribution(_Distribution): # Used by build_py, editable_wheel and install_lib commands for legacy namespaces namespace_packages: list[str] #: :meta private: DEPRECATED + # override distutils.extension.Extension with setuptools.extension.Extension + ext_modules: list[Extension] | None # type: ignore[assignment] + # Any: Dynamic assignment results in Incompatible types in assignment def __init__(self, attrs: MutableMapping[str, Any] | None = None) -> None: have_package_data = hasattr(self, "package_data") From 48f0a6751eb578ab4184346773c712cdc11a6397 Mon Sep 17 00:00:00 2001 From: Avasam Date: Mon, 21 Apr 2025 13:19:44 -0400 Subject: [PATCH 5/8] Fix odd mypy typing issue in setuptools.errors on Python 3.12+ --- setuptools/errors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setuptools/errors.py b/setuptools/errors.py index 61b0a55c72..fa580085ac 100644 --- a/setuptools/errors.py +++ b/setuptools/errors.py @@ -3,6 +3,8 @@ Provides exceptions used by setuptools modules. """ +# Odd mypy issue with this specific import, alias and base classes on 3.12+ +# mypy: disable-error-code="valid-type,misc" from __future__ import annotations from distutils import errors as _distutils_errors From 570a4aeff4484d337fa44569f484e39b6eceb118 Mon Sep 17 00:00:00 2001 From: Avasam Date: Mon, 21 Apr 2025 13:30:19 -0400 Subject: [PATCH 6/8] Fix 2 pyright issues in tests --- setuptools/tests/config/test_apply_pyprojecttoml.py | 1 + setuptools/tests/test_config_discovery.py | 1 + 2 files changed, 2 insertions(+) diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py index 489fd98e26..365530a349 100644 --- a/setuptools/tests/config/test_apply_pyprojecttoml.py +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -548,6 +548,7 @@ def test_pyproject_sets_attribute(self, tmp_path, monkeypatch): pyproject.write_text(cleandoc(toml_config), encoding="utf-8") with pytest.warns(pyprojecttoml._ExperimentalConfiguration): dist = pyprojecttoml.apply_configuration(Distribution({}), pyproject) + assert dist.ext_modules assert len(dist.ext_modules) == 1 assert dist.ext_modules[0].name == "my.ext" assert set(dist.ext_modules[0].sources) == {"hello.c", "world.c"} diff --git a/setuptools/tests/test_config_discovery.py b/setuptools/tests/test_config_discovery.py index b5df8203cd..18d7bd57ea 100644 --- a/setuptools/tests/test_config_discovery.py +++ b/setuptools/tests/test_config_discovery.py @@ -389,6 +389,7 @@ def test_skip_discovery_with_setupcfg_metadata(self, tmp_path): assert dist.get_version() == "42" assert dist.py_modules is None assert dist.packages is None + assert dist.ext_modules assert len(dist.ext_modules) == 1 assert dist.ext_modules[0].name == "proj" From 96a6fcfd33c70f16cc01da3e9f98ea05f8c9f351 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 15 May 2025 11:40:46 -0400 Subject: [PATCH 7/8] Update build_with_distutils_stubs.py --- build_with_distutils_stubs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_with_distutils_stubs.py b/build_with_distutils_stubs.py index aafc266d42..7fe8a129c5 100644 --- a/build_with_distutils_stubs.py +++ b/build_with_distutils_stubs.py @@ -1,7 +1,6 @@ """Generate distutils stub files inside the source directory before packaging. We have to do this as a custom build backend for PEP 660 editable installs. -Doing it this way also allows us to point local type-checkers to types/distutils, -overriding the stdlib types even on Python < 3.12.""" +""" from __future__ import annotations From d982abb469699e40e1a5e3a4a3f838121eaa8050 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sat, 31 May 2025 14:45:56 -0400 Subject: [PATCH 8/8] Re-run mypy after many setuptools updates --- setuptools/__init__.py | 4 +--- setuptools/build_meta.py | 4 ++-- setuptools/command/bdist_egg.py | 2 +- setuptools/command/build_ext.py | 2 +- setuptools/command/setopt.py | 2 +- setuptools/monkey.py | 6 +++--- setuptools/msvc.py | 2 +- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index b918b5a3d9..d92e8744e9 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -107,9 +107,7 @@ def setup(**attrs) -> Distribution: logging.configure() # Make sure we have any requirements needed to interpret 'attrs'. _install_setup_requires(attrs) - # Override return type of distutils.core.Distribution with setuptools.dist.Distribution - # (implicitly implemented via `setuptools.monkey.patch_all`). - return distutils.core.setup(**attrs) # type: ignore[return-value] + return distutils.core.setup(**attrs) setup.__doc__ = distutils.core.setup.__doc__ diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 6f1f9d46bc..def9c49556 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -88,11 +88,11 @@ def patch(cls) -> Iterator[None]: for the duration of this context. """ orig = distutils.core.Distribution - distutils.core.Distribution = cls + distutils.core.Distribution = cls # type: ignore[misc] try: yield finally: - distutils.core.Distribution = orig + distutils.core.Distribution = orig # type: ignore[misc] @contextlib.contextmanager diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index c4ca25a837..94c2b7a241 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -235,7 +235,7 @@ def run(self) -> None: # noqa: C901 # is too complex (14) # FIXME self.egg_output, archive_root, verbose=self.verbose, - dry_run=self.dry_run, # type: ignore[arg-type] # Is an actual boolean in vendored _distutils + dry_run=self.dry_run, mode=self.gen_header(), ) if not self.keep_temp: diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 2706041295..5378cdfc95 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -246,7 +246,7 @@ def setup_shlib_compiler(self) -> None: compiler.set_link_objects(self.link_objects) # hack so distutils' build_extension() builds a library instead - compiler.link_shared_object = link_shared_object.__get__(compiler) + compiler.link_shared_object = link_shared_object.__get__(compiler) # type: ignore[method-assign] def get_export_symbols(self, ext): if isinstance(ext, Library): diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 7e3c998d51..f04f2e6426 100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -37,7 +37,7 @@ def edit_config(filename, settings, dry_run=False) -> None: """ log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() - opts.optionxform = lambda optionstr: optionstr + opts.optionxform = lambda optionstr: optionstr # type: ignore[method-assign] _cfg_read_utf8_with_fallback(opts, filename) for section, options in settings.items(): diff --git a/setuptools/monkey.py b/setuptools/monkey.py index 783e694e08..b84ed8d4ae 100644 --- a/setuptools/monkey.py +++ b/setuptools/monkey.py @@ -73,7 +73,7 @@ def patch_all() -> None: import setuptools # we can't patch distutils.cmd, alas - distutils.core.Command = setuptools.Command + distutils.core.Command = setuptools.Command # type: ignore[misc, assignment] _patch_distribution_metadata() @@ -82,8 +82,8 @@ def patch_all() -> None: module.Distribution = setuptools.dist.Distribution # Install the patched Extension - distutils.core.Extension = setuptools.extension.Extension - distutils.extension.Extension = setuptools.extension.Extension + distutils.core.Extension = setuptools.extension.Extension # type: ignore[misc] + distutils.extension.Extension = setuptools.extension.Extension # type: ignore[misc] if 'distutils.command.build_ext' in sys.modules: sys.modules[ 'distutils.command.build_ext' diff --git a/setuptools/msvc.py b/setuptools/msvc.py index f506c8222d..bcd5c059aa 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -293,7 +293,7 @@ def windows_kits_roots(self) -> LiteralString: @overload def microsoft(self, key: LiteralString, x86: bool = False) -> LiteralString: ... @overload - def microsoft(self, key: str, x86: bool = False) -> str: ... # type: ignore[misc] + def microsoft(self, key: str, x86: bool = False) -> str: ... # type: ignore[overload-cannot-match] def microsoft(self, key: str, x86: bool = False) -> str: """ Return key in Microsoft software registry.