Ensure builtin modules are from typeshed sooner (#13155)

It should work now with custom-typeshed-dir.

Fixes #1876

Author: code-of-kpp

build-requirements.txt

@@ -1,2 +1,3 @@
 -r mypy-requirements.txt
+types-setuptools
 types-typed-ast>=1.5.0,<1.6.0

mypy-requirements.txt

@@ -1,4 +1,5 @@
 typing_extensions>=3.10
 mypy_extensions>=0.4.3
+importlib_resources; python_version<'3.7'
 typed_ast>=1.4.0,<2; python_version<'3.8'
 tomli>=1.1.0; python_version<'3.11'

mypy/build.py

@@ -657,6 +657,33 @@ def __init__(
         self.find_module_cache = FindModuleCache(
             self.search_paths, self.fscache, self.options, source_set=self.source_set
         )
+        for module in CORE_BUILTIN_MODULES:
+            if options.use_builtins_fixtures:
+                continue
+            if module == "_importlib_modulespec":
+                continue
+            path = self.find_module_cache.find_module(module)
+            if not isinstance(path, str):
+                raise CompileError(
+                    [f"Failed to find builtin module {module}, perhaps typeshed is broken?"]
+                )
+            if is_typeshed_file(path):
+                continue
+            if is_stub_package_file(path):
+                continue
+            if options.custom_typeshed_dir is not None:
+                # Check if module lives under custom_typeshed_dir subtree
+                custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir)
+                if os.path.commonpath((path, custom_typeshed_dir)) == custom_typeshed_dir:
+                    continue
+
+            raise CompileError(
+                [
+                    f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"',
+                    f'note: A user-defined top-level module with name "{module}" is not supported',
+                ]
+            )
+
         self.metastore = create_metastore(options)
 
         # a mapping from source files to their corresponding shadow files
@@ -2583,19 +2610,6 @@ def find_module_and_diagnose(
                 if is_sub_path(result, dir):
                     # Silence errors in site-package dirs and typeshed
                     follow_imports = "silent"
-        if (
-            id in CORE_BUILTIN_MODULES
-            and not is_typeshed_file(result)
-            and not is_stub_package_file(result)
-            and not options.use_builtins_fixtures
-            and not options.custom_typeshed_dir
-        ):
-            raise CompileError(
-                [
-                    f'mypy: "{os.path.relpath(result)}" shadows library module "{id}"',
-                    f'note: A user-defined top-level module with name "{id}" is not supported',
-                ]
-            )
         return (result, follow_imports)
     else:
         # Could not find a module.  Typically the reason is a

mypy/test/testgraph.py

@@ -41,6 +41,7 @@ def test_scc(self) -> None:
     def _make_manager(self) -> BuildManager:
         errors = Errors()
         options = Options()
+        options.use_builtins_fixtures = True
         fscache = FileSystemCache()
         search_paths = SearchPaths((), (), (), ())
         manager = BuildManager(

mypy/util.py

@@ -8,6 +8,12 @@
 import shutil
 import sys
 import time
+
+try:
+    from importlib import resources as importlib_resources  # type: ignore[attr-defined]
+except ImportError:  # <python3.7
+    import importlib_resources  # type: ignore
+
 from typing import (
     IO,
     Callable,
@@ -36,6 +42,13 @@
 
 T = TypeVar("T")
 
+with importlib_resources.path(
+    "mypy",  # mypy-c doesn't support __package__
+    "py.typed",  # a marker file for type information, we assume typeshed to live in the same dir
+) as _resource:
+    TYPESHED_DIR: Final = str(_resource.parent / "typeshed")
+
+
 ENCODING_RE: Final = re.compile(rb"([ \t\v]*#.*(\r\n?|\n))??[ \t\v]*#.*coding[:=][ \t]*([-\w.]+)")
 
 DEFAULT_SOURCE_OFFSET: Final = 4
@@ -772,15 +785,17 @@ def format_error(
 
 
 def is_typeshed_file(file: str) -> bool:
-    # gross, but no other clear way to tell
-    return "typeshed" in os.path.abspath(file).split(os.sep)
+    try:
+        return os.path.commonpath((TYPESHED_DIR, os.path.abspath(file))) == TYPESHED_DIR
+    except ValueError:  # Different drives on Windows
+        return False
 
 
 def is_stub_package_file(file: str) -> bool:
     # Use hacky heuristics to check whether file is part of a PEP 561 stub package.
     if not file.endswith(".pyi"):
         return False
-    return any(component.endswith("-stubs") for component in os.path.abspath(file).split(os.sep))
+    return any(component.endswith("-stubs") for component in os.path.split(os.path.abspath(file)))
 
 
 def unnamed_function(name: Optional[str]) -> bool:

mypyc/build.py

@@ -45,7 +45,7 @@
 
 try:
     # Import setuptools so that it monkey-patch overrides distutils
-    import setuptools  # type: ignore  # noqa
+    import setuptools  # noqa
 except ImportError:
     if sys.version_info >= (3, 12):
         # Raise on Python 3.12, since distutils will go away forever
@@ -61,7 +61,7 @@ def get_extension() -> Type["Extension"]:
     if not use_setuptools:
         from distutils.core import Extension
     else:
-        from setuptools import Extension  # type: ignore  # noqa
+        from setuptools import Extension  # noqa
 
     return Extension
 

test-data/unit/cmdline.test

@@ -1439,3 +1439,29 @@ b\.c \d+
 # cmd: mypy --enable-incomplete-features a.py
 [file a.py]
 pass
+
+[case testShadowTypingModuleEarlyLoad]
+# cmd: mypy dir
+[file dir/__init__.py]
+from typing import Union
+
+def foo(a: Union[int, str]) -> str:
+    return str
+[file typing.py]
+# Since this file will be picked by mypy itself, we need it to be a fully-working typing
+# A bare minimum would be NamedTuple and TypedDict, which are used in runtime,
+# everything else technically can be just mocked.
+import sys
+import os
+del sys.modules["typing"]
+path = sys.path
+try:
+    sys.path.remove(os.getcwd())
+except ValueError:
+    sys.path.remove("")  # python 3.6
+from typing import *
+sys.path = path
+[out]
+mypy: "typing.py" shadows library module "typing"
+note: A user-defined top-level module with name "typing" is not supported
+== Return code: 2