Ensure builtin modules are from typeshed sooner

It should work now with custom-typeshed-dir.

Fixes #1876

Author: Konstantin Ignatov

mypy/build.py

@@ -15,6 +15,7 @@
 import gc
 import json
 import os
+import pathlib
 import platform
 import re
 import stat
@@ -605,6 +606,29 @@ def __init__(self, data_dir: str,
         self.fscache = fscache
         self.find_module_cache = FindModuleCache(self.search_paths, self.fscache, self.options,
                                                  source_set=self.source_set)
+        for module in CORE_BUILTIN_MODULES:
+            if options.use_builtins_fixtures:
+                continue
+            if module == "_importlib_modulespec":
+                continue
+            path = self.find_module_cache.find_module(module)
+            if not isinstance(path, str):
+                raise CompileError([
+                    f"Failed to find builtin module {module}, perhaps typeshed is broken?",
+                ])
+            if is_typeshed_file(path):
+                continue
+            if is_stub_package_file(path):
+                continue
+            if options.custom_typeshed_dir is not None:
+                if pathlib.Path(path) in pathlib.Path(options.custom_typeshed_dir).parents:
+                    continue
+
+            raise CompileError([
+                f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"',
+                f'note: A user-defined top-level module with name "{module}" is not supported'
+            ])
+
         self.metastore = create_metastore(options)
 
         # a mapping from source files to their corresponding shadow files
@@ -2459,15 +2483,6 @@ def find_module_and_diagnose(manager: BuildManager,
                 if is_sub_path(result, dir):
                     # Silence errors in site-package dirs and typeshed
                     follow_imports = 'silent'
-        if (id in CORE_BUILTIN_MODULES
-                and not is_typeshed_file(result)
-                and not is_stub_package_file(result)
-                and not options.use_builtins_fixtures
-                and not options.custom_typeshed_dir):
-            raise CompileError([
-                f'mypy: "{os.path.relpath(result)}" shadows library module "{id}"',
-                f'note: A user-defined top-level module with name "{id}" is not supported'
-            ])
         return (result, follow_imports)
     else:
         # Could not find a module.  Typically the reason is a

mypy/test/testgraph.py

@@ -38,6 +38,7 @@ def test_scc(self) -> None:
     def _make_manager(self) -> BuildManager:
         errors = Errors()
         options = Options()
+        options.use_builtins_fixtures = True
         fscache = FileSystemCache()
         search_paths = SearchPaths((), (), (), ())
         manager = BuildManager(

mypy/util.py

@@ -51,6 +51,8 @@
     "__init__", "__new__", "__call__", "__init_subclass__", "__class_getitem__",
 ))
 
+TYPESHED_DIR: Final = os.path.join(os.path.dirname(__file__), 'typeshed')
+
 
 def is_dunder(name: str, exclude_special: bool = False) -> bool:
     """Returns whether name is a dunder name.
@@ -745,16 +747,15 @@ def format_error(
 
 
 def is_typeshed_file(file: str) -> bool:
-    # gross, but no other clear way to tell
-    return 'typeshed' in os.path.abspath(file).split(os.sep)
+    return os.path.commonpath((TYPESHED_DIR, os.path.abspath(file))) == TYPESHED_DIR
 
 
 def is_stub_package_file(file: str) -> bool:
     # Use hacky heuristics to check whether file is part of a PEP 561 stub package.
     if not file.endswith('.pyi'):
         return False
     return any(component.endswith('-stubs')
-               for component in os.path.abspath(file).split(os.sep))
+               for component in os.path.split(os.path.abspath(file)))
 
 
 def unnamed_function(name: Optional[str]) -> bool:

test-data/unit/cmdline.test

@@ -1425,3 +1425,22 @@ b\.c \d+
 # cmd: mypy --enable-incomplete-features a.py
 [file a.py]
 pass
+
+[case testShadowTypingModuleEarlyLoad]
+# cmd: mypy .
+from typing import Union
+
+def foo(a: Union[int, str]) -> str:
+    return str
+[file typing.py]
+import sys
+import os
+del sys.modules["typing"]
+path = sys.path
+sys.path.remove(os.getcwd())
+from typing import *
+sys.path = path
+[out]
+mypy: "tmp/typing.py" shadows library module "typing"
+note: A user-defined top-level module with name "typing" is not supported
+== Return code: 2