Fix caching of PEP 561 namespace packages with missing submodules (#12250)

Fixes #12232

Another "fail every 2nd run" bug

`if not self.is_module(sub_id):`, the parent can come before another child, and we fail to resolve the parent.

Co-authored-by: hauntsaninja <>

Author: hauntsaninja

mypy/build.py

@@ -730,20 +730,19 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
             return new_id
 
         res: List[Tuple[int, str, int]] = []
+        delayed_res: List[Tuple[int, str, int]] = []
         for imp in file.imports:
             if not imp.is_unreachable:
                 if isinstance(imp, Import):
                     pri = import_priority(imp, PRI_MED)
                     ancestor_pri = import_priority(imp, PRI_LOW)
                     for id, _ in imp.ids:
-                        # We append the target (e.g. foo.bar.baz)
-                        # before the ancestors (e.g. foo and foo.bar)
-                        # so that, if FindModuleCache finds the target
-                        # module in a package marked with py.typed
-                        # underneath a namespace package installed in
-                        # site-packages, (gasp), that cache's
-                        # knowledge of the ancestors can be primed
-                        # when it is asked to find the target.
+                        # We append the target (e.g. foo.bar.baz) before the ancestors (e.g. foo
+                        # and foo.bar) so that, if FindModuleCache finds the target module in a
+                        # package marked with py.typed underneath a namespace package installed in
+                        # site-packages, (gasp), that cache's knowledge of the ancestors
+                        # (aka FindModuleCache.ns_ancestors) can be primed when it is asked to find
+                        # the parent.
                         res.append((pri, id, imp.line))
                         ancestor_parts = id.split(".")[:-1]
                         ancestors = []
@@ -752,13 +751,15 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
                             res.append((ancestor_pri, ".".join(ancestors), imp.line))
                 elif isinstance(imp, ImportFrom):
                     cur_id = correct_rel_imp(imp)
+                    any_are_submodules = False
                     all_are_submodules = True
                     # Also add any imported names that are submodules.
                     pri = import_priority(imp, PRI_MED)
                     for name, __ in imp.names:
                         sub_id = cur_id + '.' + name
                         if self.is_module(sub_id):
                             res.append((pri, sub_id, imp.line))
+                            any_are_submodules = True
                         else:
                             all_are_submodules = False
                     # Add cur_id as a dependency, even if all of the
@@ -768,14 +769,19 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
                     # if all of the imports are submodules, do the import at a lower
                     # priority.
                     pri = import_priority(imp, PRI_HIGH if not all_are_submodules else PRI_LOW)
-                    # The imported module goes in after the
-                    # submodules, for the same namespace related
-                    # reasons discussed in the Import case.
-                    res.append((pri, cur_id, imp.line))
+                    # The imported module goes in after the submodules, for the same namespace
+                    # related reasons discussed in the Import case.
+                    # There is an additional twist: if none of the submodules exist,
+                    # we delay the import in case other imports of other submodules succeed.
+                    if any_are_submodules:
+                        res.append((pri, cur_id, imp.line))
+                    else:
+                        delayed_res.append((pri, cur_id, imp.line))
                 elif isinstance(imp, ImportAll):
                     pri = import_priority(imp, PRI_HIGH)
                     res.append((pri, correct_rel_imp(imp), imp.line))
 
+        res.extend(delayed_res)
         return res
 
     def is_module(self, id: str) -> bool:

test-data/unit/check-errorcodes.test

@@ -596,10 +596,10 @@ if int() is str():  # E: Non-overlapping identity check (left operand type: "int
 [builtins fixtures/primitives.pyi]
 
 [case testErrorCodeMissingModule]
-from defusedxml import xyz  # E: Cannot find implementation or library stub for module named "defusedxml"  [import] \
-                            # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+from defusedxml import xyz  # E: Cannot find implementation or library stub for module named "defusedxml"  [import]
 from nonexistent import foobar  # E: Cannot find implementation or library stub for module named "nonexistent"  [import]
-import nonexistent2  # E: Cannot find implementation or library stub for module named "nonexistent2"  [import]
+import nonexistent2  # E: Cannot find implementation or library stub for module named "nonexistent2"  [import] \
+                     # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 from nonexistent3 import *  # E: Cannot find implementation or library stub for module named "nonexistent3"  [import]
 from pkg import bad  # E: Module "pkg" has no attribute "bad"  [attr-defined]
 from pkg.bad2 import bad3  # E: Cannot find implementation or library stub for module named "pkg.bad2"  [import]

test-data/unit/cmdline.test

@@ -415,9 +415,9 @@ follow_imports = error
 [file a.py]
 /  # No error reported
 [out]
-main.py:1: error: Import of "a" ignored
-main.py:1: note: (Using --follow-imports=error, module not passed on command line)
 main.py:2: note: Revealed type is "Any"
+main.py:3: error: Import of "a" ignored
+main.py:3: note: (Using --follow-imports=error, module not passed on command line)
 main.py:4: note: Revealed type is "Any"
 
 [case testConfigFollowImportsSelective]

test-data/unit/pep561.test

@@ -222,3 +222,13 @@ b.bf(1)
 [out]
 testNamespacePkgWStubsWithNamespacePackagesFlag.py:7: error: Argument 1 to "bf" has incompatible type "int"; expected "bool"
 testNamespacePkgWStubsWithNamespacePackagesFlag.py:8: error: Argument 1 to "bf" has incompatible type "int"; expected "bool"
+
+
+[case testTypedPkgNamespaceRegFromImportTwiceMissing]
+# pkgs: typedpkg_ns_a
+from typedpkg_ns import b  # type: ignore
+from typedpkg_ns import a
+-- dummy should trigger a second iteration
+[file dummy.py.2]
+[out]
+[out2]

test-data/unit/semanal-errors.test

@@ -293,8 +293,8 @@ from m.n import x
 from a.b import *
 [out]
 main:2: error: Cannot find implementation or library stub for module named "m.n"
-main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 main:3: error: Cannot find implementation or library stub for module named "a.b"
+main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 
 [case testErrorInImportedModule]
 import m