Handle accidental virtual namespaces in editable install (#3512)

Author: abravalheri

changelog.d/3512.misc.rst

@@ -0,0 +1,3 @@
+Added capability of handling namespace packages created
+accidentally/purposefully via discovery configuration during editable installs.
+This should emulate the behaviour of a non-editable installation.

setuptools/command/editable_wheel.py

@@ -628,7 +628,14 @@ def _absolute_root(path: _Path) -> str:
 def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]:
     """By carefully designing ``package_dir``, it is possible to implement the logical
     structure of PEP 420 in a package without the corresponding directories.
-    This function will try to find this kind of namespaces.
+
+    Moreover a parent package can be purposefully/accidentally skipped in the discovery
+    phase (e.g. ``find_packages(include=["mypkg.*"])``, when ``mypkg.foo`` is included
+    by ``mypkg`` itself is not).
+    We consider this case to also be a virtual namespace (ignoring the original
+    directory) to emulate a non-editable installation.
+
+    This function will try to find these kinds of namespaces.
     """
     for pkg in pkg_roots:
         if "." not in pkg:
@@ -637,7 +644,8 @@ def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]:
         for i in range(len(parts) - 1, 0, -1):
             partial_name = ".".join(parts[:i])
             path = Path(find_package_path(partial_name, pkg_roots, ""))
-            if not path.exists():
+            if not path.exists() or partial_name not in pkg_roots:
+                # partial_name not in pkg_roots ==> purposefully/accidentally skipped
                 yield partial_name
 
 

setuptools/tests/test_editable_install.py

@@ -269,6 +269,54 @@ def test_namespace_created_via_package_dir(self, venv, tmp_path, editable_opts):
         venv.run(["python", "-m", "pip", "install", "-e", str(pkg_C), *opts])
         venv.run(["python", "-c", "from myns.n import pkgA, pkgB, pkgC"])
 
+    def test_namespace_accidental_config_in_lenient_mode(self, venv, tmp_path):
+        """Sometimes users might specify an ``include`` pattern that ignores parent
+        packages. In a normal installation this would ignore all modules inside the
+        parent packages, and make them namespaces (reported in issue #3504),
+        so the editable mode should preserve this behaviour.
+        """
+        files = {
+            "pkgA": {
+                "pyproject.toml": dedent("""\
+                    [build-system]
+                    requires = ["setuptools", "wheel"]
+                    build-backend = "setuptools.build_meta"
+
+                    [project]
+                    name = "pkgA"
+                    version = "3.14159"
+
+                    [tool.setuptools]
+                    packages.find.include = ["mypkg.*"]
+                    """),
+                "mypkg": {
+                    "__init__.py": "",
+                    "other.py": "b = 1",
+                    "n": {
+                        "__init__.py": "",
+                        "pkgA.py": "a = 1",
+                    },
+                 },
+                "MANIFEST.in": EXAMPLE["MANIFEST.in"],
+            },
+        }
+        jaraco.path.build(files, prefix=tmp_path)
+        pkg_A = tmp_path / "pkgA"
+
+        # use pip to install to the target directory
+        opts = ["--no-build-isolation"]  # force current version of setuptools
+        venv.run(["python", "-m", "pip", "-v", "install", "-e", str(pkg_A), *opts])
+        out = venv.run(["python", "-c", "from mypkg.n import pkgA; print(pkgA.a)"])
+        assert str(out, "utf-8").strip() == "1"
+        cmd = """\
+        try:
+            import mypkg.other
+        except ImportError:
+            print("mypkg.other not defined")
+        """
+        out = venv.run(["python", "-c", dedent(cmd)])
+        assert "mypkg.other not defined" in str(out, "utf-8")
+
 
 # Moved here from test_develop:
 @pytest.mark.xfail(
@@ -490,7 +538,7 @@ def test_pkg_roots(tmp_path):
     assert ns == {"f", "f.g"}
 
     ns = set(_find_virtual_namespaces(roots))
-    assert ns == {"a.b.c.x", "a.b.c.x.y", "m", "m.n", "m.n.o", "m.n.o.p"}
+    assert ns == {"a.b", "a.b.c.x", "a.b.c.x.y", "m", "m.n", "m.n.o", "m.n.o.p"}
 
 
 class TestOverallBehaviour: