diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aee9f47a..e03da15c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,9 @@ jobs: - python-version: "3.9" runs-on: ubuntu-latest cmake-version: "3.20.x" + - python-version: "3.9" + runs-on: macos-latest + cmake-version: "3.18.x" - python-version: "3.10" runs-on: ubuntu-latest cmake-version: "3.22.x" diff --git a/pyproject.toml b/pyproject.toml index c04af273f..ebd2b2d2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -271,9 +271,10 @@ exclude = [] [tool.ruff.per-file-ignores] "tests/**" = ["T20"] "noxfile.py" = ["T20", "TID251"] -"src/scikit_build_core/resources/*.py" = ["PTH", "ARG002", "FBT"] +"src/scikit_build_core/resources/*.py" = ["PTH", "ARG002", "FBT", "TID251"] "src/scikit_build_core/_compat/**.py" = ["TID251"] "tests/conftest.py" = ["TID251"] +"tests/packages/**.py" = ["TID251"] "docs/conf.py" = ["TID251"] diff --git a/src/scikit_build_core/resources/_editable_redirect.py b/src/scikit_build_core/resources/_editable_redirect.py index 886ee6b2c..9f0ddace5 100644 --- a/src/scikit_build_core/resources/_editable_redirect.py +++ b/src/scikit_build_core/resources/_editable_redirect.py @@ -1,12 +1,27 @@ from __future__ import annotations import importlib.abc -import importlib.machinery import importlib.util import os import subprocess import sys +TYPE_CHECKING = False +if TYPE_CHECKING: + import importlib.machinery + + if sys.version_info < (3, 8): + from typing_extensions import TypedDict + else: + from typing import TypedDict + + class KWDict_1(TypedDict, total=False): + submodule_search_locations: list[str] + +else: + KWDict_1 = dict + + DIR = os.path.abspath(os.path.dirname(__file__)) MARKER = "SKBUILD_EDITABLE_SKIP" VERBOSE = "SKBUILD_EDITABLE_VERBOSE" @@ -36,6 +51,35 @@ def __init__( self.verbose = verbose self.build_options = build_options self.install_options = install_options + # Construct the __path__ of all resource files + # I.e. the paths of all package-like objects + submodule_search_locations: dict[str, set[str]] = {} + pkgs: list[str] = [] + # Loop over both python native source files and cmake installed ones + for tree in (known_source_files, known_wheel_files): + for module, file in tree.items(): + # Strip the last element of the module + parent = ".".join(module.split(".")[:-1]) + # Check if it is a package + if "__init__.py" in file: + parent = module + pkgs.append(parent) + # Skip if it's a root module (there are no search paths for these) + if not parent: + continue + # Initialize the tree element if needed + submodule_search_locations.setdefault(parent, set()) + # Add the parent path to the dictionary values + parent_path, _ = os.path.split(file) + if not parent_path: + # root modules are skipped so all files should be in a parent package + msg = f"Unexpected path to source file: {file} [{module}]" + raise ImportError(msg) + if not os.path.isabs(parent_path): + parent_path = os.path.join(str(DIR), parent_path) + submodule_search_locations[parent].add(parent_path) + self.submodule_search_locations = submodule_search_locations + self.pkgs = pkgs def find_spec( self, @@ -43,17 +87,29 @@ def find_spec( path: object = None, target: object = None, ) -> importlib.machinery.ModuleSpec | None: + # If current item is a know package use its search locations, otherwise if it's a module use the parent's + parent = ( + fullname if fullname in self.pkgs else ".".join(fullname.split(".")[:-1]) + ) + # If no known submodule_search_locations is found, it means it is a root module. Do not populate its kwargs + # in that case + kwargs = KWDict_1() + if parent in self.submodule_search_locations: + kwargs["submodule_search_locations"] = list( + self.submodule_search_locations[parent] + ) if fullname in self.known_wheel_files: redir = self.known_wheel_files[fullname] if self.rebuild_flag: self.rebuild() return importlib.util.spec_from_file_location( - fullname, os.path.join(DIR, redir) + fullname, + os.path.join(DIR, redir), + **kwargs, ) if fullname in self.known_source_files: redir = self.known_source_files[fullname] - return importlib.util.spec_from_file_location(fullname, redir) - + return importlib.util.spec_from_file_location(fullname, redir, **kwargs) return None def rebuild(self) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index efc5fc0e9..ef52183f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,6 +92,12 @@ def __init__(self, env_dir: Path, *, wheelhouse: Path | None = None) -> None: self.wheelhouse = wheelhouse self.executable = Path(result.creator.exe) self.env_dir = env_dir.resolve() + self.platlib = Path( + self.execute("import sysconfig; print(sysconfig.get_path('platlib'))") + ) + self.purelib = Path( + self.execute("import sysconfig; print(sysconfig.get_path('purelib'))") + ) @overload def run(self, *args: str, capture: Literal[True]) -> str: @@ -279,6 +285,15 @@ def package_simplest_c(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Packa return package +@pytest.fixture() +def navigate_editable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> PackageInfo: + package = PackageInfo( + "navigate_editable", + ) + process_package(package, tmp_path, monkeypatch) + return package + + @pytest.fixture() def package_sdist_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/packages/navigate_editable/CMakeLists.txt b/tests/packages/navigate_editable/CMakeLists.txt new file mode 100644 index 000000000..2c7f3e2e6 --- /dev/null +++ b/tests/packages/navigate_editable/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.15...3.26) + +project( + ${SKBUILD_PROJECT_NAME} + LANGUAGES C + VERSION ${SKBUILD_PROJECT_VERSION}) + +find_package(Python COMPONENTS Interpreter Development.Module) + +python_add_library(c_module MODULE src/shared_pkg/c_module.c WITH_SOABI) + +set(CMakeVar "Some_value_C") +configure_file(src/shared_pkg/data/generated.txt.in + shared_pkg/data/c_generated.txt) + +install( + TARGETS c_module + DESTINATION shared_pkg/ + COMPONENT PythonModule) +install(FILES ${PROJECT_BINARY_DIR}/shared_pkg/data/c_generated.txt + DESTINATION shared_pkg/data/) diff --git a/tests/packages/navigate_editable/pyproject.toml b/tests/packages/navigate_editable/pyproject.toml new file mode 100644 index 000000000..abaef4e82 --- /dev/null +++ b/tests/packages/navigate_editable/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +[project] +name = "navigate_editable" +version = "0.0.1" +dependencies = [ + "importlib-resources; python_version<'3.9'" +] + +[tool.scikit-build] +wheel.packages = ["python/shared_pkg"] diff --git a/tests/packages/navigate_editable/python/shared_pkg/__init__.py b/tests/packages/navigate_editable/python/shared_pkg/__init__.py new file mode 100644 index 000000000..63096923e --- /dev/null +++ b/tests/packages/navigate_editable/python/shared_pkg/__init__.py @@ -0,0 +1,9 @@ +from .c_module import call_py_method +from .py_module import call_c_method, read_c_generated_txt, read_py_data_txt + +__all__ = [ + "call_py_method", + "call_c_method", + "read_py_data_txt", + "read_c_generated_txt", +] diff --git a/tests/packages/navigate_editable/python/shared_pkg/c_module.pyi b/tests/packages/navigate_editable/python/shared_pkg/c_module.pyi new file mode 100644 index 000000000..4636e1015 --- /dev/null +++ b/tests/packages/navigate_editable/python/shared_pkg/c_module.pyi @@ -0,0 +1,2 @@ +def c_method() -> str: ... +def call_py_method() -> None: ... diff --git a/tests/packages/navigate_editable/python/shared_pkg/data/py_data.txt b/tests/packages/navigate_editable/python/shared_pkg/data/py_data.txt new file mode 100644 index 000000000..21fb55e18 --- /dev/null +++ b/tests/packages/navigate_editable/python/shared_pkg/data/py_data.txt @@ -0,0 +1 @@ +Some_value_Py diff --git a/tests/packages/navigate_editable/python/shared_pkg/py_module.py b/tests/packages/navigate_editable/python/shared_pkg/py_module.py new file mode 100644 index 000000000..8ec79b73b --- /dev/null +++ b/tests/packages/navigate_editable/python/shared_pkg/py_module.py @@ -0,0 +1,28 @@ +import sys + +if sys.version_info < (3, 9): + from importlib_resources import files +else: + from importlib.resources import files + +from .c_module import c_method + + +def call_c_method(): + print(c_method()) + + +def py_method(): + print("py_method") + + +def read_py_data_txt(): + root = files("shared_pkg.data") + py_data = root / "py_data.txt" + print(py_data.read_text()) + + +def read_c_generated_txt(): + root = files("shared_pkg.data") + c_generated_txt = root / "c_generated.txt" + print(c_generated_txt.read_text()) diff --git a/tests/packages/navigate_editable/src/shared_pkg/c_module.c b/tests/packages/navigate_editable/src/shared_pkg/c_module.c new file mode 100644 index 000000000..3898e2d0f --- /dev/null +++ b/tests/packages/navigate_editable/src/shared_pkg/c_module.c @@ -0,0 +1,54 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include + +const char* c_method() { return "c_method"; } + +static PyObject *c_method_wrapper(PyObject *self, PyObject *args) { + return PyUnicode_FromString(c_method()); +} + +static PyObject *py_method_wrapper(PyObject *self, PyObject *args) { + PyObject *py_module = PyImport_ImportModule("shared_pkg.py_module"); + if (py_module == NULL) { + PyErr_Print(); + fprintf(stderr, "Failed to load shared_pkg.py_module\n"); + exit(1); + } + PyObject *py_method = PyObject_GetAttrString(py_module,(char*)"py_method"); + if (py_method == NULL) { + PyErr_Print(); + fprintf(stderr, "Failed to load shared_pkg.py_module.py_method\n"); + exit(1); + } + +#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION > 8 + PyObject *res = PyObject_CallNoArgs(py_method); +#else + PyObject *res = PyObject_CallObject(py_method, NULL); +#endif + + if (res == NULL) { + PyErr_Print(); + fprintf(stderr, "Failed to execute shared_pkg.py_module.py_method\n"); + exit(1); + } + + Py_DECREF(py_module); + Py_DECREF(py_method); + Py_DECREF(res); + Py_RETURN_NONE; +} + +static PyMethodDef c_module_methods[] = { + {"c_method", c_method_wrapper, METH_NOARGS, "C native method"}, + {"call_py_method", py_method_wrapper, METH_NOARGS, "Call python native method"}, + {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef c_module = {PyModuleDef_HEAD_INIT, "c_module", + NULL, -1, c_module_methods}; + +PyMODINIT_FUNC PyInit_c_module(void) { + return PyModule_Create(&c_module); +} diff --git a/tests/packages/navigate_editable/src/shared_pkg/data/generated.txt.in b/tests/packages/navigate_editable/src/shared_pkg/data/generated.txt.in new file mode 100644 index 000000000..40bda1771 --- /dev/null +++ b/tests/packages/navigate_editable/src/shared_pkg/data/generated.txt.in @@ -0,0 +1 @@ +@CMakeVar@ diff --git a/tests/test_editable.py b/tests/test_editable.py new file mode 100644 index 000000000..1b3805d2a --- /dev/null +++ b/tests/test_editable.py @@ -0,0 +1,50 @@ +import sys +from pathlib import Path + +import pytest + + +@pytest.mark.compile() +@pytest.mark.configure() +@pytest.mark.integration() +@pytest.mark.parametrize("isolate", [True, False], ids=["isolated", "notisolated"]) +@pytest.mark.parametrize( + "package", + [ + pytest.param( + True, + id="package", + marks=[pytest.mark.xfail(reason="Only data folders supported currently")], + ), + pytest.param(False, id="datafolder"), + ], +) +@pytest.mark.usefixtures("navigate_editable") +@pytest.mark.xfail( + sys.version_info[:2] == (3, 9), reason="Python 3.9 not supported yet" +) +def test_navigate_editable(isolated, isolate, package): + isolate_args = ["--no-build-isolation"] if not isolate else [] + isolated.install("pip>=23") + if not isolate: + isolated.install("scikit-build-core[pyproject]") + + if package: + init_py = Path("python/shared_pkg/data/__init__.py") + init_py.touch() + + isolated.install( + "-v", "--config-settings=build-dir=build/{wheel_tag}", *isolate_args, "-e", "." + ) + + value = isolated.execute("import shared_pkg; shared_pkg.call_c_method()") + assert value == "c_method" + + value = isolated.execute("import shared_pkg; shared_pkg.call_py_method()") + assert value == "py_method" + + value = isolated.execute("import shared_pkg; shared_pkg.read_py_data_txt()") + assert value == "Some_value_Py" + + value = isolated.execute("import shared_pkg; shared_pkg.read_c_generated_txt()") + assert value == "Some_value_C" diff --git a/tests/test_pyproject_pep660.py b/tests/test_pyproject_pep660.py index 80123bff2..c66ff3f75 100644 --- a/tests/test_pyproject_pep660.py +++ b/tests/test_pyproject_pep660.py @@ -1,4 +1,5 @@ import sys +import sysconfig import zipfile from pathlib import Path @@ -57,8 +58,40 @@ def test_pep660_pip_isolated(isolated, isolate): value = isolated.execute("import simplest; print(simplest.square(2))") assert value == "4.0" - location = isolated.execute("import simplest; print(*simplest.__path__)") - assert location == str(Path.cwd() / "src/simplest") + location_str = isolated.execute( + "import simplest; print(*simplest.__path__, sep=';')" + ) + locations = [Path(s).resolve() for s in location_str.split(";")] + + # First path is from the python source + python_source = Path("src/simplest").resolve() + assert any(x.samefile(python_source) for x in locations) + + # Second path is from the CMake install + cmake_install = isolated.platlib.joinpath("simplest").resolve() + assert any(x.samefile(cmake_install) for x in locations) location = isolated.execute("import simplest; print(simplest.__file__)") - assert location == str(Path.cwd() / "src/simplest/__init__.py") + # The package file is defined in the python source and __file__ must point to it + assert Path("src/simplest/__init__.py").resolve().samefile(Path(location).resolve()) + + location = isolated.execute( + "import simplest._module; print(simplest._module.__file__)" + ) + if sys.version_info < (3, 8, 7): + import distutils.sysconfig # pylint: disable=deprecated-module + + ext_suffix = distutils.sysconfig.get_config_var("EXT_SUFFIX") + else: + ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + + module_file = cmake_install / f"_module{ext_suffix}" + # Windows FindPython may produce the wrong extension + if ( + sys.version_info < (3, 8, 7) + and sys.platform.startswith("win") + and not module_file.is_file() + ): + module_file = cmake_install / "_module.pyd" + + assert module_file.samefile(Path(location).resolve())