diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4cdd7aab02..4ced21baaa 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -210,6 +210,108 @@ cmake -S pybind11/ -B build cmake --build build ``` +### Explanation of the SDist/wheel building design + +> These details below are _only_ for packaging the Python sources from git. The +> SDists and wheels created do not have any extra requirements at all and are +> completely normal. + +The main objective of the packaging system is to create SDists (Python's source +distribution packages) and wheels (Python's binary distribution packages) that +include everything that is needed to work with pybind11, and which can be +installed without any additional dependencies. This is more complex than it +appears: in order to support CMake as a first class language even when using +the PyPI package, they must include the _generated_ CMake files (so as not to +require CMake when installing the `pybind11` package itself). They should also +provide the option to install to the "standard" location +(`/include/pybind11` and `/share/cmake/pybind11`) so they are +easy to find with CMake, but this can cause problems if you are not an +environment or using ``pyproject.toml`` requirements. This was solved by having +two packages; the "nice" pybind11 package that stores the includes and CMake +files inside the package, that you get access to via functions in the package, +and a `pybind11-global` package that can be included via `pybind11[global]` if +you want the more invasive but discoverable file locations. + +If you want to install or package the GitHub source, it is best to have Pip 10 +or newer on Windows, macOS, or Linux (manylinux1 compatible, includes most +distributions). You can then build the SDists, or run any procedure that makes +SDists internally, like making wheels or installing. + + +```bash +# Editable development install example +python3 -m pip install -e . +``` + +Since Pip itself does not have an `sdist` command (it does have `wheel` and +`install`), you may want to use the upcoming `build` package: + +```bash +python3 -m pip install build + +# Normal package +python3 -m build -s . + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 -m build -s . +``` + +If you want to use the classic "direct" usage of `python setup.py`, you will +need CMake 3.15+ and either `make` or `ninja` preinstalled (possibly via `pip +install cmake ninja`), since directly running Python on `setup.py` cannot pick +up and install `pyproject.toml` requirements. As long as you have those two +things, though, everything works the way you would expect: + +```bash +# Normal package +python3 setup.py sdist + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 setup.py sdist +``` + +A detailed explanation of the build procedure design for developers wanting to +work on or maintain the packaging system is as follows: + +#### 1. Building from the source directory + +When you invoke any `setup.py` command from the source directory, including +`pip wheel .` and `pip install .`, you will activate a full source build. This +is made of the following steps: + +1. If the tool is PEP 518 compliant, like Pip 10+, it will create a temporary + virtual environment and install the build requirements (mostly CMake) into + it. (if you are not on Windows, macOS, or a manylinux compliant system, you + can disable this with `--no-build-isolation` as long as you have CMake 3.15+ + installed) +2. The environment variable `PYBIND11_GLOBAL_SDIST` is checked - if it is set + and truthy, this will be make the accessory `pybind11-global` package, + instead of the normal `pybind11` package. This package is used for + installing the files directly to your environment root directory, using + `pybind11[global]`. +2. `setup.py` reads the version from `pybind11/_version.py` and verifies it + matches `includes/pybind11/detail/common.h`. +3. CMake is run with `-DCMAKE_INSTALL_PREIFX=pybind11`. Since the CMake install + procedure uses only relative paths and is identical on all platforms, these + files are valid as long as they stay in the correct relative position to the + includes. `pybind11/share/cmake/pybind11` has the CMake files, and + `pybind11/include` has the includes. The build directory is discarded. +4. Simpler files are placed in the SDist: `tools/setup_*.py.in`, + `tools/pyproject.toml` (`main` or `global`) +5. The package is created by running the setup function in the + `tools/setup_*.py`. `setup_main.py` fills in Python packages, and + `setup_global.py` fills in only the data/header slots. +6. A context manager cleans up the temporary CMake install directory (even if + an error is thrown). + +### 2. Building from SDist + +Since the SDist has the rendered template files in `tools` along with the +includes and CMake files in the correct locations, the builds are completely +trivial and simple. No extra requirements are required. You can even use Pip 9 +if you really want to. + + [pre-commit]: https://pre-commit.com [pybind11.readthedocs.org]: http://pybind11.readthedocs.org/en/latest [issue tracker]: https://github.com/pybind/pybind11/issues diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 530d8caf38..1749d07f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,8 @@ on: - v* jobs: + # This is the "main" test suite, which tests a large number of different + # versions of default compilers and Python versions in GitHub Actions. standard: strategy: fail-fast: false @@ -23,6 +25,12 @@ jobs: - pypy2 - pypy3 + # Items in here will either be added to the build matrix (if not + # present), or add new keys to an existing matrix element if all the + # existing keys match. + # + # We support three optional keys: args (both build), args1 (first + # build), and args2 (second build). include: - runs-on: ubuntu-latest python: 3.6 @@ -52,6 +60,7 @@ jobs: args: > -DPYBIND11_FINDPYTHON=ON + # These items will be removed from the build matrix, keys must match. exclude: # Currently 32bit only, and we build 64bit - runs-on: windows-latest @@ -102,10 +111,11 @@ jobs: - name: Prepare env run: python -m pip install -r tests/requirements.txt --prefer-binary - - name: Setup annotations + - name: Setup annotations on Linux if: runner.os == 'Linux' run: python -m pip install pytest-github-actions-annotate-failures + # First build - C++11 mode and inplace - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . @@ -130,6 +140,7 @@ jobs: - name: Clean directory run: git clean -fdx + # Second build - C++17 mode and in a build directory - name: Configure ${{ matrix.args2 }} run: > cmake -S . -B build2 @@ -152,6 +163,28 @@ jobs: - name: Interface test run: cmake --build build2 --target test_cmake_build + # Eventually Microsoft might have an action for setting up + # MSVC, but for now, this action works: + - name: Prepare compiler environment for Windows 🐍 2.7 + if: matrix.python == 2.7 && runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + # This makes two environment variables available in the following step(s) + - name: Set Windows 🐍 2.7 environment variables + if: matrix.python == 2.7 && runner.os == 'Windows' + run: | + echo "::set-env name=DISTUTILS_USE_SDK::1" + echo "::set-env name=MSSdk::1" + + # This makes sure the setup_helpers module can build packages using + # setuptools + - name: Setuptools helpers test + run: pytest tests/extra_setuptools + + + # Testing on clang using the excellent silkeh clang docker images clang: runs-on: ubuntu-latest strategy: @@ -196,6 +229,7 @@ jobs: run: cmake --build build --target test_cmake_build + # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04" @@ -218,6 +252,7 @@ jobs: run: cmake --build build --target pytest + # Testing CentOS 8 + PGI compilers centos-nvhpc8: runs-on: ubuntu-latest name: "🐍 3 • CentOS8 / PGI 20.7 • x64" @@ -256,6 +291,8 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + + # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: runs-on: ubuntu-latest name: "🐍 3 • CentOS7 / PGI 20.7 • x64" @@ -303,6 +340,7 @@ jobs: - name: Interface test run: cmake3 --build build --target test_cmake_build + # Testing on GCC using the GCC docker images (only recent images supported) gcc: runs-on: ubuntu-latest strategy: @@ -351,6 +389,8 @@ jobs: run: cmake --build build --target test_cmake_build + # Testing on CentOS (manylinux uses a centos base, and this is an easy way + # to get GCC 4.8, which is the manylinux1 compiler). centos: runs-on: ubuntu-latest strategy: @@ -398,6 +438,7 @@ jobs: run: cmake --build build --target test_cmake_build + # This tests an "install" with the CMake tools install-classic: name: "🐍 3.5 • Debian • x86 • Install" runs-on: ubuntu-latest @@ -440,22 +481,22 @@ jobs: working-directory: /build-tests + # This verifies that the documentation is not horribly broken, and does a + # basic sanity check on the SDist. doxygen: name: "Documentation build test" runs-on: ubuntu-latest - container: alpine:3.12 steps: - uses: actions/checkout@v2 - - name: Install system requirements - run: apk add doxygen python3-dev + - uses: actions/setup-python@v2 - - name: Ensure pip - run: python3 -m ensurepip + - name: Install Doxygen + run: sudo apt install -y doxygen - name: Install docs & setup requirements - run: python3 -m pip install -r docs/requirements.txt pytest setuptools + run: python3 -m pip install -r docs/requirements.txt - name: Build docs run: python3 -m sphinx -W -b html docs docs/.build @@ -463,8 +504,16 @@ jobs: - name: Make SDist run: python3 setup.py sdist + - run: git status --ignored + + - name: Check local include dir + run: > + ls pybind11; + python3 -c "import pybind11, pathlib; assert (a := pybind11.get_include()) == (b := str(pathlib.Path('include').resolve())), f'{a} != {b}'" + - name: Compare Dists (headers only) + working-directory: include run: | - python3 -m pip install --user -U ./dist/* - installed=$(python3 -c "import pybind11; print(pybind11.get_include(True) + '/pybind11')") - diff -rq $installed ./include/pybind11 + python3 -m pip install --user -U ../dist/* + installed=$(python3 -c "import pybind11; print(pybind11.get_include() + '/pybind11')") + diff -rq $installed ./pybind11 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index d472f4b191..3dd248e04a 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -10,6 +10,8 @@ on: - v* jobs: + # This tests various versions of CMake in various combinations, to make sure + # the configure step passes. cmake: strategy: fail-fast: false @@ -50,11 +52,14 @@ jobs: - name: Prepare env run: python -m pip install -r tests/requirements.txt + # An action for adding a specific version of CMake: + # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} uses: jwlawson/actions-setup-cmake@v1.3 with: cmake-version: ${{ matrix.cmake }} + # These steps use a directory with a space in it intentionally - name: Make build directories run: mkdir "build dir" @@ -67,6 +72,7 @@ jobs: -DDOWNLOAD_CATCH=ON -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + # Only build and test if this was manually triggered in the GitHub UI - name: Build working-directory: build dir if: github.event_name == 'workflow_dispatch' @@ -76,3 +82,57 @@ jobs: working-directory: build dir if: github.event_name == 'workflow_dispatch' run: cmake --build . --config Release --target check + + # This builds the sdists and wheels and makes sure the files are exactly as + # expected. Using Windows and Python 2.7, since that is often the most + # challenging matrix element. + test-packaging: + name: 🐍 2.7 • 📦 tests • windows-latest + runs-on: windows-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup 🐍 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + + - name: Prepare env + run: python -m pip install -r tests/requirements.txt --prefer-binary + + - name: Python Packaging tests + run: pytest tests/extra_python_package/ + + + # This runs the packaging tests and also builds and saves the packages as + # artifacts. + packaging: + name: 🐍 3.8 • 📦 & 📦 tests • ubuntu-latest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup 🐍 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Prepare env + run: python -m pip install -r tests/requirements.txt build twine --prefer-binary + + - name: Python Packaging tests + run: pytest tests/extra_python_package/ + + - name: Build SDist and wheels + run: | + python -m build -s -w . + PYBIND11_GLOBAL_SDIST=1 python -m build -s -w . + + - name: Check metadata + run: twine check dist/* + + - uses: actions/upload-artifact@v2 + with: + path: dist/* diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 191219326d..28cfeb9b7d 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -1,3 +1,6 @@ +# This is a format job. Pre-commit has a first-party GitHub action, so we use +# that: https://github.com/pre-commit/action + name: Format on: @@ -17,6 +20,9 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.0 + with: + # Slow hooks are marked with manual - slow is okay here, run them too + extra_args: --hook-stage manual clang-tidy: name: Clang-Tidy diff --git a/.gitignore b/.gitignore index 5613b367d2..3f36b89e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ pybind11Config*.cmake pybind11Targets.cmake /*env* /.vscode +/pybind11/include/* +/pybind11/share/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6863f4c495..71513c991c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,21 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://github.com/pre-commit/pre-commit + repos: +# Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.2.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -14,11 +29,21 @@ repos: - id: trailing-whitespace - id: fix-encoding-pragma +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + # Not all Python files are Blacked, yet + files: ^(setup.py|pybind11|tests/extra) + +# Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.9 hooks: - id: remove-tabs +# Flake8 also supports pre-commit natively (same author) - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: @@ -26,6 +51,7 @@ repos: additional_dependencies: [flake8-bugbear, pep8-naming] exclude: ^(docs/.*|tools/.*)$ +# CMake formatting - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.11 hooks: @@ -34,6 +60,16 @@ repos: types: [file] files: (\.cmake|CMakeLists.txt)(.in)?$ +# Checks the manifest for missing files (native support) +- repo: https://github.com/mgedmin/check-manifest + rev: "0.42" + hooks: + - id: check-manifest + # This is a slow hook, so only run this if --hook-stage manual is passed + stages: [manual] + additional_dependencies: [cmake, ninja] + +# The original pybind11 checks for a few C++ style items - repo: local hooks: - id: disallow-caps diff --git a/CMakeLists.txt b/CMakeLists.txt index 67287d54ba..123abf77d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,10 +26,10 @@ foreach(ver ${pybind11_version_defines}) endif() endforeach() -if(PYBIND11_VERSION_PATCH MATCHES [[([a-zA-Z]+)]]) +if(PYBIND11_VERSION_PATCH MATCHES [[\.([a-zA-Z0-9]+)$]]) set(pybind11_VERSION_TYPE "${CMAKE_MATCH_1}") endif() -string(REGEX MATCH "[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") +string(REGEX MATCH "^[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") project( pybind11 diff --git a/MANIFEST.in b/MANIFEST.in index 6fe84ced8d..9336b60302 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ -recursive-include include/pybind11 *.h -include LICENSE README.md .github/CONTRIBUTING.md +recursive-include pybind11/include/pybind11 *.h +recursive-include pybind11 *.py +include pybind11/share/cmake/pybind11/*.cmake +include LICENSE README.md pyproject.toml setup.py setup.cfg diff --git a/docs/basics.rst b/docs/basics.rst index 6bb5f98222..71440c9c66 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -11,11 +11,11 @@ included set of test cases. Compiling the test cases ======================== -Linux/MacOS +Linux/macOS ----------- On Linux you'll need to install the **python-dev** or **python3-dev** packages as -well as **cmake**. On Mac OS, the included python version works out of the box, +well as **cmake**. On macOS, the included python version works out of the box, but **cmake** must still be installed. After installing the prerequisites, run @@ -138,7 +138,7 @@ On Linux, the above example can be compiled using the following command: $ c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` -For more details on the required compiler flags on Linux and MacOS, see +For more details on the required compiler flags on Linux and macOS, see :ref:`building_manually`. For complete cross-platform compilation instructions, refer to the :ref:`compiling` page. diff --git a/docs/changelog.rst b/docs/changelog.rst index 3546040033..92900a5b74 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,25 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. * ``py::memoryview`` update and documentation. `#2223 `_ +* The Python package was reworked to be more powerful and useful. + `#2433 `_ + + * :ref:`build-setuptools` is easier thanks to a new + ``pybind11.setup_helpers`` module, which provides utilities to use + setuptools with pybind11. It can be used via PEP 518, ``setup_requires``, + or by directly copying ``setup_helpers.py`` into your project. + + * CMake configuration files are now included in the Python package. Use + ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get + the directory with the CMake configuration files, or include the + site-packages location in your ``CMAKE_MODULE_PATH``. Or you can use the + new ``pybind11[global]`` extra when you install ``pybind11``, which + installs the CMake files and headers into your base environment in the + standard location + + * ``pybind11-config`` is another way to write ``python -m pybind11`` if you + have your PATH set up. + * Minimum CMake required increased to 3.4. `#2338 `_ and `#2370 `_ diff --git a/docs/compiling.rst b/docs/compiling.rst index ca4dc756e6..344dac6988 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -3,6 +3,8 @@ Build systems ############# +.. _build-setuptools: + Building with setuptools ======================== @@ -13,6 +15,135 @@ the [python_example]_ repository. .. [python_example] https://github.com/pybind/python_example +A helper file is provided with pybind11 that can simplify usage with setuptools. + +To use pybind11 inside your ``setup.py``, you have to have some system to +ensure that ``pybind11`` is installed when you build your package. There are +four possible ways to do this, and pybind11 supports all four: You can ask all +users to install pybind11 beforehand (bad), you can use +:ref:`setup_helpers-pep518` (good, but very new and requires Pip 10), +:ref:`setup_helpers-setup_requires` (discouraged by Python packagers now that +PEP 518 is available, but it still works everywhere), or you can +:ref:`setup_helpers-copy-manually` (always works but you have to manually sync +your copy to get updates). + +An example of a ``setup.py`` using pybind11's helpers: + +.. code-block:: python + + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension + + ext_modules = [ + Pybind11Extension( + "python_example", + ["src/main.cpp"], + ), + ] + + setup( + ..., + ext_modules=ext_modules + ) + +If you want to do an automatic search for the highest supported C++ standard, +that is supported via a ``build_ext`` command override; it will only affect +``Pybind11Extensions``: + +.. code-block:: python + + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension, build_ext + + ext_modules = [ + Pybind11Extension( + "python_example", + ["src/main.cpp"], + ), + ] + + setup( + ..., + cmdclass={"build_ext": build_ext}, + ext_modules=ext_modules + ) + +.. _setup_helpers-pep518: + +PEP 518 requirements (Pip 10+ required) +--------------------------------------- + +If you use `PEP 518's `_ +``pyproject.toml`` file, you can ensure that ``pybind11`` is available during +the compilation of your project. When this file exists, Pip will make a new +virtual environment, download just the packages listed here in ``requires=``, +and build a wheel (binary Python package). It will then throw away the +environment, and install your wheel. + +Your ``pyproject.toml`` file will likely look something like this: + +.. code-block:: toml + + [build-system] + requires = ["setuptools", "wheel", "pybind11==2.6.0"] + build-backend = "setuptools.build_meta" + +.. note:: + + The main drawback to this method is that a `PEP 517`_ compliant build tool, + such as Pip 10+, is required for this approach to work; older versions of + Pip completely ignore this file. If you distribute binaries (called wheels + in Python) using something like `cibuildwheel`_, remember that ``setup.py`` + and ``pyproject.toml`` are not even contained in the wheel, so this high + Pip requirement is only for source builds, and will not affect users of + your binary wheels. + +.. _PEP 517: https://www.python.org/dev/peps/pep-0517/ +.. _cibuildwheel: https://cibuildwheel.readthedocs.io + +.. _setup_helpers-setup_requires: + +Classic ``setup_requires`` +-------------------------- + +If you want to support old versions of Pip with the classic +``setup_requires=["pybind11"]`` keyword argument to setup, which triggers a +two-phase ``setup.py`` run, then you will need to use something like this to +ensure the first pass works (which has not yet installed the ``setup_requires`` +packages, since it can't install something it does not know about): + +.. code-block:: python + + try: + from pybind11.setup_helpers import Pybind11Extension + except ImportError: + from setuptools import Extension as Pybind11Extension + + +It doesn't matter that the Extension class is not the enhanced subclass for the +first pass run; and the second pass will have the ``setup_requires`` +requirements. + +This is obviously more of a hack than the PEP 518 method, but it supports +ancient versions of Pip. + +.. _setup_helpers-copy-manually: + +Copy manually +------------- + +You can also copy ``setup_helpers.py`` directly to your project; it was +designed to be usable standalone, like the old example ``setup.py``. You can +set ``include_pybind11=False`` to skip including the pybind11 package headers, +so you can use it with git submodules and a specific git version. If you use +this, you will need to import from a local file in ``setup.py`` and ensure the +helper file is part of your MANIFEST. + + +.. versionchanged:: 2.6 + + Added ``setup_helpers`` file. + Building with cppimport ======================== @@ -367,7 +498,7 @@ Besides, the ``--extension-suffix`` option may or may not be available, dependin on the distribution; in the latter case, the module extension can be manually set to ``.so``. -On Mac OS: the build command is almost the same but it also requires passing +On macOS: the build command is almost the same but it also requires passing the ``-undefined dynamic_lookup`` flag so as to ignore missing symbols when building the module: diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 502ce76eed..2164996915 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -25,6 +25,13 @@ Usage of the ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function should be replaced by ``PYBIND11_OVERRIDE*`` and ``get_override``. In the future, the old macros may be deprecated and removed. +The ``pybind11`` package on PyPI no longer fills the wheel "headers" slot - if +you were using the headers from this slot, they are available by requesting the +``global`` extra, that is, ``pip install "pybind11[global]"``. (Most users will +be unaffected, as the ``pybind11/include`` location is reported by ``python -m +pybind11 --includes`` and ``pybind11.get_include()`` is still correct and has +not changed since 2.5). + CMake support: -------------- @@ -63,9 +70,20 @@ In addition, the following changes may be of interest: * Using ``find_package(Python COMPONENTS Interpreter Development)`` before pybind11 will cause pybind11 to use the new Python mechanisms instead of its - own custom search, based on a patched version of classic - FindPythonInterp/FindPythonLibs. In the future, this may become the default. + own custom search, based on a patched version of classic ``FindPythonInterp`` + / ``FindPythonLibs``. In the future, this may become the default. + + + +v2.5 +==== +The Python package now includes the headers as data in the package itself, as +well as in the "headers" wheel slot. ``pybind11 --includes`` and +``pybind11.get_include()`` report the new location, which is always correct +regardless of how pybind11 was installed, making the old ``user=`` argument +meaningless. If you are not using the function to get the location already, you +are encouraged to switch to the package location. v2.2 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 0986521ac0..1f8390fbab 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,7 +11,7 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 6 -#define PYBIND11_VERSION_PATCH dev0 +#define PYBIND11_VERSION_PATCH 0.dev1 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 5b2f83d5cd..ad65420893 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -from ._version import version_info, __version__ # noqa: F401 imported but unused +from ._version import version_info, __version__ +from .commands import get_include, get_cmake_dir -def get_include(user=False): - import os - d = os.path.dirname(__file__) - if os.path.exists(os.path.join(d, "include")): - # Package is installed - return os.path.join(d, "include") - else: - # Package is from a source directory - return os.path.join(os.path.dirname(d), "include") + +__all__ = ( + "version_info", + "__version__", + "get_include", + "get_cmake_dir", +) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 5e393cc8f1..f4d5437836 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -5,13 +5,15 @@ import sys import sysconfig -from . import get_include +from .commands import get_include, get_cmake_dir def print_includes(): - dirs = [sysconfig.get_path('include'), - sysconfig.get_path('platinclude'), - get_include()] + dirs = [ + sysconfig.get_path("include"), + sysconfig.get_path("platinclude"), + get_include(), + ] # Make unique but preserve order unique_dirs = [] @@ -19,19 +21,29 @@ def print_includes(): if d not in unique_dirs: unique_dirs.append(d) - print(' '.join('-I' + d for d in unique_dirs)) + print(" ".join("-I" + d for d in unique_dirs)) def main(): - parser = argparse.ArgumentParser(prog='python -m pybind11') - parser.add_argument('--includes', action='store_true', - help='Include flags for both pybind11 and Python headers.') + parser = argparse.ArgumentParser() + parser.add_argument( + "--includes", + action="store_true", + help="Include flags for both pybind11 and Python headers.", + ) + parser.add_argument( + "--cmakedir", + action="store_true", + help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() if args.includes: print_includes() + if args.cmakedir: + print(get_cmake_dir()) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/pybind11/_version.py b/pybind11/_version.py index 1f2f254ce5..ca84c262c9 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -1,3 +1,12 @@ # -*- coding: utf-8 -*- -version_info = (2, 5, 'dev1') -__version__ = '.'.join(map(str, version_info)) + + +def _to_int(s): + try: + return int(s) + except ValueError: + return s + + +__version__ = "2.6.0.dev1" +version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/pybind11/commands.py b/pybind11/commands.py new file mode 100644 index 0000000000..fa7eac3ccd --- /dev/null +++ b/pybind11/commands.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +import os + + +DIR = os.path.abspath(os.path.dirname(__file__)) + + +def get_include(user=False): + installed_path = os.path.join(DIR, "include") + source_path = os.path.join(os.path.dirname(DIR), "include") + return installed_path if os.path.exists(installed_path) else source_path + + +def get_cmake_dir(): + cmake_installed_path = os.path.join(DIR, "share", "cmake", "pybind11") + if os.path.exists(cmake_installed_path): + return cmake_installed_path + else: + msg = "pybind11 not installed, installation required to access the CMake files" + raise ImportError(msg) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py new file mode 100644 index 0000000000..041e22689f --- /dev/null +++ b/pybind11/setup_helpers.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- + +""" +This module provides helpers for C++11+ projects using pybind11. + +LICENSE: + +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import contextlib +import os +import shutil +import sys +import tempfile +import threading +import warnings + +try: + from setuptools.command.build_ext import build_ext as _build_ext + from setuptools import Extension as _Extension +except ImportError: + from distutils.command.build_ext import build_ext as _build_ext + from distutils.extension import Extension as _Extension + +import distutils.errors + + +WIN = sys.platform.startswith("win32") +PY2 = sys.version_info[0] < 3 +MACOS = sys.platform.startswith("darwin") +STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" + + +# It is recommended to use PEP 518 builds if using this module. However, this +# file explicitly supports being copied into a user's project directory +# standalone, and pulling pybind11 with the deprecated setup_requires feature. +# If you copy the file, remember to add it to your MANIFEST.in, and add the current +# directory into your path if it sits beside your setup.py. + + +class Pybind11Extension(_Extension): + """ + Build a C++11+ Extension module with pybind11. This automatically adds the + recommended flags when you init the extension and assumes C++ sources - you + can further modify the options yourself. + + The customizations are: + + * ``/EHsc`` and ``/bigobj`` on Windows + * ``stdlib=libc++`` on macOS + * ``visibility=hidden`` and ``-g0`` on Unix + + Finally, you can set ``cxx_std`` via constructor or afterwords to enable + flags for C++ std, and a few extra helper flags related to the C++ standard + level. It is _highly_ recommended you either set this, or use the provided + ``build_ext``, which will search for the highest supported extension for + you if the ``cxx_std`` property is not set. Do not set the ``cxx_std`` + property more than once, as flags are added when you set it. Set the + property to None to disable the addition of C++ standard flags. + + If you want to add pybind11 headers manually, for example for an exact + git checkout, then set ``include_pybind11=False``. + + Warning: do not use property-based access to the instance on Python 2 - + this is an ugly old-style class due to Distutils. + """ + + def _add_cflags(self, *flags): + for flag in flags: + if flag not in self.extra_compile_args: + self.extra_compile_args.append(flag) + + def _add_lflags(self, *flags): + for flag in flags: + if flag not in self.extra_compile_args: + self.extra_link_args.append(flag) + + def __init__(self, *args, **kwargs): + + self._cxx_level = 0 + cxx_std = kwargs.pop("cxx_std", 0) + + if "language" not in kwargs: + kwargs["language"] = "c++" + + include_pybind11 = kwargs.pop("include_pybind11", True) + + # Can't use super here because distutils has old-style classes in + # Python 2! + _Extension.__init__(self, *args, **kwargs) + + # Include the installed package pybind11 headers + if include_pybind11: + # If using setup_requires, this fails the first time - that's okay + try: + import pybind11 + + pyinc = pybind11.get_include() + + if pyinc not in self.include_dirs: + self.include_dirs.append(pyinc) + except ImportError: + pass + + # Have to use the accessor manually to support Python 2 distutils + Pybind11Extension.cxx_std.__set__(self, cxx_std) + + if WIN: + self._add_cflags("/EHsc", "/bigobj") + else: + self._add_cflags("-fvisibility=hidden", "-g0") + if MACOS: + self._add_cflags("-stdlib=libc++") + self._add_lflags("-stdlib=libc++") + + @property + def cxx_std(self): + """ + The CXX standard level. If set, will add the required flags. If left + at 0, it will trigger an automatic search when pybind11's build_ext + is used. If None, will have no effect. Besides just the flags, this + may add a register warning/error fix for Python 2 or macos-min 10.9 + or 10.14. + """ + return self._cxx_level + + @cxx_std.setter + def cxx_std(self, level): + + if self._cxx_level: + warnings.warn("You cannot safely change the cxx_level after setting it!") + + # MSVC 2015 Update 3 and later only have 14 (and later 17) modes + if WIN and level == 11: + level = 14 + + self._cxx_level = level + + if not level: + return + + self.extra_compile_args.append(STD_TMPL.format(level)) + + if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ: + # C++17 requires a higher min version of macOS + macosx_min = "-mmacosx-version-min=" + ("10.9" if level < 17 else "10.14") + self.extra_compile_args.append(macosx_min) + self.extra_link_args.append(macosx_min) + + if PY2: + if level >= 17: + self.extra_compile_args.append("/wd503" if WIN else "-Wno-register") + elif not WIN and level >= 14: + self.extra_compile_args.append("-Wno-deprecated-register") + + +# Just in case someone clever tries to multithread +tmp_chdir_lock = threading.Lock() +cpp_cache_lock = threading.Lock() + + +@contextlib.contextmanager +def tmp_chdir(): + "Prepare and enter a temporary directory, cleanup when done" + + # Threadsafe + with tmp_chdir_lock: + olddir = os.getcwd() + try: + tmpdir = tempfile.mkdtemp() + os.chdir(tmpdir) + yield tmpdir + finally: + os.chdir(olddir) + shutil.rmtree(tmpdir) + + +# cf http://bugs.python.org/issue26689 +def has_flag(compiler, flag): + """ + Return the flag if a flag name is supported on the + specified compiler, otherwise None (can be used as a boolean). + If multiple flags are passed, return the first that matches. + """ + + with tmp_chdir(): + fname = "flagcheck.cpp" + with open(fname, "w") as f: + f.write("int main (int argc, char **argv) { return 0; }") + + try: + compiler.compile([fname], extra_postargs=[flag]) + except distutils.errors.CompileError: + return False + return True + + +# Every call will cache the result +cpp_flag_cache = None + + +def auto_cpp_level(compiler): + """ + Return the max supported C++ std level (17, 14, or 11). + """ + + global cpp_flag_cache + + # If this has been previously calculated with the same args, return that + with cpp_cache_lock: + if cpp_flag_cache: + return cpp_flag_cache + + levels = [17, 14] + ([] if WIN else [11]) + + for level in levels: + if has_flag(compiler, STD_TMPL.format(level)): + with cpp_cache_lock: + cpp_flag_cache = level + return level + + msg = "Unsupported compiler -- at least C++11 support is needed!" + raise RuntimeError(msg) + + +class build_ext(_build_ext): # noqa: N801 + """ + Customized build_ext that allows an auto-search for the highest supported + C++ level for Pybind11Extension. + """ + + def build_extensions(self): + """ + Build extensions, injecting C++ std for Pybind11Extension if needed. + """ + + for ext in self.extensions: + if hasattr(ext, "_cxx_level") and ext._cxx_level == 0: + # Python 2 syntax - old-style distutils class + ext.__class__.cxx_std.__set__(ext, auto_cpp_level(self.compiler)) + + # Python 2 doesn't allow super here, since distutils uses old-style + # classes! + _build_ext.build_extensions(self) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..3bab1c1a28 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel", "cmake==3.18.0", "ninja"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 002f38d10e..ca0d59a4d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,58 @@ +[metadata] +long_description = file: README.md +long_description_content_type = text/markdown +description = Seamless operability between C++11 and Python +author = Wenzel Jakob +author_email = "wenzel.jakob@epfl.ch" +url = "https://github.com/pybind/pybind11" +license = BSD + +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + Programming Language :: C++ + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + License :: OSI Approved :: BSD License + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Python :: Implementation :: CPython + Programming Language :: C++ + Topic :: Software Development :: Libraries :: Python Modules + +keywords = + C++11 + Python bindings + +[options] +python_requires = >=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4 +zip_safe = False + [bdist_wheel] universal=1 +[check-manifest] +ignore = + tests/** + docs/** + tools/** + include/** + .appveyor.yml + .cmake-format.yaml + .gitmodules + .pre-commit-config.yaml + .readthedocs.yml + .clang-tidy + pybind11/include/** + pybind11/share/** + CMakeLists.txt + + [flake8] max-line-length = 99 show_source = True @@ -10,3 +62,5 @@ ignore = E201, E241, W504, # camelcase 'cPickle' imported as lowercase 'pickle' N813 + # Black conflict + W503, E203 diff --git a/setup.py b/setup.py index 577a6b6c37..c9ba77d6d8 100644 --- a/setup.py +++ b/setup.py @@ -3,128 +3,113 @@ # Setup script for PyPI; use CMakeFile.txt to build extension modules -from setuptools import setup -from distutils.command.install_headers import install_headers -from distutils.command.build_py import build_py -from pybind11 import __version__ +import contextlib import os +import re +import shutil +import string +import subprocess +import sys +import tempfile -package_data = [ - 'include/pybind11/detail/class.h', - 'include/pybind11/detail/common.h', - 'include/pybind11/detail/descr.h', - 'include/pybind11/detail/init.h', - 'include/pybind11/detail/internals.h', - 'include/pybind11/detail/typeid.h', - 'include/pybind11/attr.h', - 'include/pybind11/buffer_info.h', - 'include/pybind11/cast.h', - 'include/pybind11/chrono.h', - 'include/pybind11/common.h', - 'include/pybind11/complex.h', - 'include/pybind11/eigen.h', - 'include/pybind11/embed.h', - 'include/pybind11/eval.h', - 'include/pybind11/functional.h', - 'include/pybind11/iostream.h', - 'include/pybind11/numpy.h', - 'include/pybind11/operators.h', - 'include/pybind11/options.h', - 'include/pybind11/pybind11.h', - 'include/pybind11/pytypes.h', - 'include/pybind11/stl.h', - 'include/pybind11/stl_bind.h', -] - -# Prevent installation of pybind11 headers by setting -# PYBIND11_USE_CMAKE. -if os.environ.get('PYBIND11_USE_CMAKE'): - headers = [] -else: - headers = package_data - - -class InstallHeaders(install_headers): - """Use custom header installer because the default one flattens subdirectories""" - def run(self): - if not self.distribution.headers: - return - - for header in self.distribution.headers: - subdir = os.path.dirname(os.path.relpath(header, 'include/pybind11')) - install_dir = os.path.join(self.install_dir, subdir) - self.mkpath(install_dir) - - (out, _) = self.copy_file(header, install_dir) - self.outfiles.append(out) - - -# Install the headers inside the package as well -class BuildPy(build_py): - def build_package_data(self): - build_py.build_package_data(self) - for header in package_data: - target = os.path.join(self.build_lib, 'pybind11', header) - self.mkpath(os.path.dirname(target)) - self.copy_file(header, target, preserve_mode=False) - - def get_outputs(self, include_bytecode=1): - outputs = build_py.get_outputs(self, include_bytecode=include_bytecode) - for header in package_data: - target = os.path.join(self.build_lib, 'pybind11', header) - outputs.append(target) - return outputs - - -setup( - name='pybind11', - version=__version__, - description='Seamless operability between C++11 and Python', - author='Wenzel Jakob', - author_email='wenzel.jakob@epfl.ch', - url='https://github.com/pybind/pybind11', - download_url='https://github.com/pybind/pybind11/tarball/v' + __version__, - packages=['pybind11'], - license='BSD', - headers=headers, - zip_safe=False, - cmdclass=dict(install_headers=InstallHeaders, build_py=BuildPy), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Utilities', - 'Programming Language :: C++', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'License :: OSI Approved :: BSD License' - ], - keywords='C++11, Python bindings', - long_description="""pybind11 is a lightweight header-only library that -exposes C++ types in Python and vice versa, mainly to create Python bindings of -existing C++ code. Its goals and syntax are similar to the excellent -Boost.Python by David Abrahams: to minimize boilerplate code in traditional -extension modules by inferring type information using compile-time -introspection. - -The main issue with Boost.Python-and the reason for creating such a similar -project-is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. - -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations.""") +import setuptools.command.sdist + +DIR = os.path.abspath(os.path.dirname(__file__)) +VERSION_REGEX = re.compile( + r"^\s*#\s*define\s+PYBIND11_VERSION_([A-Z]+)\s+(.*)$", re.MULTILINE +) + +# PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers +# files, and the sys.prefix files (CMake and headers). + +global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) + +setup_py = "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" +extra_cmd = 'cmdclass["sdist"] = SDist\n' + +to_src = ( + ("pyproject.toml", "tools/pyproject.toml"), + ("setup.py", setup_py), +) + +# Read the listed version +with open("pybind11/_version.py") as f: + code = compile(f.read(), "pybind11/_version.py", "exec") + loc = {} + exec(code, loc) + version = loc["__version__"] + +# Verify that the version matches the one in C++ +with open("include/pybind11/detail/common.h") as f: + matches = dict(VERSION_REGEX.findall(f.read())) +cpp_version = "{MAJOR}.{MINOR}.{PATCH}".format(**matches) +if version != cpp_version: + msg = "Python version {} does not match C++ version {}!".format( + version, cpp_version + ) + raise RuntimeError(msg) + + +def get_and_replace(filename, binary=False, **opts): + with open(filename, "rb" if binary else "r") as f: + contents = f.read() + # Replacement has to be done on text in Python 3 (both work in Python 2) + if binary: + return string.Template(contents.decode()).substitute(opts).encode() + else: + return string.Template(contents).substitute(opts) + + +# Use our input files instead when making the SDist (and anything that depends +# on it, like a wheel) +class SDist(setuptools.command.sdist.sdist): + def make_release_tree(self, base_dir, files): + setuptools.command.sdist.sdist.make_release_tree(self, base_dir, files) + + for to, src in to_src: + txt = get_and_replace(src, binary=True, version=version, extra_cmd="") + + dest = os.path.join(base_dir, to) + + # This is normally linked, so unlink before writing! + os.unlink(dest) + with open(dest, "wb") as f: + f.write(txt) + + +# Backport from Python 3 +@contextlib.contextmanager +def TemporaryDirectory(): # noqa: N802 + "Prepare a temporary directory, cleanup when done" + try: + tmpdir = tempfile.mkdtemp() + yield tmpdir + finally: + shutil.rmtree(tmpdir) + + +# Remove the CMake install directory when done +@contextlib.contextmanager +def remove_output(*sources): + try: + yield + finally: + for src in sources: + shutil.rmtree(src) + + +with remove_output("pybind11/include", "pybind11/share"): + # Generate the files if they are not present. + with TemporaryDirectory() as tmpdir: + cmd = ["cmake", "-S", ".", "-B", tmpdir] + [ + "-DCMAKE_INSTALL_PREFIX=pybind11", + "-DBUILD_TESTING=OFF", + "-DPYBIND11_NOPYTHON=ON", + ] + cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call(cmd, **cmake_opts) + subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) + + txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) + code = compile(txt, setup_py, "exec") + exec(code, {"SDist": SDist}) diff --git a/tests/extra_python_package/pytest.ini b/tests/extra_python_package/pytest.ini new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py new file mode 100644 index 0000000000..ac8ca1f97b --- /dev/null +++ b/tests/extra_python_package/test_files.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +import contextlib +import os +import string +import subprocess +import sys +import tarfile +import zipfile + +# These tests must be run explicitly +# They require CMake 3.15+ (--install) + +DIR = os.path.abspath(os.path.dirname(__file__)) +MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) + + +main_headers = { + "include/pybind11/attr.h", + "include/pybind11/buffer_info.h", + "include/pybind11/cast.h", + "include/pybind11/chrono.h", + "include/pybind11/common.h", + "include/pybind11/complex.h", + "include/pybind11/eigen.h", + "include/pybind11/embed.h", + "include/pybind11/eval.h", + "include/pybind11/functional.h", + "include/pybind11/iostream.h", + "include/pybind11/numpy.h", + "include/pybind11/operators.h", + "include/pybind11/options.h", + "include/pybind11/pybind11.h", + "include/pybind11/pytypes.h", + "include/pybind11/stl.h", + "include/pybind11/stl_bind.h", +} + +detail_headers = { + "include/pybind11/detail/class.h", + "include/pybind11/detail/common.h", + "include/pybind11/detail/descr.h", + "include/pybind11/detail/init.h", + "include/pybind11/detail/internals.h", + "include/pybind11/detail/typeid.h", +} + +cmake_files = { + "share/cmake/pybind11/FindPythonLibsNew.cmake", + "share/cmake/pybind11/pybind11Common.cmake", + "share/cmake/pybind11/pybind11Config.cmake", + "share/cmake/pybind11/pybind11ConfigVersion.cmake", + "share/cmake/pybind11/pybind11NewTools.cmake", + "share/cmake/pybind11/pybind11Targets.cmake", + "share/cmake/pybind11/pybind11Tools.cmake", +} + +py_files = { + "__init__.py", + "__main__.py", + "_version.py", + "commands.py", + "setup_helpers.py", +} + +headers = main_headers | detail_headers +src_files = headers | cmake_files +all_files = src_files | py_files + + +sdist_files = { + "pybind11", + "pybind11/include", + "pybind11/include/pybind11", + "pybind11/include/pybind11/detail", + "pybind11/share", + "pybind11/share/cmake", + "pybind11/share/cmake/pybind11", + "pyproject.toml", + "setup.cfg", + "setup.py", + "LICENSE", + "MANIFEST.in", + "README.md", + "PKG-INFO", +} + +local_sdist_files = { + ".egg-info", + ".egg-info/PKG-INFO", + ".egg-info/SOURCES.txt", + ".egg-info/dependency_links.txt", + ".egg-info/not-zip-safe", + ".egg-info/top_level.txt", +} + + +def test_build_sdist(monkeypatch, tmpdir): + + monkeypatch.chdir(MAIN_DIR) + + out = subprocess.check_output( + [ + sys.executable, + "setup.py", + "sdist", + "--formats=tar", + "--dist-dir", + str(tmpdir), + ] + ) + if hasattr(out, "decode"): + out = out.decode() + + (sdist,) = tmpdir.visit("*.tar") + + with tarfile.open(str(sdist)) as tar: + start = tar.getnames()[0] + "/" + version = start[9:-1] + simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "setup.py")) + ) as f: + setup_py = f.read() + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pyproject.toml")) + ) as f: + pyproject_toml = f.read() + + files = set("pybind11/{}".format(n) for n in all_files) + files |= sdist_files + files |= set("pybind11{}".format(n) for n in local_sdist_files) + files.add("pybind11.egg-info/entry_points.txt") + files.add("pybind11.egg-info/requires.txt") + assert simpler == files + + with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: + contents = ( + string.Template(f.read().decode()) + .substitute(version=version, extra_cmd="") + .encode() + ) + assert setup_py == contents + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + contents = f.read() + assert pyproject_toml == contents + + +def test_build_global_dist(monkeypatch, tmpdir): + + monkeypatch.chdir(MAIN_DIR) + monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") + + out = subprocess.check_output( + [ + sys.executable, + "setup.py", + "sdist", + "--formats=tar", + "--dist-dir", + str(tmpdir), + ] + ) + if hasattr(out, "decode"): + out = out.decode() + + (sdist,) = tmpdir.visit("*.tar") + + with tarfile.open(str(sdist)) as tar: + start = tar.getnames()[0] + "/" + version = start[16:-1] + simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "setup.py")) + ) as f: + setup_py = f.read() + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pyproject.toml")) + ) as f: + pyproject_toml = f.read() + + files = set("pybind11/{}".format(n) for n in all_files) + files |= sdist_files + files |= set("pybind11_global{}".format(n) for n in local_sdist_files) + assert simpler == files + + with open(os.path.join(MAIN_DIR, "tools", "setup_global.py.in"), "rb") as f: + contents = ( + string.Template(f.read().decode()) + .substitute(version=version, extra_cmd="") + .encode() + ) + assert setup_py == contents + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + contents = f.read() + assert pyproject_toml == contents + + +def tests_build_wheel(monkeypatch, tmpdir): + monkeypatch.chdir(MAIN_DIR) + + subprocess.check_output( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + ) + + (wheel,) = tmpdir.visit("*.whl") + + files = set("pybind11/{}".format(n) for n in all_files) + files |= { + "dist-info/LICENSE", + "dist-info/METADATA", + "dist-info/RECORD", + "dist-info/WHEEL", + "dist-info/entry_points.txt", + "dist-info/top_level.txt", + } + + with zipfile.ZipFile(str(wheel)) as z: + names = z.namelist() + + trimmed = set(n for n in names if "dist-info" not in n) + trimmed |= set( + "dist-info/{}".format(n.split("/", 1)[-1]) for n in names if "dist-info" in n + ) + assert files == trimmed + + +def tests_build_global_wheel(monkeypatch, tmpdir): + monkeypatch.chdir(MAIN_DIR) + monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") + + subprocess.check_output( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + ) + + (wheel,) = tmpdir.visit("*.whl") + + files = set("data/data/{}".format(n) for n in src_files) + files |= set("data/headers/{}".format(n[8:]) for n in headers) + files |= { + "dist-info/LICENSE", + "dist-info/METADATA", + "dist-info/WHEEL", + "dist-info/top_level.txt", + "dist-info/RECORD", + } + + with zipfile.ZipFile(str(wheel)) as z: + names = z.namelist() + + beginning = names[0].split("/", 1)[0].rsplit(".", 1)[0] + trimmed = set(n[len(beginning) + 1 :] for n in names) + + assert files == trimmed diff --git a/tests/extra_setuptools/pytest.ini b/tests/extra_setuptools/pytest.ini new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py new file mode 100644 index 0000000000..de0b516a9f --- /dev/null +++ b/tests/extra_setuptools/test_setuphelper.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +import os +import sys +import subprocess +from textwrap import dedent + +import pytest + +DIR = os.path.abspath(os.path.dirname(__file__)) +MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) + + +@pytest.mark.parametrize("std", [11, 0]) +def test_simple_setup_py(monkeypatch, tmpdir, std): + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + u"""\ + import sys + sys.path.append({MAIN_DIR!r}) + + from setuptools import setup, Extension + from pybind11.setup_helpers import build_ext, Pybind11Extension + + std = {std} + + ext_modules = [ + Pybind11Extension( + "simple_setup", + sorted(["main.cpp"]), + cxx_std=std, + ), + ] + + cmdclass = dict() + if std == 0: + cmdclass["build_ext"] = build_ext + + + setup( + name="simple_setup_package", + cmdclass=cmdclass, + ext_modules=ext_modules, + ) + """ + ).format(MAIN_DIR=MAIN_DIR, std=std), + encoding="ascii", + ) + + (tmpdir / "main.cpp").write_text( + dedent( + u"""\ + #include + + int f(int x) { + return x * 3; + } + PYBIND11_MODULE(simple_setup, m) { + m.def("f", &f); + } + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "--inplace"], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + # Debug helper printout, normally hidden + for item in tmpdir.listdir(): + print(item.basename) + + assert ( + len([f for f in tmpdir.listdir() if f.basename.startswith("simple_setup")]) == 1 + ) + assert len(list(tmpdir.listdir())) == 4 # two files + output + build_dir + + (tmpdir / "test.py").write_text( + dedent( + u"""\ + import simple_setup + assert simple_setup.f(3) == 9 + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) diff --git a/tests/pytest.ini b/tests/pytest.ini index 6d758ea6ac..c47cbe9c1e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,6 +1,6 @@ [pytest] minversion = 3.1 -norecursedirs = test_cmake_build test_embed +norecursedirs = test_* extra_* xfail_strict = True addopts = # show summary of skipped tests diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 96e958e646..26a1e04892 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -300,7 +300,7 @@ _pybind11_generate_lto(pybind11::thin_lto TRUE) # ---------------------- pybind11_strip ----------------------------- function(pybind11_strip target_name) - # Strip unnecessary sections of the binary on Linux/Mac OS + # Strip unnecessary sections of the binary on Linux/macOS if(CMAKE_STRIP) if(APPLE) set(x_opt -x) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 812ec094aa..27eb4d9205 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -197,7 +197,7 @@ function(pybind11_add_module target_name) endif() if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) - # Strip unnecessary sections of the binary on Linux/Mac OS + # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/tools/pyproject.toml b/tools/pyproject.toml new file mode 100644 index 0000000000..9787c3bdf0 --- /dev/null +++ b/tools/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in new file mode 100644 index 0000000000..3325cd0ead --- /dev/null +++ b/tools/setup_global.py.in @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Setup script for pybind11-global (in the sdist or in tools/setup_global.py in the repository) +# This package is targeted for easy use from CMake. + +import contextlib +import glob +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# Setuptools has to be before distutils +from setuptools import setup + +from distutils.command.install_headers import install_headers + +class InstallHeadersNested(install_headers): + def run(self): + headers = self.distribution.headers or [] + for header in headers: + # Remove pybind11/include/ + short_header = header.split("/", 2)[-1] + + dst = os.path.join(self.install_dir, os.path.dirname(short_header)) + self.mkpath(dst) + (out, _) = self.copy_file(header, dst) + self.outfiles.append(out) + + +main_headers = glob.glob("pybind11/include/pybind11/*.h") +detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") +cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") +headers = main_headers + detail_headers + +cmdclass = {"install_headers": InstallHeadersNested} +$extra_cmd + +setup( + name="pybind11_global", + version="$version", + packages=[], + headers=headers, + data_files=[ + ("share/cmake/pybind11", cmake_files), + ("include/pybind11", main_headers), + ("include/pybind11/detail", detail_headers), + ], + cmdclass=cmdclass, +) diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in new file mode 100644 index 0000000000..c859c1f755 --- /dev/null +++ b/tools/setup_main.py.in @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Setup script (in the sdist or in tools/setup_main.py in the repository) + +from setuptools import setup + +cmdclass = {} +$extra_cmd + +setup( + name="pybind11", + version="$version", + download_url='https://github.com/pybind/pybind11/tarball/v$version', + packages=[ + "pybind11", + "pybind11.include.pybind11", + "pybind11.include.pybind11.detail", + "pybind11.share.cmake.pybind11", + ], + package_data={ + "pybind11.include.pybind11": ["*.h"], + "pybind11.include.pybind11.detail": ["*.h"], + "pybind11.share.cmake.pybind11": ["*.cmake"], + }, + extras_require={ + "global": ["pybind11_global==$version"] + }, + entry_points={ + "console_scripts": [ + "pybind11-config = pybind11.__main__:main", + ] + }, + cmdclass=cmdclass +)