From e551c52cef3e81f752bc56d951f38460430c349c Mon Sep 17 00:00:00 2001 From: Henry Fredrick Schreiner Date: Sun, 23 Aug 2020 15:37:10 -0400 Subject: [PATCH 01/28] feat: setup.py redesign and helpers --- .github/workflows/ci.yml | 24 +- .github/workflows/format.yml | 2 + .gitignore | 2 + .pre-commit-config.yaml | 13 ++ MANIFEST.in | 7 +- docs/compiling.rst | 45 ++++ pybind11/__init__.py | 19 +- pybind11/__main__.py | 21 +- pybind11/_version.py | 5 +- pybind11/commands.py | 21 ++ pybind11/setup_helpers.py | 165 ++++++++++++++ pyproject.toml | 3 + setup.cfg | 70 +++++- setup.py | 290 ++++++++++++++---------- tests/pytest.ini | 2 +- tests/test_python_package/pytest.ini | 0 tests/test_python_package/test_sdist.py | 26 +++ 17 files changed, 565 insertions(+), 150 deletions(-) create mode 100644 pybind11/commands.py create mode 100644 pybind11/setup_helpers.py create mode 100644 pyproject.toml create mode 100644 tests/test_python_package/pytest.ini create mode 100644 tests/test_python_package/test_sdist.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 530d8caf38..b2fdd0d62e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -443,19 +443,17 @@ jobs: doxygen: name: "Documentation build test" runs-on: ubuntu-latest - container: alpine:3.12 steps: - uses: actions/checkout@v2 - - name: Install system requirements - run: apk add doxygen python3-dev + - uses: actions/setup-python@v2 - - name: Ensure pip - run: python3 -m ensurepip + - name: Install Doxygen + run: sudo apt install -y doxygen - name: Install docs & setup requirements - run: python3 -m pip install -r docs/requirements.txt pytest setuptools + run: python3 -m pip install -r docs/requirements.txt - name: Build docs run: python3 -m sphinx -W -b html docs docs/.build @@ -463,8 +461,16 @@ jobs: - name: Make SDist run: python3 setup.py sdist + - run: git status --ignored + + - name: Check local include dir + run: > + ls pybind11; + python3 -c "import pybind11, pathlib; assert (a := pybind11.get_include()) == (b := str(pathlib.Path('include').resolve())), f'{a} != {b}'" + - name: Compare Dists (headers only) + working-directory: include run: | - python3 -m pip install --user -U ./dist/* - installed=$(python3 -c "import pybind11; print(pybind11.get_include(True) + '/pybind11')") - diff -rq $installed ./include/pybind11 + python3 -m pip install --user -U ../dist/* + installed=$(python3 -c "import pybind11; print(pybind11.get_include() + '/pybind11')") + diff -rq $installed ./pybind11 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 191219326d..49613ba5f5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -17,6 +17,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.0 + with: + extra_args: --hook-stage manual clang-tidy: name: Clang-Tidy diff --git a/.gitignore b/.gitignore index 5613b367d2..3f36b89e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ pybind11Config*.cmake pybind11Targets.cmake /*env* /.vscode +/pybind11/include/* +/pybind11/share/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6863f4c495..ce150b188f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,12 @@ repos: - id: trailing-whitespace - id: fix-encoding-pragma +- repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black + files: ^(setup.py|pybind11) + - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.9 hooks: @@ -34,6 +40,13 @@ repos: types: [file] files: (\.cmake|CMakeLists.txt)(.in)?$ +- repo: https://github.com/mgedmin/check-manifest + rev: "0.42" + hooks: + - id: check-manifest + stages: [manual] + additional_dependencies: [cmake, ninja] + - repo: local hooks: - id: disallow-caps diff --git a/MANIFEST.in b/MANIFEST.in index 6fe84ced8d..16701166f8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,7 @@ recursive-include include/pybind11 *.h -include LICENSE README.md .github/CONTRIBUTING.md +recursive-include pybind11 *.h +recursive-include pybind11 *.cmake +recursive-include pybind11 *.py +recursive-include tools *.cmake +recursive-include tools *.in +include CMakeLists.txt LICENSE README.md .github/CONTRIBUTING.md diff --git a/docs/compiling.rst b/docs/compiling.rst index ca4dc756e6..b7e8b2d02a 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -13,6 +13,51 @@ the [python_example]_ repository. .. [python_example] https://github.com/pybind/python_example +A helper file is provided with pybind11 that can simplify usage with setuptools +if you have pybind11 installed as a Python package; the file is also standalone, +if you want to copy it to your package. If use use PEP518's ``pyproject.toml`` +file: + +.. code-block:: toml + + [build-system] + requires = ["setuptools", "wheel", "pybind11==2.6.0"] + build-backend = "setuptools.build_meta" + +you can ensure that pybind11 is available during the building of your project +(pip 10+ required). + +An example of a ``setup.py`` using pybind11's helpers: + +.. code-block:: python + + from setuptools import setup, Extension + from pybind11.setup_helpers import BuildExt + + ext_modules = [ + Extension( + "python_example", + sorted(["src/main.cpp"]), + language="c++", + ), + ] + + setup( + ..., + cmdclass={"build_ext": BuildExt}, + ext_modules=ext_modules + ) + + +If you copy ``setup_helpers.py`` into your local project to try to support the +classic build procedure, then you will need to use the deprecated +``setup_requires=["pybind11>=2.6.0"]`` keyword argument to setup; +``setup_helpers`` tries to support this as well. + +.. versionchanged:: 2.6 + + Added support file. + Building with cppimport ======================== diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 5b2f83d5cd..ad65420893 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -from ._version import version_info, __version__ # noqa: F401 imported but unused +from ._version import version_info, __version__ +from .commands import get_include, get_cmake_dir -def get_include(user=False): - import os - d = os.path.dirname(__file__) - if os.path.exists(os.path.join(d, "include")): - # Package is installed - return os.path.join(d, "include") - else: - # Package is from a source directory - return os.path.join(os.path.dirname(d), "include") + +__all__ = ( + "version_info", + "__version__", + "get_include", + "get_cmake_dir", +) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 5e393cc8f1..4e5128aaf9 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -9,9 +9,11 @@ def print_includes(): - dirs = [sysconfig.get_path('include'), - sysconfig.get_path('platinclude'), - get_include()] + dirs = [ + sysconfig.get_path("include"), + sysconfig.get_path("platinclude"), + get_include(), + ] # Make unique but preserve order unique_dirs = [] @@ -19,13 +21,16 @@ def print_includes(): if d not in unique_dirs: unique_dirs.append(d) - print(' '.join('-I' + d for d in unique_dirs)) + print(" ".join("-I" + d for d in unique_dirs)) def main(): - parser = argparse.ArgumentParser(prog='python -m pybind11') - parser.add_argument('--includes', action='store_true', - help='Include flags for both pybind11 and Python headers.') + parser = argparse.ArgumentParser(prog="python -m pybind11") + parser.add_argument( + "--includes", + action="store_true", + help="Include flags for both pybind11 and Python headers.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() @@ -33,5 +38,5 @@ def main(): print_includes() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/pybind11/_version.py b/pybind11/_version.py index 1f2f254ce5..b548f96dc8 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -1,3 +1,4 @@ # -*- coding: utf-8 -*- -version_info = (2, 5, 'dev1') -__version__ = '.'.join(map(str, version_info)) + +version_info = (2, 6, 0, "dev1") +__version__ = ".".join(map(str, version_info)) diff --git a/pybind11/commands.py b/pybind11/commands.py new file mode 100644 index 0000000000..ee1ef6dc98 --- /dev/null +++ b/pybind11/commands.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +import os + + +DIR = os.path.abspath(os.path.dirname(__file__)) + + +def get_include(user=False): + installed_path = os.path.join(DIR, "include") + source_path = os.path.join(os.path.dirname(DIR), "include") + return installed_path if os.path.exists(installed_path) else source_path + + +def get_cmake_dir(): + cmake_installed_path = os.path.join(DIR, "share", "cmake", "pybind11") + if os.path.exists(cmake_installed_path): + return cmake_installed_path + else: + raise ImportError( + "pybind11 not installed, installation required to access the CMake files" + ) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py new file mode 100644 index 0000000000..a07069afda --- /dev/null +++ b/pybind11/setup_helpers.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- + +""" +This module provides a way to check to see if flag is available, +has_flag (built-in to distutils.CCompiler in Python 3.6+), and +a cpp_flag function, which will compute the highest available +flag (or if a flag is supported). + +LICENSE: + +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import os +import sys +import tempfile + +import distutils.errors +from distutils.command.build_ext import build_ext + + +# It is recommended to use PEP 518 builds if using this module. However, this +# file explicitly supports being copied into a user's project directory +# standalone, and pulling pybind11 with the deprecated setup_requires feature. + + +class DelayedPybindInclude(object): + """ + Helper class to determine the pybind11 include path The purpose of this + class is to postpone importing pybind11 until it is actually installed, so + that the ``get_include()`` method can be invoked if pybind11 is loaded via + setup_requires. + """ + + def __str__(self): + import pybind11 + + return pybind11.get_include() + + +# cf http://bugs.python.org/issue26689 +def has_flag(compiler, flagname): + """ + Return a boolean indicating whether a flag name is supported on the + specified compiler. + """ + + with tempfile.NamedTemporaryFile("w", suffix=".cpp", delete=False) as f: + f.write("int main (int argc, char **argv) { return 0; }") + fname = f.name + try: + compiler.compile([fname], extra_postargs=[flagname]) + # distutils/ccompiler.py, unixcompiler.py, etc. + # compiler.compile generates output file at + # os.path.join(output_dir, fname[1:]) - which drops leading /, + # so we use output_dir == '/' to put it back on + # TODO: not sure what Windows does so leave it alone + outdir = os.path.sep if sys.platform != "win32" else None + compiler.compile([fname], extra_postargs=[flagname], output_dir=outdir) + except distutils.errors.CompileError: + return False + finally: + try: + os.remove(fname) + except OSError: + pass + return True + + +def cpp_flag(compiler, value=None): + """ + Return the ``-std=c++[11/14/17]`` compiler flag. + The newer version is preferred over c++11 (when it is available). + """ + + flags = ["-std=c++17", "-std=c++14", "-std=c++11"] + + if value is not None: + mapping = {17: 0, 14: 1, 11: 2} + flags = [flags[mapping[value]]] + + for flag in flags: + if has_flag(compiler, flag): + return flag + + raise RuntimeError("Unsupported compiler -- at least C++11 support is needed!") + + +c_opts = { + "msvc": ["/EHsc"], + "unix": [], +} + +l_opts = { + "msvc": [], + "unix": [], +} + +if sys.platform == "darwin": + darwin_opts = ["-stdlib=libc++", "-mmacosx-version-min=10.9"] + c_opts["unix"] += darwin_opts + l_opts["unix"] += darwin_opts + + +class BuildExt(build_ext): + """ + Customized build_ext that can be further customized by users. + + Most use cases can be addressed by adding items to the extensions. + However, if you need to customize, try: + + class BuildExt(pybind11.setup_utils.BuildExt): + def build_extensions(self): + # Do something here, like add things to extensions + + super(BuildExt, self).build_extensions() + + One simple customization point is provided: ``self.cxx_std`` lets + you set a C++ standard (None is the default search). + """ + + def __init__(self, *args, **kwargs): + super(BuildExt, self).__init__(*args, **kwargs) + self.cxx_std = None + + def build_extensions(self): + ct = self.compiler.compiler_type + comp_opts = c_opts.get(ct, []) + link_opts = l_opts.get(ct, []) + if ct == "unix": + comp_opts.append(cpp_flag(self.compiler, self.cxx_std)) + if has_flag(self.compiler, "-fvisibility=hidden"): + comp_opts.append("-fvisibility=hidden") + + for ext in self.extensions: + ext.extra_compile_args += comp_opts + ext.extra_link_args += link_opts + ext.include_dirs += [DelayedPybindInclude()] + + super(BuildExt, self).build_extensions() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..3bab1c1a28 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel", "cmake==3.18.0", "ninja"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 002f38d10e..0860f73c14 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,72 @@ -[bdist_wheel] +[metadata] +name = pybind11 +long_description = file: README.md +long_description_content_type = text/markdown +version = attr: pybind11.__version__ +description = Seamless operability between C++11 and Python +author = Wenzel Jakob +author_email = "wenzel.jakob@epfl.ch" +url = "https://github.com/pybind/pybind11" +license = BSD + +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + Programming Language :: C++ + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + License :: OSI Approved :: BSD License + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Python :: Implementation :: CPython + Programming Language :: C++ + Topic :: Software Development :: Libraries :: Python Modules + +keywords = + C++11 + Python bindings + +[options] +python_requires = >=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4 +find_package_data = True +zip_safe = False +packages = + pybind11 + pybind11.include.pybind11 + pybind11.include.pybind11.detail + pybind11.share.cmake.pybind11 + +[options.package_data] +pybind11.include.pybind11 = *.h +pybind11.include.pybind11.detail = *.h +pybind11.share.cmake.pybind11 = *.cmake + + +[sbdist_wheel] universal=1 +[check-manifest] +ignore = + tests/** + docs/** + tools/check-style.sh + tools/clang + tools/libsize.py + tools/mkdoc.py + .appveyor.yml + .cmake-format.yaml + .gitmodules + .pre-commit-config.yaml + .readthedocs.yml + .clang-tidy + pybind11/include/** + pybind11/share/** + [flake8] max-line-length = 99 show_source = True @@ -10,3 +76,5 @@ ignore = E201, E241, W504, # camelcase 'cPickle' imported as lowercase 'pickle' N813 + # Black conflict + W503 diff --git a/setup.py b/setup.py index 577a6b6c37..76cb80c066 100644 --- a/setup.py +++ b/setup.py @@ -3,128 +3,182 @@ # Setup script for PyPI; use CMakeFile.txt to build extension modules -from setuptools import setup -from distutils.command.install_headers import install_headers -from distutils.command.build_py import build_py -from pybind11 import __version__ +import contextlib +import glob import os +import re +import shutil +import subprocess +import sys +import tempfile +from distutils.command.install_headers import install_headers + +from setuptools import setup + +# For now, there are three parts to this package. Besides the "normal" module: +# PYBIND11_USE_HEADERS will include the python-headers files. +# PYBIND11_USE_SYSTEM will include the sys.prefix files (CMake and headers). +# The final version will likely only include the normal module or come in +# different versions. + +use_headers = os.environ.get("PYBIND11_USE_HEADERS", False) +use_system = os.environ.get("PYBIND11_USE_SYSTEM", False) + +setup_opts = dict() + +# In a PEP 518 build, this will be in its own environment, so it will not +# create extra files in the source + +DIR = os.path.abspath(os.path.dirname(__file__)) + +prexist_include = os.path.exists("pybind11/include") +prexist_share = os.path.exists("pybind11/share") + + +@contextlib.contextmanager +def monkey_patch_file(input_file): + "Allow a file to be temporarily modified" + + with open(os.path.join(DIR, input_file), "r") as f: + contents = f.read() + try: + yield contents + finally: + with open(os.path.join(DIR, input_file), "w") as f: + f.write(contents) + + +@contextlib.contextmanager +def TemporaryDirectory(): # noqa: N802 + "Prepare a temporary directory, cleanup when done" + try: + tmpdir = tempfile.mkdtemp() + yield tmpdir + finally: + shutil.rmtree(tmpdir) -package_data = [ - 'include/pybind11/detail/class.h', - 'include/pybind11/detail/common.h', - 'include/pybind11/detail/descr.h', - 'include/pybind11/detail/init.h', - 'include/pybind11/detail/internals.h', - 'include/pybind11/detail/typeid.h', - 'include/pybind11/attr.h', - 'include/pybind11/buffer_info.h', - 'include/pybind11/cast.h', - 'include/pybind11/chrono.h', - 'include/pybind11/common.h', - 'include/pybind11/complex.h', - 'include/pybind11/eigen.h', - 'include/pybind11/embed.h', - 'include/pybind11/eval.h', - 'include/pybind11/functional.h', - 'include/pybind11/iostream.h', - 'include/pybind11/numpy.h', - 'include/pybind11/operators.h', - 'include/pybind11/options.h', - 'include/pybind11/pybind11.h', - 'include/pybind11/pytypes.h', - 'include/pybind11/stl.h', - 'include/pybind11/stl_bind.h', -] - -# Prevent installation of pybind11 headers by setting -# PYBIND11_USE_CMAKE. -if os.environ.get('PYBIND11_USE_CMAKE'): - headers = [] -else: - headers = package_data - - -class InstallHeaders(install_headers): - """Use custom header installer because the default one flattens subdirectories""" - def run(self): - if not self.distribution.headers: - return - for header in self.distribution.headers: - subdir = os.path.dirname(os.path.relpath(header, 'include/pybind11')) - install_dir = os.path.join(self.install_dir, subdir) - self.mkpath(install_dir) +@contextlib.contextmanager +def remove_output(*sources): + try: + yield + finally: + for src in sources: + shutil.rmtree(src) - (out, _) = self.copy_file(header, install_dir) + +def check_compare(input_set, *patterns): + "Just a quick way to make sure all files are present" + disk_files = set() + for pattern in patterns: + disk_files |= set(glob.glob(pattern, recursive=True)) + + assert input_set == disk_files, "{} setup.py only, {} on disk only".format( + input_set - disk_files, disk_files - input_set + ) + + +class InstallHeadersNested(install_headers): + def run(self): + headers = self.distribution.headers or [] + for header in headers: + # Remove include/*/ + short_header = header.split("/", 2)[-1] + + dst = os.path.join(self.install_dir, os.path.dirname(short_header)) + self.mkpath(dst) + (out, _) = self.copy_file(header, dst) self.outfiles.append(out) -# Install the headers inside the package as well -class BuildPy(build_py): - def build_package_data(self): - build_py.build_package_data(self) - for header in package_data: - target = os.path.join(self.build_lib, 'pybind11', header) - self.mkpath(os.path.dirname(target)) - self.copy_file(header, target, preserve_mode=False) - - def get_outputs(self, include_bytecode=1): - outputs = build_py.get_outputs(self, include_bytecode=include_bytecode) - for header in package_data: - target = os.path.join(self.build_lib, 'pybind11', header) - outputs.append(target) - return outputs - - -setup( - name='pybind11', - version=__version__, - description='Seamless operability between C++11 and Python', - author='Wenzel Jakob', - author_email='wenzel.jakob@epfl.ch', - url='https://github.com/pybind/pybind11', - download_url='https://github.com/pybind/pybind11/tarball/v' + __version__, - packages=['pybind11'], - license='BSD', - headers=headers, - zip_safe=False, - cmdclass=dict(install_headers=InstallHeaders, build_py=BuildPy), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Utilities', - 'Programming Language :: C++', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'License :: OSI Approved :: BSD License' - ], - keywords='C++11, Python bindings', - long_description="""pybind11 is a lightweight header-only library that -exposes C++ types in Python and vice versa, mainly to create Python bindings of -existing C++ code. Its goals and syntax are similar to the excellent -Boost.Python by David Abrahams: to minimize boilerplate code in traditional -extension modules by inferring type information using compile-time -introspection. - -The main issue with Boost.Python-and the reason for creating such a similar -project-is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. - -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations.""") +main_headers = { + "include/pybind11/attr.h", + "include/pybind11/buffer_info.h", + "include/pybind11/cast.h", + "include/pybind11/chrono.h", + "include/pybind11/common.h", + "include/pybind11/complex.h", + "include/pybind11/eigen.h", + "include/pybind11/embed.h", + "include/pybind11/eval.h", + "include/pybind11/functional.h", + "include/pybind11/iostream.h", + "include/pybind11/numpy.h", + "include/pybind11/operators.h", + "include/pybind11/options.h", + "include/pybind11/pybind11.h", + "include/pybind11/pytypes.h", + "include/pybind11/stl.h", + "include/pybind11/stl_bind.h", +} + +detail_headers = { + "include/pybind11/detail/class.h", + "include/pybind11/detail/common.h", + "include/pybind11/detail/descr.h", + "include/pybind11/detail/init.h", + "include/pybind11/detail/internals.h", + "include/pybind11/detail/typeid.h", +} + +headers = main_headers | detail_headers +check_compare(headers, "include/**/*.h") + +if use_headers: + setup_opts["headers"] = headers + setup_opts["cmdclass"] = {"install_headers": InstallHeadersNested} + +cmake_files = { + "pybind11/share/cmake/pybind11/FindPythonLibsNew.cmake", + "pybind11/share/cmake/pybind11/pybind11Common.cmake", + "pybind11/share/cmake/pybind11/pybind11Config.cmake", + "pybind11/share/cmake/pybind11/pybind11ConfigVersion.cmake", + "pybind11/share/cmake/pybind11/pybind11NewTools.cmake", + "pybind11/share/cmake/pybind11/pybind11Targets.cmake", + "pybind11/share/cmake/pybind11/pybind11Tools.cmake", +} + + +package_headers = set("pybind11/{}".format(h) for h in headers) +package_files = package_headers | cmake_files + +# Generate the files if they are not generated (will be present in tarball) +GENERATED = ( + [] + if all(os.path.exists(h) for h in package_files) + else ["pybind11/include", "pybind11/share"] +) +with remove_output(*GENERATED): + # Generate the files if they are not present. + if GENERATED: + with TemporaryDirectory() as tmpdir: + cmd = ["cmake", "-S", ".", "-B", tmpdir] + [ + "-DCMAKE_INSTALL_PREFIX=pybind11", + "-DBUILD_TESTING=OFF", + "-DPYBIND11_NOPYTHON=ON", + ] + cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call(cmd, **cmake_opts) + subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) + + # Make sure all files are present + check_compare(package_files, "pybind11/include/**/*.h", "pybind11/share/**/*.cmake") + + if use_system: + setup_opts["data_files"] = [ + ("share/cmake", cmake_files), + ("include/pybind11", main_headers), + ("include/pybind11/detail", detail_headers), + ] + + # Remove the cmake / ninja requirements as now all files are guaranteed to exist + if GENERATED: + REQUIRES = re.compile(r"requires\s*=.+?\]", re.DOTALL | re.MULTILINE) + with monkey_patch_file("pyproject.toml") as txt: + with open("pyproject.toml", "w") as f: + new_txt = REQUIRES.sub('requires = ["setuptools", "wheel"]', txt) + f.write(new_txt) + + setup(**setup_opts) + else: + setup(**setup_opts) diff --git a/tests/pytest.ini b/tests/pytest.ini index 6d758ea6ac..d0646a0915 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,6 +1,6 @@ [pytest] minversion = 3.1 -norecursedirs = test_cmake_build test_embed +norecursedirs = test_cmake_build test_embed test_python_package xfail_strict = True addopts = # show summary of skipped tests diff --git a/tests/test_python_package/pytest.ini b/tests/test_python_package/pytest.ini new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_python_package/test_sdist.py b/tests/test_python_package/test_sdist.py new file mode 100644 index 0000000000..8c83fdb95a --- /dev/null +++ b/tests/test_python_package/test_sdist.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +import subprocess +import sys +import os + +# These tests must be run explicitly +# They require CMake 3.15+ (--install) + +DIR = os.path.abspath(os.path.dirname(__file__)) + + +def test_build_sdist(monkeypatch): + + main_dir = os.path.dirname(os.path.dirname(DIR)) + monkeypatch.chdir(main_dir) + + out = subprocess.check_output([sys.executable, "setup.py", "sdist"]) + if hasattr(out, 'decode'): + out = out.decode() + + print(out) + + assert 'pybind11/share/cmake/pybind11' in out + assert 'pybind11/include/pybind11' in out + + assert out.count("copying") == 82 From 1b880db7012020117e6f68429d4bffa52dcf50ad Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 24 Aug 2020 23:56:44 -0400 Subject: [PATCH 02/28] refactor: simpler design with two outputs --- .github/workflows/ci.yml | 4 + .github/workflows/configure.yml | 58 +++++ .pre-commit-config.yaml | 6 +- MANIFEST.in | 9 +- docs/basics.rst | 6 +- docs/compiling.rst | 4 +- pybind11/setup_helpers.py | 60 +++-- setup.cfg | 25 +- setup.py | 166 +++--------- .../pytest.ini | 0 tests/extra_python_package/test_files.py | 245 ++++++++++++++++++ tests/extra_setuptools/pytest.ini | 0 tests/extra_setuptools/test_setuphelper.py | 78 ++++++ tests/pytest.ini | 2 +- tests/test_python_package/test_sdist.py | 26 -- tests/test_sequences_and_iterators.cpp | 1 + tools/pybind11Common.cmake | 2 +- tools/pybind11NewTools.cmake | 2 +- tools/pyproject.toml | 3 + tools/setup_alt.py | 49 ++++ tools/setup_main.py | 22 ++ 21 files changed, 546 insertions(+), 222 deletions(-) rename tests/{test_python_package => extra_python_package}/pytest.ini (100%) create mode 100644 tests/extra_python_package/test_files.py create mode 100644 tests/extra_setuptools/pytest.ini create mode 100644 tests/extra_setuptools/test_setuphelper.py delete mode 100644 tests/test_python_package/test_sdist.py create mode 100644 tools/pyproject.toml create mode 100644 tools/setup_alt.py create mode 100644 tools/setup_main.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2fdd0d62e..bef228439f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,10 @@ jobs: - name: Interface test run: cmake --build build2 --target test_cmake_build + - name: Setuptools helpers test + run: pytest tests/extra_setuptools + + clang: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index d472f4b191..0ce235c52d 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -76,3 +76,61 @@ jobs: working-directory: build dir if: github.event_name == 'workflow_dispatch' run: cmake --build . --config Release --target check + + test-packaging: + name: 🐍 2.7 • 📦 tests • windows-latest + runs-on: windows-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup 🐍 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + + - name: Prepare env + run: python -m pip install -r tests/requirements.txt --prefer-binary + + - name: Python Packaging tests + run: pytest tests/extra_python_package/ + + + packaging: + name: 🐍 3.8 • 📦 & 📦 tests • ubuntu-latest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup 🐍 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Prepare env + run: python -m pip install -r tests/requirements.txt wheel twine --prefer-binary + + - name: Python Packaging tests + run: pytest tests/extra_python_package/ + + - name: Build SDist + run: | + python setup.py sdist + PYBIND11_ALT_SDIST=1 python setup.py sdist + + - uses: actions/upload-artifact@v2 + with: + path: dist/* + + - name: Build wheel + run: | + python -m pip wheel . -w wheels + PYBIND11_ALT_SDIST=1 python -m pip wheel . -w wheels + + - uses: actions/upload-artifact@v2 + with: + path: wheels/pybind11*.whl + + - name: Check metadata + run: twine check dist/* wheels/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce150b188f..3ebd310fe0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.2.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -15,10 +15,10 @@ repos: - id: fix-encoding-pragma - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black - files: ^(setup.py|pybind11) + files: ^(setup.py|pybind11|tests/extra) - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.9 diff --git a/MANIFEST.in b/MANIFEST.in index 16701166f8..9336b60302 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,4 @@ -recursive-include include/pybind11 *.h -recursive-include pybind11 *.h -recursive-include pybind11 *.cmake +recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py -recursive-include tools *.cmake -recursive-include tools *.in -include CMakeLists.txt LICENSE README.md .github/CONTRIBUTING.md +include pybind11/share/cmake/pybind11/*.cmake +include LICENSE README.md pyproject.toml setup.py setup.cfg diff --git a/docs/basics.rst b/docs/basics.rst index 6bb5f98222..71440c9c66 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -11,11 +11,11 @@ included set of test cases. Compiling the test cases ======================== -Linux/MacOS +Linux/macOS ----------- On Linux you'll need to install the **python-dev** or **python3-dev** packages as -well as **cmake**. On Mac OS, the included python version works out of the box, +well as **cmake**. On macOS, the included python version works out of the box, but **cmake** must still be installed. After installing the prerequisites, run @@ -138,7 +138,7 @@ On Linux, the above example can be compiled using the following command: $ c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` -For more details on the required compiler flags on Linux and MacOS, see +For more details on the required compiler flags on Linux and macOS, see :ref:`building_manually`. For complete cross-platform compilation instructions, refer to the :ref:`compiling` page. diff --git a/docs/compiling.rst b/docs/compiling.rst index b7e8b2d02a..9617f50dd3 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -15,7 +15,7 @@ the [python_example]_ repository. A helper file is provided with pybind11 that can simplify usage with setuptools if you have pybind11 installed as a Python package; the file is also standalone, -if you want to copy it to your package. If use use PEP518's ``pyproject.toml`` +if you want to copy it to your package. If you use PEP518's ``pyproject.toml`` file: .. code-block:: toml @@ -412,7 +412,7 @@ Besides, the ``--extension-suffix`` option may or may not be available, dependin on the distribution; in the latter case, the module extension can be manually set to ``.so``. -On Mac OS: the build command is almost the same but it also requires passing +On macOS: the build command is almost the same but it also requires passing the ``-undefined dynamic_lookup`` flag so as to ignore missing symbols when building the module: diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index a07069afda..6a69c27bc1 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -36,9 +36,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ +import contextlib import os +import shutil import sys import tempfile +import threading import distutils.errors from distutils.command.build_ext import build_ext @@ -63,6 +66,26 @@ def __str__(self): return pybind11.get_include() +# Just in case someone clever tries to multithread +tmp_chdir_lock = threading.Lock() + + +@contextlib.contextmanager +def tmp_chdir(): + "Prepare and enter a temporary directory, cleanup when done" + # Threadsafe + + with tmp_chdir_lock: + olddir = os.getcwd() + try: + tmpdir = tempfile.mkdtemp() + os.chdir(tmpdir) + yield tmpdir + finally: + os.chdir(olddir) + shutil.rmtree(tmpdir) + + # cf http://bugs.python.org/issue26689 def has_flag(compiler, flagname): """ @@ -70,26 +93,16 @@ def has_flag(compiler, flagname): specified compiler. """ - with tempfile.NamedTemporaryFile("w", suffix=".cpp", delete=False) as f: - f.write("int main (int argc, char **argv) { return 0; }") - fname = f.name - try: - compiler.compile([fname], extra_postargs=[flagname]) - # distutils/ccompiler.py, unixcompiler.py, etc. - # compiler.compile generates output file at - # os.path.join(output_dir, fname[1:]) - which drops leading /, - # so we use output_dir == '/' to put it back on - # TODO: not sure what Windows does so leave it alone - outdir = os.path.sep if sys.platform != "win32" else None - compiler.compile([fname], extra_postargs=[flagname], output_dir=outdir) - except distutils.errors.CompileError: - return False - finally: + with tmp_chdir(): + fname = "flagcheck.cpp" + with open(fname, "w") as f: + f.write("int main (int argc, char **argv) { return 0; }") + try: - os.remove(fname) - except OSError: - pass - return True + compiler.compile([fname], extra_postargs=[flagname]) + return True + except distutils.errors.CompileError: + return False def cpp_flag(compiler, value=None): @@ -144,16 +157,12 @@ def build_extensions(self): you set a C++ standard (None is the default search). """ - def __init__(self, *args, **kwargs): - super(BuildExt, self).__init__(*args, **kwargs) - self.cxx_std = None - def build_extensions(self): ct = self.compiler.compiler_type comp_opts = c_opts.get(ct, []) link_opts = l_opts.get(ct, []) if ct == "unix": - comp_opts.append(cpp_flag(self.compiler, self.cxx_std)) + comp_opts.append(cpp_flag(self.compiler, getattr(self, "cxx_std", None))) if has_flag(self.compiler, "-fvisibility=hidden"): comp_opts.append("-fvisibility=hidden") @@ -162,4 +171,5 @@ def build_extensions(self): ext.extra_link_args += link_opts ext.include_dirs += [DelayedPybindInclude()] - super(BuildExt, self).build_extensions() + # Python 2 doesn't allow super here, since it's not a "class" + build_ext.build_extensions(self) diff --git a/setup.cfg b/setup.cfg index 0860f73c14..ee0b363735 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,4 @@ [metadata] -name = pybind11 long_description = file: README.md long_description_content_type = text/markdown version = attr: pybind11.__version__ @@ -33,31 +32,17 @@ keywords = [options] python_requires = >=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4 -find_package_data = True zip_safe = False -packages = - pybind11 - pybind11.include.pybind11 - pybind11.include.pybind11.detail - pybind11.share.cmake.pybind11 -[options.package_data] -pybind11.include.pybind11 = *.h -pybind11.include.pybind11.detail = *.h -pybind11.share.cmake.pybind11 = *.cmake - - -[sbdist_wheel] +[bdist_wheel] universal=1 [check-manifest] ignore = tests/** docs/** - tools/check-style.sh - tools/clang - tools/libsize.py - tools/mkdoc.py + tools/** + include/** .appveyor.yml .cmake-format.yaml .gitmodules @@ -66,6 +51,8 @@ ignore = .clang-tidy pybind11/include/** pybind11/share/** + CMakeLists.txt + [flake8] max-line-length = 99 @@ -77,4 +64,4 @@ ignore = # camelcase 'cPickle' imported as lowercase 'pickle' N813 # Black conflict - W503 + W503, E203 diff --git a/setup.py b/setup.py index 76cb80c066..db3c0c88f5 100644 --- a/setup.py +++ b/setup.py @@ -4,47 +4,41 @@ # Setup script for PyPI; use CMakeFile.txt to build extension modules import contextlib -import glob import os -import re import shutil import subprocess import sys import tempfile -from distutils.command.install_headers import install_headers -from setuptools import setup +# PYBIND11_ALT_SDIST will build a different sdist, with the python-headers +# files, and the sys.prefix files (CMake and headers). -# For now, there are three parts to this package. Besides the "normal" module: -# PYBIND11_USE_HEADERS will include the python-headers files. -# PYBIND11_USE_SYSTEM will include the sys.prefix files (CMake and headers). -# The final version will likely only include the normal module or come in -# different versions. - -use_headers = os.environ.get("PYBIND11_USE_HEADERS", False) -use_system = os.environ.get("PYBIND11_USE_SYSTEM", False) - -setup_opts = dict() +alt_sdist = os.environ.get("PYBIND11_ALT_SDIST", False) +setup_py = "tools/setup_alt.py" if alt_sdist else "tools/setup_main.py" +pyproject_toml = "tools/pyproject.toml" # In a PEP 518 build, this will be in its own environment, so it will not # create extra files in the source DIR = os.path.abspath(os.path.dirname(__file__)) -prexist_include = os.path.exists("pybind11/include") -prexist_share = os.path.exists("pybind11/share") - @contextlib.contextmanager -def monkey_patch_file(input_file): - "Allow a file to be temporarily modified" +def monkey_patch_file(input_file, replacement_file): + "Allow a file to be temporarily replaced" + inp_file = os.path.abspath(os.path.join(DIR, input_file)) + rep_file = os.path.abspath(os.path.join(DIR, replacement_file)) - with open(os.path.join(DIR, input_file), "r") as f: + with open(inp_file, "rb") as f: contents = f.read() + with open(rep_file, "rb") as f: + replacement = f.read() try: - yield contents + with open(inp_file, "wb") as f: + f.write(replacement) + yield finally: - with open(os.path.join(DIR, input_file), "w") as f: + with open(inp_file, "wb") as f: f.write(contents) @@ -67,118 +61,20 @@ def remove_output(*sources): shutil.rmtree(src) -def check_compare(input_set, *patterns): - "Just a quick way to make sure all files are present" - disk_files = set() - for pattern in patterns: - disk_files |= set(glob.glob(pattern, recursive=True)) - - assert input_set == disk_files, "{} setup.py only, {} on disk only".format( - input_set - disk_files, disk_files - input_set - ) - - -class InstallHeadersNested(install_headers): - def run(self): - headers = self.distribution.headers or [] - for header in headers: - # Remove include/*/ - short_header = header.split("/", 2)[-1] - - dst = os.path.join(self.install_dir, os.path.dirname(short_header)) - self.mkpath(dst) - (out, _) = self.copy_file(header, dst) - self.outfiles.append(out) - - -main_headers = { - "include/pybind11/attr.h", - "include/pybind11/buffer_info.h", - "include/pybind11/cast.h", - "include/pybind11/chrono.h", - "include/pybind11/common.h", - "include/pybind11/complex.h", - "include/pybind11/eigen.h", - "include/pybind11/embed.h", - "include/pybind11/eval.h", - "include/pybind11/functional.h", - "include/pybind11/iostream.h", - "include/pybind11/numpy.h", - "include/pybind11/operators.h", - "include/pybind11/options.h", - "include/pybind11/pybind11.h", - "include/pybind11/pytypes.h", - "include/pybind11/stl.h", - "include/pybind11/stl_bind.h", -} - -detail_headers = { - "include/pybind11/detail/class.h", - "include/pybind11/detail/common.h", - "include/pybind11/detail/descr.h", - "include/pybind11/detail/init.h", - "include/pybind11/detail/internals.h", - "include/pybind11/detail/typeid.h", -} - -headers = main_headers | detail_headers -check_compare(headers, "include/**/*.h") - -if use_headers: - setup_opts["headers"] = headers - setup_opts["cmdclass"] = {"install_headers": InstallHeadersNested} - -cmake_files = { - "pybind11/share/cmake/pybind11/FindPythonLibsNew.cmake", - "pybind11/share/cmake/pybind11/pybind11Common.cmake", - "pybind11/share/cmake/pybind11/pybind11Config.cmake", - "pybind11/share/cmake/pybind11/pybind11ConfigVersion.cmake", - "pybind11/share/cmake/pybind11/pybind11NewTools.cmake", - "pybind11/share/cmake/pybind11/pybind11Targets.cmake", - "pybind11/share/cmake/pybind11/pybind11Tools.cmake", -} - - -package_headers = set("pybind11/{}".format(h) for h in headers) -package_files = package_headers | cmake_files - -# Generate the files if they are not generated (will be present in tarball) -GENERATED = ( - [] - if all(os.path.exists(h) for h in package_files) - else ["pybind11/include", "pybind11/share"] -) -with remove_output(*GENERATED): +with remove_output("pybind11/include", "pybind11/share"): # Generate the files if they are not present. - if GENERATED: - with TemporaryDirectory() as tmpdir: - cmd = ["cmake", "-S", ".", "-B", tmpdir] + [ - "-DCMAKE_INSTALL_PREFIX=pybind11", - "-DBUILD_TESTING=OFF", - "-DPYBIND11_NOPYTHON=ON", - ] - cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) - subprocess.check_call(cmd, **cmake_opts) - subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) - - # Make sure all files are present - check_compare(package_files, "pybind11/include/**/*.h", "pybind11/share/**/*.cmake") - - if use_system: - setup_opts["data_files"] = [ - ("share/cmake", cmake_files), - ("include/pybind11", main_headers), - ("include/pybind11/detail", detail_headers), + with TemporaryDirectory() as tmpdir: + cmd = ["cmake", "-S", ".", "-B", tmpdir] + [ + "-DCMAKE_INSTALL_PREFIX=pybind11", + "-DBUILD_TESTING=OFF", + "-DPYBIND11_NOPYTHON=ON", ] - - # Remove the cmake / ninja requirements as now all files are guaranteed to exist - if GENERATED: - REQUIRES = re.compile(r"requires\s*=.+?\]", re.DOTALL | re.MULTILINE) - with monkey_patch_file("pyproject.toml") as txt: - with open("pyproject.toml", "w") as f: - new_txt = REQUIRES.sub('requires = ["setuptools", "wheel"]', txt) - f.write(new_txt) - - setup(**setup_opts) - else: - setup(**setup_opts) + cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call(cmd, **cmake_opts) + subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) + + with monkey_patch_file("pyproject.toml", pyproject_toml): + with monkey_patch_file("setup.py", setup_py): + with open(setup_py) as f: + code = compile(f.read(), setup_py, "exec") + exec(code, globals(), {}) diff --git a/tests/test_python_package/pytest.ini b/tests/extra_python_package/pytest.ini similarity index 100% rename from tests/test_python_package/pytest.ini rename to tests/extra_python_package/pytest.ini diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py new file mode 100644 index 0000000000..18c40a8182 --- /dev/null +++ b/tests/extra_python_package/test_files.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- +import subprocess +import sys +import os +import tarfile +import contextlib +import zipfile + +# These tests must be run explicitly +# They require CMake 3.15+ (--install) + +DIR = os.path.abspath(os.path.dirname(__file__)) +MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) + + +main_headers = { + "include/pybind11/attr.h", + "include/pybind11/buffer_info.h", + "include/pybind11/cast.h", + "include/pybind11/chrono.h", + "include/pybind11/common.h", + "include/pybind11/complex.h", + "include/pybind11/eigen.h", + "include/pybind11/embed.h", + "include/pybind11/eval.h", + "include/pybind11/functional.h", + "include/pybind11/iostream.h", + "include/pybind11/numpy.h", + "include/pybind11/operators.h", + "include/pybind11/options.h", + "include/pybind11/pybind11.h", + "include/pybind11/pytypes.h", + "include/pybind11/stl.h", + "include/pybind11/stl_bind.h", +} + +detail_headers = { + "include/pybind11/detail/class.h", + "include/pybind11/detail/common.h", + "include/pybind11/detail/descr.h", + "include/pybind11/detail/init.h", + "include/pybind11/detail/internals.h", + "include/pybind11/detail/typeid.h", +} + +cmake_files = { + "share/cmake/pybind11/FindPythonLibsNew.cmake", + "share/cmake/pybind11/pybind11Common.cmake", + "share/cmake/pybind11/pybind11Config.cmake", + "share/cmake/pybind11/pybind11ConfigVersion.cmake", + "share/cmake/pybind11/pybind11NewTools.cmake", + "share/cmake/pybind11/pybind11Targets.cmake", + "share/cmake/pybind11/pybind11Tools.cmake", +} + +py_files = { + "__init__.py", + "__main__.py", + "_version.py", + "commands.py", + "setup_helpers.py", +} + +headers = main_headers | detail_headers +src_files = headers | cmake_files +all_files = src_files | py_files + + +sdist_files = { + "pybind11", + "pybind11/include", + "pybind11/include/pybind11", + "pybind11/include/pybind11/detail", + "pybind11/share", + "pybind11/share/cmake", + "pybind11/share/cmake/pybind11", + "pyproject.toml", + "setup.cfg", + "setup.py", + "LICENSE", + "MANIFEST.in", + "README.md", + "PKG-INFO", +} + +local_sdist_files = { + ".egg-info", + ".egg-info/PKG-INFO", + ".egg-info/SOURCES.txt", + ".egg-info/dependency_links.txt", + ".egg-info/not-zip-safe", + ".egg-info/top_level.txt", +} + + +def test_build_sdist(monkeypatch, tmpdir): + + monkeypatch.chdir(MAIN_DIR) + + out = subprocess.check_output( + [ + sys.executable, + "setup.py", + "sdist", + "--formats=tar", + "--dist-dir", + str(tmpdir), + ] + ) + if hasattr(out, "decode"): + out = out.decode() + + assert out.count("copying") == 48 + + (sdist,) = tmpdir.visit("*.tar") + + with tarfile.open(str(sdist)) as tar: + start = tar.getnames()[0] + "/" + simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "setup.py")) + ) as f: + setup_py = f.read() + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pyproject.toml")) + ) as f: + pyproject_toml = f.read() + + files = set("pybind11/{}".format(n) for n in all_files) + files |= sdist_files + files |= set("pybind11{}".format(n) for n in local_sdist_files) + assert simpler == files + + with open(os.path.join(MAIN_DIR, "tools", "setup_main.py"), "rb") as f: + assert setup_py == f.read() + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + assert pyproject_toml == f.read() + + +def test_build_alt_dist(monkeypatch, tmpdir): + + monkeypatch.chdir(MAIN_DIR) + monkeypatch.setenv("PYBIND11_ALT_SDIST", "1") + + out = subprocess.check_output( + [ + sys.executable, + "setup.py", + "sdist", + "--formats=tar", + "--dist-dir", + str(tmpdir), + ] + ) + if hasattr(out, "decode"): + out = out.decode() + + assert out.count("copying") == 48 + + (sdist,) = tmpdir.visit("*.tar") + + with tarfile.open(str(sdist)) as tar: + start = tar.getnames()[0] + "/" + simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "setup.py")) + ) as f: + setup_py = f.read() + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pyproject.toml")) + ) as f: + pyproject_toml = f.read() + + files = set("pybind11/{}".format(n) for n in all_files) + files |= sdist_files + files |= set("pybind11_inplace{}".format(n) for n in local_sdist_files) + assert simpler == files + + with open(os.path.join(MAIN_DIR, "tools", "setup_alt.py"), "rb") as f: + assert setup_py == f.read() + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + assert pyproject_toml == f.read() + + +def tests_build_wheel(monkeypatch, tmpdir): + monkeypatch.chdir(MAIN_DIR) + + subprocess.check_output( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + ) + + (wheel,) = tmpdir.visit("*.whl") + + files = set("pybind11/{}".format(n) for n in all_files) + files |= { + "dist-info/LICENSE", + "dist-info/METADATA", + "dist-info/WHEEL", + "dist-info/top_level.txt", + "dist-info/RECORD", + } + + with zipfile.ZipFile(str(wheel)) as z: + names = z.namelist() + + trimmed = set(n for n in names if "dist-info" not in n) + trimmed |= set( + "dist-info/{}".format(n.split("/", 1)[-1]) for n in names if "dist-info" in n + ) + assert files == trimmed + + +def tests_build_alt_wheel(monkeypatch, tmpdir): + monkeypatch.chdir(MAIN_DIR) + monkeypatch.setenv("PYBIND11_ALT_SDIST", "1") + + subprocess.check_output( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + ) + + (wheel,) = tmpdir.visit("*.whl") + + files = set("data/data/{}".format(n) for n in src_files) + files |= set("data/headers/{}".format(n[8:]) for n in headers) + files |= { + "dist-info/LICENSE", + "dist-info/METADATA", + "dist-info/WHEEL", + "dist-info/top_level.txt", + "dist-info/RECORD", + } + + with zipfile.ZipFile(str(wheel)) as z: + names = z.namelist() + + beginning = names[0].split("/", 1)[0].rsplit(".", 1)[0] + trimmed = set(n[len(beginning) + 1 :] for n in names) + + assert files == trimmed diff --git a/tests/extra_setuptools/pytest.ini b/tests/extra_setuptools/pytest.ini new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py new file mode 100644 index 0000000000..c88a009cfa --- /dev/null +++ b/tests/extra_setuptools/test_setuphelper.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +import os +import sys +import subprocess +from textwrap import dedent + +DIR = os.path.abspath(os.path.dirname(__file__)) +MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) + + +def test_simple_setup_py(monkeypatch, tmpdir): + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + u"""\ + import sys + sys.path.append("{MAIN_DIR}") + + from setuptools import setup, Extension + from pybind11.setup_helpers import BuildExt + + ext_modules = [ + Extension( + "simple_setup", + sorted(["main.cpp"]), + language="c++", + ), + ] + + setup( + name="simple_setup_package", + cmdclass=dict(build_ext=BuildExt), + ext_modules=ext_modules + ) + """ + ).format(MAIN_DIR=MAIN_DIR), + encoding="ascii", + ) + + (tmpdir / "main.cpp").write_text( + dedent( + u"""\ + #include + + int f(int x) { + return x * 3; + } + PYBIND11_MODULE(simple_setup, m) { + m.def("f", &f); + } + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "--inplace"], + stdout=sys.stdout, + stderr=sys.stderr, + ) + assert len(list(tmpdir.visit("simple_setup*"))) == 1 + assert len(list(tmpdir.listdir())) == 4 # two files + output + build_dir + + (tmpdir / "test.py").write_text( + dedent( + u"""\ + import simple_setup + assert simple_setup.f(3) == 9 + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) diff --git a/tests/pytest.ini b/tests/pytest.ini index d0646a0915..c47cbe9c1e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,6 +1,6 @@ [pytest] minversion = 3.1 -norecursedirs = test_cmake_build test_embed test_python_package +norecursedirs = test_* extra_* xfail_strict = True addopts = # show summary of skipped tests diff --git a/tests/test_python_package/test_sdist.py b/tests/test_python_package/test_sdist.py deleted file mode 100644 index 8c83fdb95a..0000000000 --- a/tests/test_python_package/test_sdist.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -import subprocess -import sys -import os - -# These tests must be run explicitly -# They require CMake 3.15+ (--install) - -DIR = os.path.abspath(os.path.dirname(__file__)) - - -def test_build_sdist(monkeypatch): - - main_dir = os.path.dirname(os.path.dirname(DIR)) - monkeypatch.chdir(main_dir) - - out = subprocess.check_output([sys.executable, "setup.py", "sdist"]) - if hasattr(out, 'decode'): - out = out.decode() - - print(out) - - assert 'pybind11/share/cmake/pybind11' in out - assert 'pybind11/include/pybind11' in out - - assert out.count("copying") == 82 diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index 545dc45d08..d561430a55 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -12,6 +12,7 @@ #include "constructor_stats.h" #include #include +#include #include diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 96e958e646..26a1e04892 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -300,7 +300,7 @@ _pybind11_generate_lto(pybind11::thin_lto TRUE) # ---------------------- pybind11_strip ----------------------------- function(pybind11_strip target_name) - # Strip unnecessary sections of the binary on Linux/Mac OS + # Strip unnecessary sections of the binary on Linux/macOS if(CMAKE_STRIP) if(APPLE) set(x_opt -x) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 812ec094aa..27eb4d9205 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -197,7 +197,7 @@ function(pybind11_add_module target_name) endif() if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) - # Strip unnecessary sections of the binary on Linux/Mac OS + # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/tools/pyproject.toml b/tools/pyproject.toml new file mode 100644 index 0000000000..9787c3bdf0 --- /dev/null +++ b/tools/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/tools/setup_alt.py b/tools/setup_alt.py new file mode 100644 index 0000000000..29db477107 --- /dev/null +++ b/tools/setup_alt.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Setup script for PyPI; use CMakeFile.txt to build extension modules + +import contextlib +import glob +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# Setuptools has to be before distutils +from setuptools import setup + +from distutils.command.install_headers import install_headers + +class InstallHeadersNested(install_headers): + def run(self): + headers = self.distribution.headers or [] + for header in headers: + # Remove pybind11/include/ + short_header = header.split("/", 2)[-1] + + dst = os.path.join(self.install_dir, os.path.dirname(short_header)) + self.mkpath(dst) + (out, _) = self.copy_file(header, dst) + self.outfiles.append(out) + + +main_headers = glob.glob("pybind11/include/pybind11/*.h") +detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") +cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") +headers = main_headers + detail_headers + + +setup( + name="pybind11_inplace", + packages=[], + headers=headers, + cmdclass={"install_headers": InstallHeadersNested}, + data_files=[ + ("share/cmake/pybind11", cmake_files), + ("include/pybind11", main_headers), + ("include/pybind11/detail", detail_headers), + ], +) diff --git a/tools/setup_main.py b/tools/setup_main.py new file mode 100644 index 0000000000..3b58c0550f --- /dev/null +++ b/tools/setup_main.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Setup script for PyPI - placed in the sdist + +from setuptools import setup + + +setup( + name="pybind11", + packages=[ + "pybind11", + "pybind11.include.pybind11", + "pybind11.include.pybind11.detail", + "pybind11.share.cmake.pybind11", + ], + package_data={ + "pybind11.include.pybind11": ["*.h"], + "pybind11.include.pybind11.detail": ["*.h"], + "pybind11.share.cmake.pybind11": ["*.cmake"], + }, +) From 962f94d3a4454cda7de5851cea2e6c85f66ec0c7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 26 Aug 2020 21:31:39 -0400 Subject: [PATCH 03/28] refactor: helper file update and Windows support --- .github/workflows/ci.yml | 12 +++ pybind11/setup_helpers.py | 92 +++++++++++----------- tests/extra_setuptools/test_setuphelper.py | 5 +- 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bef228439f..e5a5d76821 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,18 @@ jobs: - name: Interface test run: cmake --build build2 --target test_cmake_build + - name: Prepare compiler environment for Windows 🐍 2.7 + if: matrix.python == 2.7 && runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Set Windows 🐍 2.7 environment variables + if: matrix.python == 2.7 && runner.os == 'Windows' + run: | + echo "::set-env name=DISTUTILS_USE_SDK::1" + echo "::set-env name=MSSdk::1" + - name: Setuptools helpers test run: pytest tests/extra_setuptools diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 6a69c27bc1..103f8885bd 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -47,25 +47,15 @@ from distutils.command.build_ext import build_ext +WIN = sys.platform.startswith("win") +PY2 = sys.version_info[0] < 3 + + # It is recommended to use PEP 518 builds if using this module. However, this # file explicitly supports being copied into a user's project directory # standalone, and pulling pybind11 with the deprecated setup_requires feature. -class DelayedPybindInclude(object): - """ - Helper class to determine the pybind11 include path The purpose of this - class is to postpone importing pybind11 until it is actually installed, so - that the ``get_include()`` method can be invoked if pybind11 is loaded via - setup_requires. - """ - - def __str__(self): - import pybind11 - - return pybind11.get_include() - - # Just in case someone clever tries to multithread tmp_chdir_lock = threading.Lock() @@ -107,39 +97,36 @@ def has_flag(compiler, flagname): def cpp_flag(compiler, value=None): """ - Return the ``-std=c++[11/14/17]`` compiler flag. - The newer version is preferred over c++11 (when it is available). + Return the ``-std=c++[11/14/17]`` compiler flag(s) as a list. May add + register fix for Python 2. The newer version is preferred over c++11 (when + it is available). """ flags = ["-std=c++17", "-std=c++14", "-std=c++11"] if value is not None: - mapping = {17: 0, 14: 1, 11: 2} - flags = [flags[mapping[value]]] + flags = ["-std=c++{}".format(value)] for flag in flags: + if sys.platform.startswith("win32"): + # MSVC 2017+ + flag = "/std:{}".format(flag[5:]).replace("11", "14") + if has_flag(compiler, flag): - return flag + cxx17plus = (value is not None and value >= 17) or flag == "-std=c++17" + cxx14plus = ( + cxx17plus or (value is not None and value >= 14) or flag == "-std=c++14" + ) + if PY2: + if cxx17plus: + return [flag, "/wd503" if WIN else "-Wno-register"] + elif cxx14plus and not WIN: + return [flag, "-Wno-deprecated-register"] + return [flag] raise RuntimeError("Unsupported compiler -- at least C++11 support is needed!") -c_opts = { - "msvc": ["/EHsc"], - "unix": [], -} - -l_opts = { - "msvc": [], - "unix": [], -} - -if sys.platform == "darwin": - darwin_opts = ["-stdlib=libc++", "-mmacosx-version-min=10.9"] - c_opts["unix"] += darwin_opts - l_opts["unix"] += darwin_opts - - class BuildExt(build_ext): """ Customized build_ext that can be further customized by users. @@ -158,18 +145,35 @@ def build_extensions(self): """ def build_extensions(self): - ct = self.compiler.compiler_type - comp_opts = c_opts.get(ct, []) - link_opts = l_opts.get(ct, []) - if ct == "unix": - comp_opts.append(cpp_flag(self.compiler, getattr(self, "cxx_std", None))) + # Import here to support `setup_requires` if someone really has to use it. + import pybind11 + + visibility_flag = None + + std_flags = cpp_flag(self.compiler, getattr(self, "cxx_std", None)) + + if self.compiler.compiler_type == "unix": if has_flag(self.compiler, "-fvisibility=hidden"): - comp_opts.append("-fvisibility=hidden") + visibility_flag = "-fvisibility=hidden" for ext in self.extensions: - ext.extra_compile_args += comp_opts - ext.extra_link_args += link_opts - ext.include_dirs += [DelayedPybindInclude()] + ext.extra_compile_args += std_flags + + if sys.platform.startswith("win32"): + ext.extra_compile_args.append("/EHsc") + + if visibility_flag: + ext.extra_compile_args.append(visibility_flag) + + if sys.platform.startswith("darwin"): + # Question: Do we need this as long as macos min version is more than 10.9? + ext.extra_compile_args.append("-stdlib=libc++") + + if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: + ext.extra_compile_args.append("-mmacos-version-min=10.9") + ext.extra_link_args.append("-mmacos-version-min=10.9") + + ext.include_dirs += [pybind11.get_include()] # Python 2 doesn't allow super here, since it's not a "class" build_ext.build_extensions(self) diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index c88a009cfa..4a3e7bcc5b 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -60,7 +60,10 @@ def test_simple_setup_py(monkeypatch, tmpdir): stdout=sys.stdout, stderr=sys.stderr, ) - assert len(list(tmpdir.visit("simple_setup*"))) == 1 + print(tmpdir.listdir()) + so = list(tmpdir.visit("simple_setup*so")) + pyd = list(tmpdir.visit("simple_setup*pyd")) + assert len(so + pyd) == 1 assert len(list(tmpdir.listdir())) == 4 # two files + output + build_dir (tmpdir / "test.py").write_text( From 05eea6a74f3cef40112deb2831167c65129d22bd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 27 Aug 2020 14:32:34 -0400 Subject: [PATCH 04/28] fix: review points from @YannickJadoul --- docs/compiling.rst | 4 +-- pybind11/__main__.py | 2 +- pybind11/commands.py | 5 ++- tests/extra_setuptools/test_setuphelper.py | 38 +++++++++++----------- tests/test_sequences_and_iterators.cpp | 1 - 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 9617f50dd3..7dd67c6968 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -51,8 +51,8 @@ An example of a ``setup.py`` using pybind11's helpers: If you copy ``setup_helpers.py`` into your local project to try to support the classic build procedure, then you will need to use the deprecated -``setup_requires=["pybind11>=2.6.0"]`` keyword argument to setup; -``setup_helpers`` tries to support this as well. +``setup_requires=["pybind11"]`` keyword argument to setup; ``setup_helpers`` +tries to support this as well. .. versionchanged:: 2.6 diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 4e5128aaf9..a7359d7414 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -5,7 +5,7 @@ import sys import sysconfig -from . import get_include +from .commands import get_include def print_includes(): diff --git a/pybind11/commands.py b/pybind11/commands.py index ee1ef6dc98..fa7eac3ccd 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -16,6 +16,5 @@ def get_cmake_dir(): if os.path.exists(cmake_installed_path): return cmake_installed_path else: - raise ImportError( - "pybind11 not installed, installation required to access the CMake files" - ) + msg = "pybind11 not installed, installation required to access the CMake files" + raise ImportError(msg) diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 4a3e7bcc5b..5b07ea0764 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -15,26 +15,26 @@ def test_simple_setup_py(monkeypatch, tmpdir): (tmpdir / "setup.py").write_text( dedent( u"""\ - import sys - sys.path.append("{MAIN_DIR}") + import sys + sys.path.append({MAIN_DIR!r}) - from setuptools import setup, Extension - from pybind11.setup_helpers import BuildExt + from setuptools import setup, Extension + from pybind11.setup_helpers import BuildExt - ext_modules = [ - Extension( - "simple_setup", - sorted(["main.cpp"]), - language="c++", - ), - ] + ext_modules = [ + Extension( + "simple_setup", + sorted(["main.cpp"]), + language="c++", + ), + ] - setup( - name="simple_setup_package", - cmdclass=dict(build_ext=BuildExt), - ext_modules=ext_modules - ) - """ + setup( + name="simple_setup_package", + cmdclass=dict(build_ext=BuildExt), + ext_modules=ext_modules + ) + """ ).format(MAIN_DIR=MAIN_DIR), encoding="ascii", ) @@ -50,7 +50,7 @@ def test_simple_setup_py(monkeypatch, tmpdir): PYBIND11_MODULE(simple_setup, m) { m.def("f", &f); } - """ + """ ), encoding="ascii", ) @@ -71,7 +71,7 @@ def test_simple_setup_py(monkeypatch, tmpdir): u"""\ import simple_setup assert simple_setup.f(3) == 9 - """ + """ ), encoding="ascii", ) diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index d561430a55..545dc45d08 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -12,7 +12,6 @@ #include "constructor_stats.h" #include #include -#include #include From 86d850774ca22ef3920c113113e240491cfbc64e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 28 Aug 2020 09:14:35 -0400 Subject: [PATCH 05/28] refactor: fixes to naming and more docs --- docs/changelog.rst | 14 ++++++++++++++ docs/compiling.rst | 4 ++-- docs/upgrade.rst | 21 +++++++++++++++++++-- pybind11/__main__.py | 9 ++++++++- pybind11/setup_helpers.py | 11 ++++++----- tests/extra_setuptools/test_setuphelper.py | 4 ++-- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3546040033..e2096519a2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,20 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. * ``py::memoryview`` update and documentation. `#2223 `_ +* The Python package was reworked to be more powerful and useful. + `#2433 `_ + + * A new ``pybind11.setup_helpers`` module provides utilities to easily use + setuptools with pybind11, and can be used via PEP 518 or by directly + copying ``setup_helpers.py`` into your project. + + * CMake configuration files are now included in the Python package. Use + ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get + the CMake directory, or include the site-packages location in your + ``CMAKE_MODULE_PATH``. Or you can use the new ``pybind11-inplace`` module, + which installs the includes and headers into your base environment in the + standard location. + * Minimum CMake required increased to 3.4. `#2338 `_ and `#2370 `_ diff --git a/docs/compiling.rst b/docs/compiling.rst index 7dd67c6968..3707a4defb 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -32,7 +32,7 @@ An example of a ``setup.py`` using pybind11's helpers: .. code-block:: python from setuptools import setup, Extension - from pybind11.setup_helpers import BuildExt + from pybind11.setup_helpers import build_ext ext_modules = [ Extension( @@ -44,7 +44,7 @@ An example of a ``setup.py`` using pybind11's helpers: setup( ..., - cmdclass={"build_ext": BuildExt}, + cmdclass={"build_ext": build_ext}, ext_modules=ext_modules ) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 502ce76eed..64d32c0f74 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -25,6 +25,12 @@ Usage of the ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function should be replaced by ``PYBIND11_OVERRIDE*`` and ``get_override``. In the future, the old macros may be deprecated and removed. +The ``pybind11`` package on PyPI no longer fills the wheel "headers" slot - if +you were using the headers from this slot, they are available in the +``pybind11-inplace`` package. (Most users will be unaffected, as the +``pybind11/include`` location is reported by ``pybind11 --includes`` and +``pybind11.get_include()`` is still correct and has not changed since 2.5). + CMake support: -------------- @@ -63,9 +69,20 @@ In addition, the following changes may be of interest: * Using ``find_package(Python COMPONENTS Interpreter Development)`` before pybind11 will cause pybind11 to use the new Python mechanisms instead of its - own custom search, based on a patched version of classic - FindPythonInterp/FindPythonLibs. In the future, this may become the default. + own custom search, based on a patched version of classic ``FindPythonInterp`` + / ``FindPythonLibs``. In the future, this may become the default. + + + +v2.5 +==== +The Python package now includes the headers as data in the package itself, as +well as in the "headers" wheel slot. ``pybind11 --includes`` and +``pybind11.get_include()`` report the new location, which is always correct +regardless of how pybind11 was installed, making the old ``user=`` argument +meaningless. If you are not using the helper, you are encouraged to switch to +the package location. v2.2 diff --git a/pybind11/__main__.py b/pybind11/__main__.py index a7359d7414..18f8bd7403 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -5,7 +5,7 @@ import sys import sysconfig -from .commands import get_include +from .commands import get_include, get_cmake_dir def print_includes(): @@ -31,11 +31,18 @@ def main(): action="store_true", help="Include flags for both pybind11 and Python headers.", ) + parser.add_argument( + "--cmakedir", + action="store_true", + help="Print the CMake module directory, ideal for pybind11_ROOT.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() if args.includes: print_includes() + if args.includes: + print(get_cmake_dir()) if __name__ == "__main__": diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 103f8885bd..568be1523c 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -44,7 +44,7 @@ import threading import distutils.errors -from distutils.command.build_ext import build_ext +import distutils.command.build_ext WIN = sys.platform.startswith("win") @@ -127,18 +127,19 @@ def cpp_flag(compiler, value=None): raise RuntimeError("Unsupported compiler -- at least C++11 support is needed!") -class BuildExt(build_ext): +class build_ext(distutils.command.build_ext.build_ext): # noqa: N801 """ Customized build_ext that can be further customized by users. Most use cases can be addressed by adding items to the extensions. However, if you need to customize, try: - class BuildExt(pybind11.setup_utils.BuildExt): + class build_ext(pybind11.setup_utils.build_ext): def build_extensions(self): # Do something here, like add things to extensions - super(BuildExt, self).build_extensions() + # super only works on Python 3 due to distutils oddities + pybind11.setup_utils.build_ext.build_extensions(self) One simple customization point is provided: ``self.cxx_std`` lets you set a C++ standard (None is the default search). @@ -176,4 +177,4 @@ def build_extensions(self): ext.include_dirs += [pybind11.get_include()] # Python 2 doesn't allow super here, since it's not a "class" - build_ext.build_extensions(self) + distutils.command.build_ext.build_ext.build_extensions(self) diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 5b07ea0764..69c09437df 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -19,7 +19,7 @@ def test_simple_setup_py(monkeypatch, tmpdir): sys.path.append({MAIN_DIR!r}) from setuptools import setup, Extension - from pybind11.setup_helpers import BuildExt + from pybind11.setup_helpers import build_ext ext_modules = [ Extension( @@ -31,7 +31,7 @@ def test_simple_setup_py(monkeypatch, tmpdir): setup( name="simple_setup_package", - cmdclass=dict(build_ext=BuildExt), + cmdclass=dict(build_ext=build_ext), ext_modules=ext_modules ) """ From 83dc3873de703f7010b32531bcee6fcff48007e0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 29 Aug 2020 11:15:40 -0400 Subject: [PATCH 06/28] feat: more customization points --- pybind11/setup_helpers.py | 207 +++++++++++++++++++++++++------------- 1 file changed, 135 insertions(+), 72 deletions(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 568be1523c..9f6d965987 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -54,6 +54,8 @@ # It is recommended to use PEP 518 builds if using this module. However, this # file explicitly supports being copied into a user's project directory # standalone, and pulling pybind11 with the deprecated setup_requires feature. +# If you copy the file, remember to add it to your MANIFEST.in, and add the current +# directory into your path if it sits beside your setup.py. # Just in case someone clever tries to multithread @@ -63,8 +65,8 @@ @contextlib.contextmanager def tmp_chdir(): "Prepare and enter a temporary directory, cleanup when done" - # Threadsafe + # Threadsafe with tmp_chdir_lock: olddir = os.getcwd() try: @@ -76,63 +78,13 @@ def tmp_chdir(): shutil.rmtree(tmpdir) -# cf http://bugs.python.org/issue26689 -def has_flag(compiler, flagname): - """ - Return a boolean indicating whether a flag name is supported on the - specified compiler. - """ - - with tmp_chdir(): - fname = "flagcheck.cpp" - with open(fname, "w") as f: - f.write("int main (int argc, char **argv) { return 0; }") - - try: - compiler.compile([fname], extra_postargs=[flagname]) - return True - except distutils.errors.CompileError: - return False - - -def cpp_flag(compiler, value=None): - """ - Return the ``-std=c++[11/14/17]`` compiler flag(s) as a list. May add - register fix for Python 2. The newer version is preferred over c++11 (when - it is available). - """ - - flags = ["-std=c++17", "-std=c++14", "-std=c++11"] - - if value is not None: - flags = ["-std=c++{}".format(value)] - - for flag in flags: - if sys.platform.startswith("win32"): - # MSVC 2017+ - flag = "/std:{}".format(flag[5:]).replace("11", "14") - - if has_flag(compiler, flag): - cxx17plus = (value is not None and value >= 17) or flag == "-std=c++17" - cxx14plus = ( - cxx17plus or (value is not None and value >= 14) or flag == "-std=c++14" - ) - if PY2: - if cxx17plus: - return [flag, "/wd503" if WIN else "-Wno-register"] - elif cxx14plus and not WIN: - return [flag, "-Wno-deprecated-register"] - return [flag] - - raise RuntimeError("Unsupported compiler -- at least C++11 support is needed!") - - class build_ext(distutils.command.build_ext.build_ext): # noqa: N801 """ Customized build_ext that can be further customized by users. Most use cases can be addressed by adding items to the extensions. - However, if you need to customize, try: + However, if you need to customize beyond the customization points provided, + try: class build_ext(pybind11.setup_utils.build_ext): def build_extensions(self): @@ -141,39 +93,150 @@ def build_extensions(self): # super only works on Python 3 due to distutils oddities pybind11.setup_utils.build_ext.build_extensions(self) - One simple customization point is provided: ``self.cxx_std`` lets - you set a C++ standard (None is the default search). + Simple customization points are provided: CPP_FLAGS, UNIX_FLAGS, + LINUX_FLAGS, DARWIN_FLAGS, and WIN_FLAGS (along with ``*_LINK_FLAGS`` + versions). You can override these per class or per instance. + + If CPP_FLAGS is empty, no search is done. The first supported flag is used. + Reasonable defaults are selected for the flags for optimal extensions. An + ALL_COMPILERS set lists flags that will always pass "has_flag". + + Two flags are special, and are not listed here. One is the Python 2 + + C++14+/C++17 register flag; this is added by cpp_flags. The other is the + macos-version-min flag, which is only added if MACOSX_DEPLOYMENT_TARGET is + not set, and is based on the C++ standard selected. """ + CPP_FLAGS = ("-std=c++17", "-std=c++14", "-std=c++11") + LINUX_FLAGS = () + LINUX_LINK_FLAGS = () + UNIX_FLAGS = ("-fvisibility=hidden", "-g0") + UNIX_LINK_FLAGS = () + DARWIN_FLAGS = ("-stdlib=libc++",) + DARWIN_LINK_FLAGS = ("-stdlib=libc++",) + WIN_FLAGS = ("/EHsc", "/bigobj") + WIN_LINK_FLAGS = () + ALL_COMPILERS = {"-g0", "/EHsc", "/bigobj"} + + # cf http://bugs.python.org/issue26689 + def has_flag(self, *flagnames): + """ + Return the flag if a flag name is supported on the + specified compiler, otherwise None (can be used as a boolean). + If multiple flags are passed, return the first that matches. + """ + + with tmp_chdir(): + fname = "flagcheck.cpp" + for flagname in flagnames: + if flagname in self.ALL_COMPILERS or "mmacosx-version-min" in flagname: + return flagname + with open(fname, "w") as f: + f.write("int main (int argc, char **argv) { return 0; }") + + try: + self.compiler.compile([fname], extra_postargs=[flagname]) + return flagname + except distutils.errors.CompileError: + pass + + return None + + def cpp_flags(self): + """ + Return the ``-std=c++[11/14/17]`` compiler flag(s) as a list. May add + register fix for Python 2. The newer version is preferred over c++11 + (when it is available). Windows will not fail, since MSVC 15 doesn't + have these flags but supports C++14 (for the most part). The first flag + is always the C++ selection flag. + """ + + # None or missing attribute, provide default list; if empty list, return nothing + if not self.CPP_FLAGS: + return [] + + # Windows uses a different form + if WIN: + # MSVC 2017+ + flags = [ + "/std:{}".format(flag[5:]).replace("11", "14") + for flag in self.CPP_FLAGS + ] + else: + flags = self.CPP_FLAGS + + flag = self.has_flag(*flags) + + if flag is None: + # On Windows, the default is to support C++14 on MSVC 2015, and it is + # not a specific flag so not failing here if on Windows. An empty list + # also passes since it doesn't look for anything. + if WIN: + return [] + else: + msg = "Unsupported compiler -- at least C++11 support is needed!" + raise RuntimeError(msg) + + if PY2: + try: + value = int(flag[-2:]) + if value >= 17: + return [flag, "/wd503" if WIN else "-Wno-register"] + elif not WIN and value >= 14: + return [flag, "-Wno-deprecated-register"] + except ValueError: + return [flag, "/wd503" if WIN else "-Wno-register"] + + return [flag] + def build_extensions(self): + """ + Build extensions, injecting extra flags and includes as needed. + """ # Import here to support `setup_requires` if someone really has to use it. import pybind11 - visibility_flag = None - - std_flags = cpp_flag(self.compiler, getattr(self, "cxx_std", None)) + def valid_flags(flagnames): + return [flag for flag in flagnames if self.has_flag(flag)] - if self.compiler.compiler_type == "unix": - if has_flag(self.compiler, "-fvisibility=hidden"): - visibility_flag = "-fvisibility=hidden" - - for ext in self.extensions: - ext.extra_compile_args += std_flags + extra_compile_args = [] + extra_link_args = [] - if sys.platform.startswith("win32"): - ext.extra_compile_args.append("/EHsc") + cpp_flags = self.cpp_flags() + extra_compile_args += cpp_flags - if visibility_flag: - ext.extra_compile_args.append(visibility_flag) + if WIN: + extra_compile_args += valid_flags(self.WIN_FLAGS) + extra_link_args += self.WIN_LINK_FLAGS + else: + extra_compile_args += valid_flags(self.UNIX_FLAGS) + extra_link_args += self.UNIX_LINK_FLAGS if sys.platform.startswith("darwin"): - # Question: Do we need this as long as macos min version is more than 10.9? - ext.extra_compile_args.append("-stdlib=libc++") + extra_compile_args += valid_flags(self.DARWIN_FLAGS) + extra_link_args += self.DARWIN_LINK_FLAGS if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: - ext.extra_compile_args.append("-mmacos-version-min=10.9") - ext.extra_link_args.append("-mmacos-version-min=10.9") + macosx_min = "-mmacosx-version-min=10.9" + # C++17 requires a higher min version of macOS + if cpp_flags: + try: + if int(cpp_flags[0][-2:]) >= 17: + macosx_min = "-mmacosx-version-min=10.14" + except ValueError: + pass + + extra_compile_args.append(macosx_min) + extra_link_args.append(macosx_min) + + else: + extra_compile_args += valid_flags(self.LINUX_FLAGS) + extra_link_args += self.LINUX_LINK_FLAGS + + for ext in self.extensions: + ext.extra_compile_args += extra_compile_args + ext.extra_link_args += extra_link_args ext.include_dirs += [pybind11.get_include()] # Python 2 doesn't allow super here, since it's not a "class" From 7b4a6666f761f82077b3b1ecfe22c8195f0cded0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 29 Aug 2020 13:39:19 -0400 Subject: [PATCH 07/28] feat: add entry point pybind11-config --- pybind11/__main__.py | 2 +- tests/extra_python_package/test_files.py | 6 ++++-- tools/setup_main.py | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 18f8bd7403..911c075f40 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -25,7 +25,7 @@ def print_includes(): def main(): - parser = argparse.ArgumentParser(prog="python -m pybind11") + parser = argparse.ArgumentParser() parser.add_argument( "--includes", action="store_true", diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 18c40a8182..4c3ae0c23a 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -110,7 +110,7 @@ def test_build_sdist(monkeypatch, tmpdir): if hasattr(out, "decode"): out = out.decode() - assert out.count("copying") == 48 + assert out.count("copying") == 49 (sdist,) = tmpdir.visit("*.tar") @@ -131,6 +131,7 @@ def test_build_sdist(monkeypatch, tmpdir): files = set("pybind11/{}".format(n) for n in all_files) files |= sdist_files files |= set("pybind11{}".format(n) for n in local_sdist_files) + files.add("pybind11.egg-info/entry_points.txt") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py"), "rb") as f: @@ -201,9 +202,10 @@ def tests_build_wheel(monkeypatch, tmpdir): files |= { "dist-info/LICENSE", "dist-info/METADATA", + "dist-info/RECORD", "dist-info/WHEEL", + "dist-info/entry_points.txt", "dist-info/top_level.txt", - "dist-info/RECORD", } with zipfile.ZipFile(str(wheel)) as z: diff --git a/tools/setup_main.py b/tools/setup_main.py index 3b58c0550f..649a00f203 100644 --- a/tools/setup_main.py +++ b/tools/setup_main.py @@ -19,4 +19,9 @@ "pybind11.include.pybind11.detail": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], }, + entry_points={ + "console_scripts": [ + "pybind11-config = pybind11.__main__:main", + ] + } ) From d02875aca43f165972e8fee417f391ce8f2bed1a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 29 Aug 2020 22:29:58 -0400 Subject: [PATCH 08/28] refactor: Try Extension-focused method --- docs/compiling.rst | 34 ++- pybind11/__main__.py | 2 +- pybind11/setup_helpers.py | 283 +++++++++++---------- tests/extra_setuptools/test_setuphelper.py | 36 ++- 4 files changed, 204 insertions(+), 151 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 3707a4defb..b5fd9c1b59 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -31,20 +31,18 @@ An example of a ``setup.py`` using pybind11's helpers: .. code-block:: python - from setuptools import setup, Extension - from pybind11.setup_helpers import build_ext + from setuptools import setup # Always import first + from pybind11.setup_helpers import Pybind11Extension ext_modules = [ - Extension( + Pybind11Extension( "python_example", sorted(["src/main.cpp"]), - language="c++", ), ] setup( ..., - cmdclass={"build_ext": build_ext}, ext_modules=ext_modules ) @@ -52,7 +50,31 @@ An example of a ``setup.py`` using pybind11's helpers: If you copy ``setup_helpers.py`` into your local project to try to support the classic build procedure, then you will need to use the deprecated ``setup_requires=["pybind11"]`` keyword argument to setup; ``setup_helpers`` -tries to support this as well. +tries to support this as well. As with all distutils compatible tools, you need +to import setuptools before importing from ``pybind11.setup_helpers``. + +If you want to do an automatic search for the highest supported C++ standard, +that is supported via a ``build_ext`` command override; it will only affect +Pybind11Extensions: + +.. code-block:: python + + from setuptools import setup # Always import first + from pybind11.setup_helpers import Pybind11Extension, build_ext + + ext_modules = [ + Pybind11Extension( + "python_example", + sorted(["src/main.cpp"]), + ), + ] + + setup( + ..., + cmdclass={"build_ext": build_ext}, + ext_modules=ext_modules + ) + .. versionchanged:: 2.6 diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 911c075f40..3553726449 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -41,7 +41,7 @@ def main(): parser.print_help() if args.includes: print_includes() - if args.includes: + if args.cmakedir: print(get_cmake_dir()) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 9f6d965987..4cc8cbab50 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- """ -This module provides a way to check to see if flag is available, -has_flag (built-in to distutils.CCompiler in Python 3.6+), and -a cpp_flag function, which will compute the highest available -flag (or if a flag is supported). +This module provides a LICENSE: @@ -42,13 +39,17 @@ import sys import tempfile import threading +import warnings import distutils.errors import distutils.command.build_ext +import distutils.extension -WIN = sys.platform.startswith("win") +WIN = sys.platform.startswith("win32") PY2 = sys.version_info[0] < 3 +MACOS = sys.platform.startswith("darwin") +STD_TMPL = "/std:cxx{}" if WIN else "-std=c++{}" # It is recommended to use PEP 518 builds if using this module. However, this @@ -60,6 +61,7 @@ # Just in case someone clever tries to multithread tmp_chdir_lock = threading.Lock() +cpp_cache_lock = threading.Lock() @contextlib.contextmanager @@ -78,166 +80,181 @@ def tmp_chdir(): shutil.rmtree(tmpdir) -class build_ext(distutils.command.build_ext.build_ext): # noqa: N801 +class Pybind11Extension(distutils.extension.Extension): + """ + Build a Pybind11 Extension module. This automatically adds the recommended + flags when you init the extension and assumes C++ sources - you can further + modify the options yourself. + + The customizations are: + + * ``/EHsc`` and ``/bigobj`` on Windows + * ``stdlib=libc++`` on macOS + * ``visibility=hidden`` and ``-g0`` on Unix + + Finally, you can set ``cxx_std`` via constructor or afterwords to enable + flags for C++ std, and a few extra helper flags related to the C++ standard + level. It is _highly_ recommended you either set this, or use the provided + ``build_ext``, which will search for the highest supported extension for + you if the ``cxx_std`` property is not set. Do not set the ``cxx_std`` + property more than once, as flags are added when you set it. Set the + property to None to disable the addition of C++ standard flags. + + Warning: do not use property-based access to the instance on Python 2 - + this is an ugly old-style class due to Distutils. """ - Customized build_ext that can be further customized by users. - Most use cases can be addressed by adding items to the extensions. - However, if you need to customize beyond the customization points provided, - try: + def _add_cflags(self, *flags): + for flag in flags: + if flag not in self.extra_compile_args: + self.extra_compile_args.append(flag) - class build_ext(pybind11.setup_utils.build_ext): - def build_extensions(self): - # Do something here, like add things to extensions + def _add_lflags(self, *flags): + for flag in flags: + if flag not in self.extra_compile_args: + self.extra_link_args.append(flag) - # super only works on Python 3 due to distutils oddities - pybind11.setup_utils.build_ext.build_extensions(self) + def __init__(self, *args, **kwargs): - Simple customization points are provided: CPP_FLAGS, UNIX_FLAGS, - LINUX_FLAGS, DARWIN_FLAGS, and WIN_FLAGS (along with ``*_LINK_FLAGS`` - versions). You can override these per class or per instance. + self._cxx_level = 0 + cxx_std = kwargs.pop("cxx_std", 0) - If CPP_FLAGS is empty, no search is done. The first supported flag is used. - Reasonable defaults are selected for the flags for optimal extensions. An - ALL_COMPILERS set lists flags that will always pass "has_flag". + if "language" not in kwargs: + kwargs["language"] = "c++" - Two flags are special, and are not listed here. One is the Python 2 + - C++14+/C++17 register flag; this is added by cpp_flags. The other is the - macos-version-min flag, which is only added if MACOSX_DEPLOYMENT_TARGET is - not set, and is based on the C++ standard selected. - """ + # Can't use super here because distutils has old-style classes in + # Python 2! + distutils.extension.Extension.__init__(self, *args, **kwargs) - CPP_FLAGS = ("-std=c++17", "-std=c++14", "-std=c++11") - LINUX_FLAGS = () - LINUX_LINK_FLAGS = () - UNIX_FLAGS = ("-fvisibility=hidden", "-g0") - UNIX_LINK_FLAGS = () - DARWIN_FLAGS = ("-stdlib=libc++",) - DARWIN_LINK_FLAGS = ("-stdlib=libc++",) - WIN_FLAGS = ("/EHsc", "/bigobj") - WIN_LINK_FLAGS = () - ALL_COMPILERS = {"-g0", "/EHsc", "/bigobj"} - - # cf http://bugs.python.org/issue26689 - def has_flag(self, *flagnames): - """ - Return the flag if a flag name is supported on the - specified compiler, otherwise None (can be used as a boolean). - If multiple flags are passed, return the first that matches. - """ + # Have to use the accessor manually to support Python 2 distutils + CppExtension.cxx_std.__set__(self, cxx_std) - with tmp_chdir(): - fname = "flagcheck.cpp" - for flagname in flagnames: - if flagname in self.ALL_COMPILERS or "mmacosx-version-min" in flagname: - return flagname - with open(fname, "w") as f: - f.write("int main (int argc, char **argv) { return 0; }") + # If using setup_requires, this fails the first time - that's okay + try: + import pybind11 + + pyinc = pybind11.get_include() - try: - self.compiler.compile([fname], extra_postargs=[flagname]) - return flagname - except distutils.errors.CompileError: - pass + if pyinc not in self.include_dirs: + self.include_dirs.append(pyinc) + except ImportError: + pass - return None + if WIN: + self._add_cflags("/EHsc", "/bigobj") + else: + self._add_cflags("-fvisibility=hidden", "-g0") + if MACOS: + self._add_cflags("-stdlib=libc++") + self._add_lflags("-stdlib=libc++") - def cpp_flags(self): + @property + def cxx_std(self): """ - Return the ``-std=c++[11/14/17]`` compiler flag(s) as a list. May add - register fix for Python 2. The newer version is preferred over c++11 - (when it is available). Windows will not fail, since MSVC 15 doesn't - have these flags but supports C++14 (for the most part). The first flag - is always the C++ selection flag. + The CXX standard level. If set, will add the required flags. If left + at 0, it will trigger an automatic search when pybind11's build_ext + is used. If None, will have no effect. Besides just the flags, this + may add a register warning/error fix for Python 2 or macos-min 10.9 + or 10.14. """ + return self._cxx_level - # None or missing attribute, provide default list; if empty list, return nothing - if not self.CPP_FLAGS: - return [] + @cxx_std.setter + def cxx_std(self, level): - # Windows uses a different form - if WIN: - # MSVC 2017+ - flags = [ - "/std:{}".format(flag[5:]).replace("11", "14") - for flag in self.CPP_FLAGS - ] - else: - flags = self.CPP_FLAGS + if self._cxx_level: + warnings.warn("You cannot safely change the cxx_level after setting it!") + + self._cxx_level = level - flag = self.has_flag(*flags) + if not level or (WIN and level == 11): + return - if flag is None: - # On Windows, the default is to support C++14 on MSVC 2015, and it is - # not a specific flag so not failing here if on Windows. An empty list - # also passes since it doesn't look for anything. - if WIN: - return [] - else: - msg = "Unsupported compiler -- at least C++11 support is needed!" - raise RuntimeError(msg) + self.extra_compile_args.append(STD_TMPL.format(level)) + + if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ: + # C++17 requires a higher min version of macOS + macosx_min = "-mmacosx-version-min=" + ("10.9" if level < 17 else "10.14") + self.extra_compile_args.append(macosx_min) + self.extra_link_args.append(macosx_min) if PY2: - try: - value = int(flag[-2:]) - if value >= 17: - return [flag, "/wd503" if WIN else "-Wno-register"] - elif not WIN and value >= 14: - return [flag, "-Wno-deprecated-register"] - except ValueError: - return [flag, "/wd503" if WIN else "-Wno-register"] + if level >= 17: + self.extra_compile_args.append("/wd503" if WIN else "-Wno-register") + elif not WIN and level >= 14: + self.extra_compile_args.append("-Wno-deprecated-register") - return [flag] - def build_extensions(self): - """ - Build extensions, injecting extra flags and includes as needed. - """ - # Import here to support `setup_requires` if someone really has to use it. - import pybind11 +# cf http://bugs.python.org/issue26689 +def has_flag(compiler, flag): + """ + Return the flag if a flag name is supported on the + specified compiler, otherwise None (can be used as a boolean). + If multiple flags are passed, return the first that matches. + """ - def valid_flags(flagnames): - return [flag for flag in flagnames if self.has_flag(flag)] + with tmp_chdir(): + fname = "flagcheck.cpp" + with open(fname, "w") as f: + f.write("int main (int argc, char **argv) { return 0; }") - extra_compile_args = [] - extra_link_args = [] + try: + compiler.compile([fname], extra_postargs=[flag]) + except distutils.errors.CompileError: + return False + return True - cpp_flags = self.cpp_flags() - extra_compile_args += cpp_flags - if WIN: - extra_compile_args += valid_flags(self.WIN_FLAGS) - extra_link_args += self.WIN_LINK_FLAGS - else: - extra_compile_args += valid_flags(self.UNIX_FLAGS) - extra_link_args += self.UNIX_LINK_FLAGS +# Every call will cache the result +cpp_flag_cache = None + + +def auto_cpp_level(compiler): + """ + Return the max supported C++ std level (17, 14, or 11). If neither 17 or 14 + is supported on Windows, "11" is returned. Caches result. + """ + + global cpp_flag_cache + + # If this has been previously calculated with the same args, return that + with cpp_cache_lock: + if cpp_flag_cache: + return cpp_flag_cache - if sys.platform.startswith("darwin"): - extra_compile_args += valid_flags(self.DARWIN_FLAGS) - extra_link_args += self.DARWIN_LINK_FLAGS + levels = [17, 14] + ([] if WIN else [11]) - if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: - macosx_min = "-mmacosx-version-min=10.9" + for level in levels: + if has_flag(compiler, STD_TMPL.format(level)): + with cpp_cache_lock: + cpp_flag_cache = level + return level - # C++17 requires a higher min version of macOS - if cpp_flags: - try: - if int(cpp_flags[0][-2:]) >= 17: - macosx_min = "-mmacosx-version-min=10.14" - except ValueError: - pass + if WIN: + with cpp_cache_lock: + cpp_flag_cache = 11 + return 11 - extra_compile_args.append(macosx_min) - extra_link_args.append(macosx_min) + msg = "Unsupported compiler -- at least C++11 support is needed!" + raise RuntimeError(msg) - else: - extra_compile_args += valid_flags(self.LINUX_FLAGS) - extra_link_args += self.LINUX_LINK_FLAGS + +class build_ext(distutils.command.build_ext.build_ext): # noqa: N801 + """ + Customized build_ext that allows an auto-search for the highest supported + C++ level for Pybind11Extension. + """ + + def build_extensions(self): + """ + Build extensions, injecting C++ std for Pybind11Extension if needed. + """ for ext in self.extensions: - ext.extra_compile_args += extra_compile_args - ext.extra_link_args += extra_link_args - ext.include_dirs += [pybind11.get_include()] + if hasattr(ext, "_cxx_level") and ext._cxx_level == 0: + # Python 2 syntax - old-style distutils class + ext.__class__.cxx_std.__set__(ext, auto_cpp_level(self.compiler)) - # Python 2 doesn't allow super here, since it's not a "class" + # Python 2 doesn't allow super here, since distutils uses old-style + # classes! distutils.command.build_ext.build_ext.build_extensions(self) diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 69c09437df..de0b516a9f 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -4,11 +4,14 @@ import subprocess from textwrap import dedent +import pytest + DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) -def test_simple_setup_py(monkeypatch, tmpdir): +@pytest.mark.parametrize("std", [11, 0]) +def test_simple_setup_py(monkeypatch, tmpdir, std): monkeypatch.chdir(tmpdir) monkeypatch.syspath_prepend(MAIN_DIR) @@ -19,23 +22,30 @@ def test_simple_setup_py(monkeypatch, tmpdir): sys.path.append({MAIN_DIR!r}) from setuptools import setup, Extension - from pybind11.setup_helpers import build_ext + from pybind11.setup_helpers import build_ext, Pybind11Extension + + std = {std} ext_modules = [ - Extension( + Pybind11Extension( "simple_setup", sorted(["main.cpp"]), - language="c++", + cxx_std=std, ), ] + cmdclass = dict() + if std == 0: + cmdclass["build_ext"] = build_ext + + setup( name="simple_setup_package", - cmdclass=dict(build_ext=build_ext), - ext_modules=ext_modules + cmdclass=cmdclass, + ext_modules=ext_modules, ) """ - ).format(MAIN_DIR=MAIN_DIR), + ).format(MAIN_DIR=MAIN_DIR, std=std), encoding="ascii", ) @@ -60,10 +70,14 @@ def test_simple_setup_py(monkeypatch, tmpdir): stdout=sys.stdout, stderr=sys.stderr, ) - print(tmpdir.listdir()) - so = list(tmpdir.visit("simple_setup*so")) - pyd = list(tmpdir.visit("simple_setup*pyd")) - assert len(so + pyd) == 1 + + # Debug helper printout, normally hidden + for item in tmpdir.listdir(): + print(item.basename) + + assert ( + len([f for f in tmpdir.listdir() if f.basename.startswith("simple_setup")]) == 1 + ) assert len(list(tmpdir.listdir())) == 4 # two files + output + build_dir (tmpdir / "test.py").write_text( From 40b61f11e10439b510a0c99a7fa454a3174c856c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 30 Aug 2020 00:17:13 -0400 Subject: [PATCH 09/28] refactor: rename alt/inplace to global --- .github/workflows/configure.yml | 4 ++-- docs/changelog.rst | 2 +- docs/upgrade.rst | 6 +++--- setup.py | 6 +++--- tests/extra_python_package/test_files.py | 8 ++++---- tools/{setup_alt.py => setup_global.py} | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) rename tools/{setup_alt.py => setup_global.py} (97%) diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 0ce235c52d..07122fbc6d 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -117,7 +117,7 @@ jobs: - name: Build SDist run: | python setup.py sdist - PYBIND11_ALT_SDIST=1 python setup.py sdist + PYBIND11_GLOBAL_SDIST=1 python setup.py sdist - uses: actions/upload-artifact@v2 with: @@ -126,7 +126,7 @@ jobs: - name: Build wheel run: | python -m pip wheel . -w wheels - PYBIND11_ALT_SDIST=1 python -m pip wheel . -w wheels + PYBIND11_GLOBAL_SDIST=1 python -m pip wheel . -w wheels - uses: actions/upload-artifact@v2 with: diff --git a/docs/changelog.rst b/docs/changelog.rst index e2096519a2..eb2f63a68d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,7 +47,7 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. * CMake configuration files are now included in the Python package. Use ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get the CMake directory, or include the site-packages location in your - ``CMAKE_MODULE_PATH``. Or you can use the new ``pybind11-inplace`` module, + ``CMAKE_MODULE_PATH``. Or you can use the new ``pybind11-global`` module, which installs the includes and headers into your base environment in the standard location. diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 64d32c0f74..f1148ac07f 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -27,9 +27,9 @@ old macros may be deprecated and removed. The ``pybind11`` package on PyPI no longer fills the wheel "headers" slot - if you were using the headers from this slot, they are available in the -``pybind11-inplace`` package. (Most users will be unaffected, as the -``pybind11/include`` location is reported by ``pybind11 --includes`` and -``pybind11.get_include()`` is still correct and has not changed since 2.5). +``pybind11-global`` package. (Most users will be unaffected, as the +``pybind11/include`` location is reported by ``python -m pybind11 --includes`` +and ``pybind11.get_include()`` is still correct and has not changed since 2.5). CMake support: -------------- diff --git a/setup.py b/setup.py index db3c0c88f5..5b724d20bd 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ import sys import tempfile -# PYBIND11_ALT_SDIST will build a different sdist, with the python-headers +# PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers # files, and the sys.prefix files (CMake and headers). -alt_sdist = os.environ.get("PYBIND11_ALT_SDIST", False) -setup_py = "tools/setup_alt.py" if alt_sdist else "tools/setup_main.py" +alt_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) +setup_py = "tools/setup_global.py" if alt_sdist else "tools/setup_main.py" pyproject_toml = "tools/pyproject.toml" # In a PEP 518 build, this will be in its own environment, so it will not diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 4c3ae0c23a..32d2cbdf8d 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -144,7 +144,7 @@ def test_build_sdist(monkeypatch, tmpdir): def test_build_alt_dist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - monkeypatch.setenv("PYBIND11_ALT_SDIST", "1") + monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") out = subprocess.check_output( [ @@ -179,10 +179,10 @@ def test_build_alt_dist(monkeypatch, tmpdir): files = set("pybind11/{}".format(n) for n in all_files) files |= sdist_files - files |= set("pybind11_inplace{}".format(n) for n in local_sdist_files) + files |= set("pybind11_global{}".format(n) for n in local_sdist_files) assert simpler == files - with open(os.path.join(MAIN_DIR, "tools", "setup_alt.py"), "rb") as f: + with open(os.path.join(MAIN_DIR, "tools", "setup_global.py"), "rb") as f: assert setup_py == f.read() with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: @@ -220,7 +220,7 @@ def tests_build_wheel(monkeypatch, tmpdir): def tests_build_alt_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - monkeypatch.setenv("PYBIND11_ALT_SDIST", "1") + monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") subprocess.check_output( [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] diff --git a/tools/setup_alt.py b/tools/setup_global.py similarity index 97% rename from tools/setup_alt.py rename to tools/setup_global.py index 29db477107..b4a7170a93 100644 --- a/tools/setup_alt.py +++ b/tools/setup_global.py @@ -37,7 +37,7 @@ def run(self): setup( - name="pybind11_inplace", + name="pybind11_global", packages=[], headers=headers, cmdclass={"install_headers": InstallHeadersNested}, From ea71cba653085632111478c3f25206d22534ef28 Mon Sep 17 00:00:00 2001 From: Henry Fredrick Schreiner Date: Tue, 1 Sep 2020 00:55:28 -0400 Subject: [PATCH 10/28] fix: allow usage with git modules, better docs --- docs/compiling.rst | 69 ++++++++++++++++++++++++++------------- pybind11/setup_helpers.py | 41 +++++++++++++++-------- 2 files changed, 74 insertions(+), 36 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index b5fd9c1b59..5b9c888df1 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -14,24 +14,11 @@ the [python_example]_ repository. .. [python_example] https://github.com/pybind/python_example A helper file is provided with pybind11 that can simplify usage with setuptools -if you have pybind11 installed as a Python package; the file is also standalone, -if you want to copy it to your package. If you use PEP518's ``pyproject.toml`` -file: - -.. code-block:: toml - - [build-system] - requires = ["setuptools", "wheel", "pybind11==2.6.0"] - build-backend = "setuptools.build_meta" - -you can ensure that pybind11 is available during the building of your project -(pip 10+ required). - An example of a ``setup.py`` using pybind11's helpers: .. code-block:: python - from setuptools import setup # Always import first + from setuptools import setup # Always import setuptools first from pybind11.setup_helpers import Pybind11Extension ext_modules = [ @@ -46,13 +33,6 @@ An example of a ``setup.py`` using pybind11's helpers: ext_modules=ext_modules ) - -If you copy ``setup_helpers.py`` into your local project to try to support the -classic build procedure, then you will need to use the deprecated -``setup_requires=["pybind11"]`` keyword argument to setup; ``setup_helpers`` -tries to support this as well. As with all distutils compatible tools, you need -to import setuptools before importing from ``pybind11.setup_helpers``. - If you want to do an automatic search for the highest supported C++ standard, that is supported via a ``build_ext`` command override; it will only affect Pybind11Extensions: @@ -75,10 +55,55 @@ Pybind11Extensions: ext_modules=ext_modules ) +PEP 518 requirements (Pip 10+ required) +--------------------------------------- + +If you use PEP518's ``pyproject.toml`` file: + +.. code-block:: toml + + [build-system] + requires = ["setuptools", "wheel", "pybind11==2.6.0"] + build-backend = "setuptools.build_meta" + +you can ensure that pybind11 is available during the building of your project +(pip 10+ required). An added benefit is that ``pybind11`` will not be required +to install or use your package. + +Classic ``setup_requires`` +-------------------------- + +If you want to support old versions of Pip with the classic +``setup_requires=["pybind11"]`` keyword argument to setup, which triggers a +two-phase ``setup.py`` run, then you will need to use something like this to +ensure the first pass works (which has not yet installed the ``setup_requires`` +packages, since it can't install something it does not know about): + +.. code-block:: python + + try: + from pybind11.setup_helpers import Pybind11Extension + except ImportError: + from setuptools import Extension as Pybind11Extension + + +It doesn't matter that the Extension class (or ``build_ext``) is not the +enhanced subclass for the first pass run; and the second pass will have the +``setup_requires`` requirements. + +Local copy +---------- + +You can also copy ``setup_helpers.py`` directly to your project; it was +designed to be usable standalone, like the old example ``setup.py``. The +``CppExtension`` class is identical to ``Pybind11Extension``, except it does +not include the pybind11 package headers, so you can use it with git submodules +and a specific git version. + .. versionchanged:: 2.6 - Added support file. + Added ``setup_helpers`` file. Building with cppimport ======================== diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 4cc8cbab50..a53f4899c0 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -This module provides a +This module provides helpers for C++11+ projects using pybind11. LICENSE: @@ -80,9 +80,9 @@ def tmp_chdir(): shutil.rmtree(tmpdir) -class Pybind11Extension(distutils.extension.Extension): +class CppExtension(distutils.extension.Extension): """ - Build a Pybind11 Extension module. This automatically adds the recommended + Build a C++11+ Extension module. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you can further modify the options yourself. @@ -129,17 +129,6 @@ def __init__(self, *args, **kwargs): # Have to use the accessor manually to support Python 2 distutils CppExtension.cxx_std.__set__(self, cxx_std) - # If using setup_requires, this fails the first time - that's okay - try: - import pybind11 - - pyinc = pybind11.get_include() - - if pyinc not in self.include_dirs: - self.include_dirs.append(pyinc) - except ImportError: - pass - if WIN: self._add_cflags("/EHsc", "/bigobj") else: @@ -185,6 +174,30 @@ def cxx_std(self, level): self.extra_compile_args.append("-Wno-deprecated-register") +class Pybind11Extension(CppExtension): + """ + A pybind11 Extension subclass. Includes the header directory from the + package. + """ + + def __init__(self, *args, **kwargs): + CppExtension.__init__(self, *args, **kwargs) + + # If using setup_requires, this fails the first time - that's okay + try: + import pybind11 + + pyinc = pybind11.get_include() + + if pyinc not in self.include_dirs: + self.include_dirs.append(pyinc) + except ImportError: + pass + + +Pybind11Extension.__doc__ += CppExtension.__doc__ + + # cf http://bugs.python.org/issue26689 def has_flag(compiler, flag): """ From c6bf4e167e2e9b7b7ef7e11a81418adfe05eccd7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 1 Sep 2020 09:41:15 -0400 Subject: [PATCH 11/28] feat: global as an extra (@YannickJadoul's suggestion) --- docs/changelog.rst | 2 +- docs/upgrade.rst | 9 +++++---- setup.cfg | 1 - tools/setup_main.py | 4 +++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index eb2f63a68d..0835b9f64f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,7 +47,7 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. * CMake configuration files are now included in the Python package. Use ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get the CMake directory, or include the site-packages location in your - ``CMAKE_MODULE_PATH``. Or you can use the new ``pybind11-global`` module, + ``CMAKE_MODULE_PATH``. Or you can use the new ``pybind11[global]`` option, which installs the includes and headers into your base environment in the standard location. diff --git a/docs/upgrade.rst b/docs/upgrade.rst index f1148ac07f..f37c872d49 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -26,10 +26,11 @@ be replaced by ``PYBIND11_OVERRIDE*`` and ``get_override``. In the future, the old macros may be deprecated and removed. The ``pybind11`` package on PyPI no longer fills the wheel "headers" slot - if -you were using the headers from this slot, they are available in the -``pybind11-global`` package. (Most users will be unaffected, as the -``pybind11/include`` location is reported by ``python -m pybind11 --includes`` -and ``pybind11.get_include()`` is still correct and has not changed since 2.5). +you were using the headers from this slot, they are available by requesting the +``global`` extra, that is, ``pip install "pybind11[global]"``. (Most users will +be unaffected, as the ``pybind11/include`` location is reported by ``python -m +pybind11 --includes`` and ``pybind11.get_include()`` is still correct and has +not changed since 2.5). CMake support: -------------- diff --git a/setup.cfg b/setup.cfg index ee0b363735..ca0d59a4d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,6 @@ [metadata] long_description = file: README.md long_description_content_type = text/markdown -version = attr: pybind11.__version__ description = Seamless operability between C++11 and Python author = Wenzel Jakob author_email = "wenzel.jakob@epfl.ch" diff --git a/tools/setup_main.py b/tools/setup_main.py index 649a00f203..6a3abe7311 100644 --- a/tools/setup_main.py +++ b/tools/setup_main.py @@ -5,7 +5,6 @@ from setuptools import setup - setup( name="pybind11", packages=[ @@ -19,6 +18,9 @@ "pybind11.include.pybind11.detail": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], }, + extras_require={ + "global": ["pybind11_global==2.6.0"] + }, entry_points={ "console_scripts": [ "pybind11-config = pybind11.__main__:main", From 9a848fe572f86b121507441ebfd90c5f8f119a99 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 1 Sep 2020 16:22:19 -0400 Subject: [PATCH 12/28] feat: single version location --- CMakeLists.txt | 4 +-- include/pybind11/detail/common.h | 2 +- pybind11/_version.py | 35 ++++++++++++++++++++-- setup.py | 38 +++++++++++++++++++----- tests/extra_python_package/test_files.py | 32 +++++++++++++------- tools/_version.py | 12 ++++++++ tools/setup_global.py | 1 + tools/setup_main.py | 4 ++- 8 files changed, 104 insertions(+), 24 deletions(-) create mode 100644 tools/_version.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 67287d54ba..123abf77d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,10 +26,10 @@ foreach(ver ${pybind11_version_defines}) endif() endforeach() -if(PYBIND11_VERSION_PATCH MATCHES [[([a-zA-Z]+)]]) +if(PYBIND11_VERSION_PATCH MATCHES [[\.([a-zA-Z0-9]+)$]]) set(pybind11_VERSION_TYPE "${CMAKE_MATCH_1}") endif() -string(REGEX MATCH "[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") +string(REGEX MATCH "^[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") project( pybind11 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 0986521ac0..53262f74db 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,7 +11,7 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 6 -#define PYBIND11_VERSION_PATCH dev0 +#define PYBIND11_VERSION_PATCH 1.dev0 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } diff --git a/pybind11/_version.py b/pybind11/_version.py index b548f96dc8..de8fb753b1 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -1,4 +1,35 @@ # -*- coding: utf-8 -*- -version_info = (2, 6, 0, "dev1") -__version__ = ".".join(map(str, version_info)) +import os + + +DIR = os.path.abspath(os.path.dirname(__file__)) + + +def _to_int(s): + try: + return int(s) + except ValueError: + return s + + +# Get the version from the C++ file (in-source) +versions = {} + +common_h = os.path.join( + os.path.dirname(DIR), "include", "pybind11", "detail", "common.h" +) + +with open(common_h) as f: + for line in f: + if "PYBIND11_VERSION_" in line: + _, name, vers = line.split() + versions[name[17:].lower()] = vers + if len(versions) >= 3: + break + else: + msg = "Version number not read correctly from {}: {}".format(common_h, versions) + raise RuntimeError(msg) + +__version__ = "{v[major]}.{v[minor]}.{v[patch]}".format(v=versions) +version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/setup.py b/setup.py index 5b724d20bd..865548d84d 100644 --- a/setup.py +++ b/setup.py @@ -3,28 +3,42 @@ # Setup script for PyPI; use CMakeFile.txt to build extension modules +import collections import contextlib import os import shutil +import string import subprocess import sys import tempfile + +ToFrom = collections.namedtuple("ToFrom", ("to", "src")) + # PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers # files, and the sys.prefix files (CMake and headers). alt_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) -setup_py = "tools/setup_global.py" if alt_sdist else "tools/setup_main.py" -pyproject_toml = "tools/pyproject.toml" +version_py = ToFrom("pybind11/_version.py", "tools/_version.py") +pyproject_toml = ToFrom("pyproject.toml", "tools/pyproject.toml") +setup_py = ToFrom( + "setup.py", "tools/setup_global.py" if alt_sdist else "tools/setup_main.py" +) # In a PEP 518 build, this will be in its own environment, so it will not # create extra files in the source DIR = os.path.abspath(os.path.dirname(__file__)) +with open(version_py.to) as f: + loc = {"__file__": version_py.to} + code = compile(f.read(), version_py.to, "exec") + exec(code, loc) + version = loc["__version__"] + @contextlib.contextmanager -def monkey_patch_file(input_file, replacement_file): +def monkey_patch_file(input_file, replacement_file, **template): "Allow a file to be temporarily replaced" inp_file = os.path.abspath(os.path.join(DIR, input_file)) rep_file = os.path.abspath(os.path.join(DIR, replacement_file)) @@ -33,6 +47,13 @@ def monkey_patch_file(input_file, replacement_file): contents = f.read() with open(rep_file, "rb") as f: replacement = f.read() + + if template: + # We convert from/to binary, so that newline style is preserved. + replacement = replacement.decode() + replacement = string.Template(replacement).substitute(template) + replacement = replacement.encode() + try: with open(inp_file, "wb") as f: f.write(replacement) @@ -73,8 +94,9 @@ def remove_output(*sources): subprocess.check_call(cmd, **cmake_opts) subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) - with monkey_patch_file("pyproject.toml", pyproject_toml): - with monkey_patch_file("setup.py", setup_py): - with open(setup_py) as f: - code = compile(f.read(), setup_py, "exec") - exec(code, globals(), {}) + with monkey_patch_file(*pyproject_toml, version=version): + with monkey_patch_file(*setup_py, version=version): + with monkey_patch_file(*version_py, version=version): + with open(setup_py.to) as f: + code = compile(f.read(), setup_py.to, "exec") + exec(code, {}) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 32d2cbdf8d..6e0f6f552d 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- +import contextlib +import os +import string import subprocess import sys -import os import tarfile -import contextlib import zipfile # These tests must be run explicitly @@ -110,12 +111,11 @@ def test_build_sdist(monkeypatch, tmpdir): if hasattr(out, "decode"): out = out.decode() - assert out.count("copying") == 49 - (sdist,) = tmpdir.visit("*.tar") with tarfile.open(str(sdist)) as tar: start = tar.getnames()[0] + "/" + version = start[9:-1] simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) with contextlib.closing( @@ -132,13 +132,20 @@ def test_build_sdist(monkeypatch, tmpdir): files |= sdist_files files |= set("pybind11{}".format(n) for n in local_sdist_files) files.add("pybind11.egg-info/entry_points.txt") + files.add("pybind11.egg-info/requires.txt") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py"), "rb") as f: - assert setup_py == f.read() + contents = ( + string.Template(f.read().decode()).substitute(version=version).encode() + ) + assert setup_py == contents with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: - assert pyproject_toml == f.read() + contents = ( + string.Template(f.read().decode()).substitute(version=version).encode() + ) + assert pyproject_toml == contents def test_build_alt_dist(monkeypatch, tmpdir): @@ -159,12 +166,11 @@ def test_build_alt_dist(monkeypatch, tmpdir): if hasattr(out, "decode"): out = out.decode() - assert out.count("copying") == 48 - (sdist,) = tmpdir.visit("*.tar") with tarfile.open(str(sdist)) as tar: start = tar.getnames()[0] + "/" + version = start[16:-1] simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) with contextlib.closing( @@ -183,10 +189,16 @@ def test_build_alt_dist(monkeypatch, tmpdir): assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_global.py"), "rb") as f: - assert setup_py == f.read() + contents = ( + string.Template(f.read().decode()).substitute(version=version).encode() + ) + assert setup_py == contents with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: - assert pyproject_toml == f.read() + contents = ( + string.Template(f.read().decode()).substitute(version=version).encode() + ) + assert pyproject_toml == contents def tests_build_wheel(monkeypatch, tmpdir): diff --git a/tools/_version.py b/tools/_version.py new file mode 100644 index 0000000000..8a5dea18ec --- /dev/null +++ b/tools/_version.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + + +def _to_int(s): + try: + return int(s) + except ValueError: + return s + + +__version__ = "$version" +version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/tools/setup_global.py b/tools/setup_global.py index b4a7170a93..5fb604ae67 100644 --- a/tools/setup_global.py +++ b/tools/setup_global.py @@ -38,6 +38,7 @@ def run(self): setup( name="pybind11_global", + version="$version", packages=[], headers=headers, cmdclass={"install_headers": InstallHeadersNested}, diff --git a/tools/setup_main.py b/tools/setup_main.py index 6a3abe7311..3bee529e3a 100644 --- a/tools/setup_main.py +++ b/tools/setup_main.py @@ -7,6 +7,8 @@ setup( name="pybind11", + version="$version", + download_url='https://github.com/pybind/pybind11/tarball/v$version', packages=[ "pybind11", "pybind11.include.pybind11", @@ -19,7 +21,7 @@ "pybind11.share.cmake.pybind11": ["*.cmake"], }, extras_require={ - "global": ["pybind11_global==2.6.0"] + "global": ["pybind11_global==$version"] }, entry_points={ "console_scripts": [ From d81911112d60945c61321a2f5dca1a6a1ce076ae Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 2 Sep 2020 18:41:11 -0400 Subject: [PATCH 13/28] fix: remove the requirement that setuptools must be imported first --- docs/compiling.rst | 4 ++-- pybind11/setup_helpers.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 5b9c888df1..8ed0b9b1ac 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -18,7 +18,7 @@ An example of a ``setup.py`` using pybind11's helpers: .. code-block:: python - from setuptools import setup # Always import setuptools first + from setuptools import setup from pybind11.setup_helpers import Pybind11Extension ext_modules = [ @@ -39,7 +39,7 @@ Pybind11Extensions: .. code-block:: python - from setuptools import setup # Always import first + from setuptools import setup from pybind11.setup_helpers import Pybind11Extension, build_ext ext_modules = [ diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index a53f4899c0..b3352ec8c1 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -41,9 +41,14 @@ import threading import warnings +try: + from setuptools.command.build_ext import build_ext as _build_ext + from setuptools import Extension as _Extension +except ImportError: + from distutils.command.build_ext import build_ext as _build_ext + from distutils.extension import Extension as _Extension + import distutils.errors -import distutils.command.build_ext -import distutils.extension WIN = sys.platform.startswith("win32") @@ -80,7 +85,7 @@ def tmp_chdir(): shutil.rmtree(tmpdir) -class CppExtension(distutils.extension.Extension): +class CppExtension(_Extension): """ Build a C++11+ Extension module. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you can further @@ -124,7 +129,7 @@ def __init__(self, *args, **kwargs): # Can't use super here because distutils has old-style classes in # Python 2! - distutils.extension.Extension.__init__(self, *args, **kwargs) + _Extension.__init__(self, *args, **kwargs) # Have to use the accessor manually to support Python 2 distutils CppExtension.cxx_std.__set__(self, cxx_std) @@ -252,7 +257,7 @@ def auto_cpp_level(compiler): raise RuntimeError(msg) -class build_ext(distutils.command.build_ext.build_ext): # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. @@ -270,4 +275,4 @@ def build_extensions(self): # Python 2 doesn't allow super here, since distutils uses old-style # classes! - distutils.command.build_ext.build_ext.build_extensions(self) + _build_ext.build_extensions(self) From 547f15285b7d59d4ab8cfa3b603aa6aea27f227b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 2 Sep 2020 21:04:23 -0400 Subject: [PATCH 14/28] fix: some review points from @wjacob --- .github/workflows/ci.yml | 27 ++++++++++++++- .github/workflows/configure.yml | 11 ++++++ .github/workflows/format.yml | 4 +++ .pre-commit-config.yaml | 23 +++++++++++++ docs/changelog.rst | 13 +++---- docs/compiling.rst | 61 +++++++++++++++++++++++++-------- 6 files changed, 118 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5a5d76821..79ba2e6595 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,8 @@ on: - v* jobs: + # This is the "main" test suite, which tests a large number of different + # versions of default compilers and Python versions in GitHub Actions. standard: strategy: fail-fast: false @@ -23,6 +25,12 @@ jobs: - pypy2 - pypy3 + # Items in here will either be added to the build matrix (if not + # present), or add new keys to an existing matrix element if all the + # existing keys match. + # + # We support three optional keys: args (both build), args1 (first + # build), and args2 (second build). include: - runs-on: ubuntu-latest python: 3.6 @@ -52,6 +60,7 @@ jobs: args: > -DPYBIND11_FINDPYTHON=ON + # These items will be removed from the build matrix, keys must match. exclude: # Currently 32bit only, and we build 64bit - runs-on: windows-latest @@ -102,10 +111,11 @@ jobs: - name: Prepare env run: python -m pip install -r tests/requirements.txt --prefer-binary - - name: Setup annotations + - name: Setup annotations on Linux if: runner.os == 'Linux' run: python -m pip install pytest-github-actions-annotate-failures + # First build - C++11 mode and inplace - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . @@ -130,6 +140,7 @@ jobs: - name: Clean directory run: git clean -fdx + # Second build - C++17 mode and in a build directory - name: Configure ${{ matrix.args2 }} run: > cmake -S . -B build2 @@ -158,16 +169,20 @@ jobs: with: arch: x64 + # This makes two environment variables available in the following step(s) - name: Set Windows 🐍 2.7 environment variables if: matrix.python == 2.7 && runner.os == 'Windows' run: | echo "::set-env name=DISTUTILS_USE_SDK::1" echo "::set-env name=MSSdk::1" + # This makes sure the setup_helpers module can build packages using + # setuptools - name: Setuptools helpers test run: pytest tests/extra_setuptools + # Testing on clang using the sikleh clang docker images clang: runs-on: ubuntu-latest strategy: @@ -212,6 +227,7 @@ jobs: run: cmake --build build --target test_cmake_build + # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04" @@ -234,6 +250,7 @@ jobs: run: cmake --build build --target pytest + # Testing CentOS 8 + PGI compilers centos-nvhpc8: runs-on: ubuntu-latest name: "🐍 3 • CentOS8 / PGI 20.7 • x64" @@ -272,6 +289,8 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + + # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: runs-on: ubuntu-latest name: "🐍 3 • CentOS7 / PGI 20.7 • x64" @@ -319,6 +338,7 @@ jobs: - name: Interface test run: cmake3 --build build --target test_cmake_build + # Testing on GCC using the GCC docker images (only recent images supported) gcc: runs-on: ubuntu-latest strategy: @@ -367,6 +387,8 @@ jobs: run: cmake --build build --target test_cmake_build + # Testing on CentOS (manylinux uses a centos base, and this is an easy way + # to get GCC 4.8, which is the manylinux1 compiler). centos: runs-on: ubuntu-latest strategy: @@ -414,6 +436,7 @@ jobs: run: cmake --build build --target test_cmake_build + # This tests an "install" with the CMake tools install-classic: name: "🐍 3.5 • Debian • x86 • Install" runs-on: ubuntu-latest @@ -456,6 +479,8 @@ jobs: working-directory: /build-tests + # This verifies that the documentation is not horribly broken, and does a + # basic sanity check on the SDist. doxygen: name: "Documentation build test" runs-on: ubuntu-latest diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 07122fbc6d..ce5d7d0de3 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -10,6 +10,8 @@ on: - v* jobs: + # This tests various versions of CMake in various combinations, to make sure + # the configure step passes. cmake: strategy: fail-fast: false @@ -50,11 +52,14 @@ jobs: - name: Prepare env run: python -m pip install -r tests/requirements.txt + # An action for adding a specific version of CMake: + # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} uses: jwlawson/actions-setup-cmake@v1.3 with: cmake-version: ${{ matrix.cmake }} + # These steps use a directory with a space in it intentionally - name: Make build directories run: mkdir "build dir" @@ -67,6 +72,7 @@ jobs: -DDOWNLOAD_CATCH=ON -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + # Only build and test if this was manually triggered in the GitHub UI - name: Build working-directory: build dir if: github.event_name == 'workflow_dispatch' @@ -77,6 +83,9 @@ jobs: if: github.event_name == 'workflow_dispatch' run: cmake --build . --config Release --target check + # This builds the sdists and wheels and makes sure the files are exactly as + # expected. Using Windows and Python 2.7, since that is often the most + # challenging matrix element. test-packaging: name: 🐍 2.7 • 📦 tests • windows-latest runs-on: windows-latest @@ -96,6 +105,8 @@ jobs: run: pytest tests/extra_python_package/ + # This runs the packaging tests and also builds and saves the packages as + # artifacts. packaging: name: 🐍 3.8 • 📦 & 📦 tests • ubuntu-latest runs-on: ubuntu-latest diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 49613ba5f5..28cfeb9b7d 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -1,3 +1,6 @@ +# This is a format job. Pre-commit has a first-party GitHub action, so we use +# that: https://github.com/pre-commit/action + name: Format on: @@ -18,6 +21,7 @@ jobs: - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.0 with: + # Slow hooks are marked with manual - slow is okay here, run them too extra_args: --hook-stage manual clang-tidy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ebd310fe0..6bb36da56d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,18 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://github.com/pre-commit/pre-commit repos: +# Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: @@ -14,17 +28,21 @@ repos: - id: trailing-whitespace - id: fix-encoding-pragma +# Black natively supports pre-commit - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black + # Not all Python files are Blacked, yet files: ^(setup.py|pybind11|tests/extra) +# Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.9 hooks: - id: remove-tabs +# Flake8 also supports pre-commit natively (same author) - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: @@ -32,6 +50,8 @@ repos: additional_dependencies: [flake8-bugbear, pep8-naming] exclude: ^(docs/.*|tools/.*)$ +# CMake formatting (first party, but in a separate repo due to author's desire +# to use PyPI for pinning) - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.11 hooks: @@ -40,13 +60,16 @@ repos: types: [file] files: (\.cmake|CMakeLists.txt)(.in)?$ +# Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest rev: "0.42" hooks: - id: check-manifest + # This is a slow hook, so only run this if --hook-stage manual is passed stages: [manual] additional_dependencies: [cmake, ninja] +# The original pybind11 checks for a few C++ style items - repo: local hooks: - id: disallow-caps diff --git a/docs/changelog.rst b/docs/changelog.rst index 0835b9f64f..cf00e0d84c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,16 +40,17 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. * The Python package was reworked to be more powerful and useful. `#2433 `_ - * A new ``pybind11.setup_helpers`` module provides utilities to easily use + * :ref:`build-setuptools` is easier thanks to a new + ``pybind11.setup_helpers`` module, which provides utilities to easily use setuptools with pybind11, and can be used via PEP 518 or by directly - copying ``setup_helpers.py`` into your project. + copying ``setup_helpers.py`` into your project . * CMake configuration files are now included in the Python package. Use ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get - the CMake directory, or include the site-packages location in your - ``CMAKE_MODULE_PATH``. Or you can use the new ``pybind11[global]`` option, - which installs the includes and headers into your base environment in the - standard location. + the directory with the CMake configuration files, or include the + site-packages location in your ``CMAKE_MODULE_PATH``. Or you can use the + new ``pybind11[global]`` option, which installs the CMake files and headers + into your base environment in the standard location. * Minimum CMake required increased to 3.4. `#2338 `_ and diff --git a/docs/compiling.rst b/docs/compiling.rst index 8ed0b9b1ac..bf30cf1930 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -3,6 +3,8 @@ Build systems ############# +.. _build-setuptools: + Building with setuptools ======================== @@ -13,7 +15,18 @@ the [python_example]_ repository. .. [python_example] https://github.com/pybind/python_example -A helper file is provided with pybind11 that can simplify usage with setuptools +A helper file is provided with pybind11 that can simplify usage with setuptools. + +To use pybind11 inside your ``setup.py``, you have to have some system to +ensure that ``pybind11`` is installed when you build your package. There are +four possible ways to do this, and pybind11 supports all four: You can ask all +users to install pybind11 beforehand (bad), you can use +:ref:`setup_helpers-pep518` (good, but very new and requires Pip 10), +:ref:`setup_helpers-setup_requires` (discouraged py Python packagers now that +PEP 518 is available, but still works everywhere), or you can +:ref:`setup_helpers-copy-manually` (always works but you have to manually sync +your copy to get updates). + An example of a ``setup.py`` using pybind11's helpers: .. code-block:: python @@ -24,7 +37,7 @@ An example of a ``setup.py`` using pybind11's helpers: ext_modules = [ Pybind11Extension( "python_example", - sorted(["src/main.cpp"]), + ["src/main.cpp"], ), ] @@ -35,7 +48,7 @@ An example of a ``setup.py`` using pybind11's helpers: If you want to do an automatic search for the highest supported C++ standard, that is supported via a ``build_ext`` command override; it will only affect -Pybind11Extensions: +``Pybind11Extensions``: .. code-block:: python @@ -45,7 +58,7 @@ Pybind11Extensions: ext_modules = [ Pybind11Extension( "python_example", - sorted(["src/main.cpp"]), + ["src/main.cpp"], ), ] @@ -55,10 +68,25 @@ Pybind11Extensions: ext_modules=ext_modules ) +.. _setup_helpers-pep518: + PEP 518 requirements (Pip 10+ required) --------------------------------------- -If you use PEP518's ``pyproject.toml`` file: +If you use `PEP 518's `_ +``pyproject.toml`` file, you can ensure that pybind11 is available during the +building of your project. When this file exists, Pip will make a new virtual +environment, download just the packages listed here in ``requires=``, and build +a wheel (binary Python package). It will then throw away the environment, and +install your wheel. + +The main drawback is that Pip 10+ is required to build from source; older +versions completely ignore this file. If you distribute wheels, using something +like `cibuildwheel `_, remember that +``setup.py`` and ``pyproject.toml`` are not even contained in the wheel, so +this high Pip requirement is only for source builds. + +Your ``pyproject.toml`` file will likely look something like this: .. code-block:: toml @@ -66,9 +94,8 @@ If you use PEP518's ``pyproject.toml`` file: requires = ["setuptools", "wheel", "pybind11==2.6.0"] build-backend = "setuptools.build_meta" -you can ensure that pybind11 is available during the building of your project -(pip 10+ required). An added benefit is that ``pybind11`` will not be required -to install or use your package. + +.. _setup_helpers-setup_requires: Classic ``setup_requires`` -------------------------- @@ -87,18 +114,24 @@ packages, since it can't install something it does not know about): from setuptools import Extension as Pybind11Extension -It doesn't matter that the Extension class (or ``build_ext``) is not the -enhanced subclass for the first pass run; and the second pass will have the -``setup_requires`` requirements. +It doesn't matter that the Extension class is not the enhanced subclass for the +first pass run; and the second pass will have the ``setup_requires`` +requirements. + +This is obviously more of a hack than the PEP 518 method, but it supports +ancient versions of Pip. + +.. _setup_helpers-copy-manually: -Local copy ----------- +Copy manually +------------- You can also copy ``setup_helpers.py`` directly to your project; it was designed to be usable standalone, like the old example ``setup.py``. The ``CppExtension`` class is identical to ``Pybind11Extension``, except it does not include the pybind11 package headers, so you can use it with git submodules -and a specific git version. +and a specific git version. If you use this, you will need to import from a +local file in ``setup.py`` and ensure the helper file is part of your MANIFEST. .. versionchanged:: 2.6 From 43ad95495f7a5077000cc3889333d7eead410560 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 2 Sep 2020 22:39:32 -0400 Subject: [PATCH 15/28] fix: use .in, add procedure to docs --- .github/CONTRIBUTING.md | 37 +++++++++++++++++++ setup.py | 4 +- tools/{_version.py => _version.py.in} | 0 tools/{setup_global.py => setup_global.py.in} | 0 tools/{setup_main.py => setup_main.py.in} | 0 5 files changed, 39 insertions(+), 2 deletions(-) rename tools/{_version.py => _version.py.in} (100%) rename tools/{setup_global.py => setup_global.py.in} (100%) rename tools/{setup_main.py => setup_main.py.in} (100%) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4cdd7aab02..deb71277a6 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -210,6 +210,43 @@ cmake -S pybind11/ -B build cmake --build build ``` + +### Explanation of the SDist/wheel building design + +In order to support CMake output files (`pybind11Config.cmake` and helper +files), the `setup.py` _in the source_ has a few tricks to make the simplest +possible SDists and wheels. The build procedure is as follows: + +#### 1. Building from the source directory + +When you invoke any `setup.py` command from the source directory, including +`pip wheel .` and `pip install .`, you will activate a full source build. This is made of the +following steps: + +1. If the tool is PEP 518 compliant, it will create a temporary virtual + environment and install the build requirements (mostly CMake) into it. (if + you are not on Windows, macOS, ora a manylinux compliant system, you can + disable this with `--no-build-isolation` as long as you have CMake 3.15+ + installed) +2. The environment variable `PYBIND11_GLOBAL_DIST` is checked - if it is set + and truthy, this will be make the accessory `pybind11-global` package, + instead of the normal `pybind11` package. This package is used for + installing the "global" headers and CMake files, using `pybind11[global]`. +2. `setup.py` reads the version from `includes/pybind11/detail/common.h`. +3. CMake is run with `-DCMAKE_INSTALL_PREIFX=pybind11`. Since the CMake install + procedure uses only relative paths and is identical on all platforms, these + files are valid as long as they stay in the correct relative position to the + includes. `pybind11/share/cmake/pybind11` has the CMake files, and + `pybind11/include` has the includes. The build directory is discarded. +4. Three files are temporarily substituted in: `tools/setup_main.py.in`, + `tools/pyproject.toml`, and `tools/_version.py.in`. The previously read + version is included in these files during the copy. +5. The package (such as the SDist) is created using the setup function in the + new setup.py. Since the SDist procedure just copies existing files. +6. Context managers clean up all changes (even if an error is thrown). + + + [pre-commit]: https://pre-commit.com [pybind11.readthedocs.org]: http://pybind11.readthedocs.org/en/latest [issue tracker]: https://github.com/pybind/pybind11/issues diff --git a/setup.py b/setup.py index 865548d84d..623c8f072c 100644 --- a/setup.py +++ b/setup.py @@ -19,10 +19,10 @@ # files, and the sys.prefix files (CMake and headers). alt_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) -version_py = ToFrom("pybind11/_version.py", "tools/_version.py") +version_py = ToFrom("pybind11/_version.py", "tools/_version.py.in") pyproject_toml = ToFrom("pyproject.toml", "tools/pyproject.toml") setup_py = ToFrom( - "setup.py", "tools/setup_global.py" if alt_sdist else "tools/setup_main.py" + "setup.py", "tools/setup_global.py.in" if alt_sdist else "tools/setup_main.py.in" ) # In a PEP 518 build, this will be in its own environment, so it will not diff --git a/tools/_version.py b/tools/_version.py.in similarity index 100% rename from tools/_version.py rename to tools/_version.py.in diff --git a/tools/setup_global.py b/tools/setup_global.py.in similarity index 100% rename from tools/setup_global.py rename to tools/setup_global.py.in diff --git a/tools/setup_main.py b/tools/setup_main.py.in similarity index 100% rename from tools/setup_main.py rename to tools/setup_main.py.in From ad97858bb047ff9cf66324359571020d54e26ee8 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 2 Sep 2020 23:48:48 -0400 Subject: [PATCH 16/28] refactor: avoid monkeypatch copy --- .github/CONTRIBUTING.md | 22 ++++--- setup.py | 81 ++++++++++++------------ tests/extra_python_package/test_files.py | 34 ++++++++-- tools/setup_global.py.in | 4 +- tools/setup_main.py.in | 6 +- 5 files changed, 91 insertions(+), 56 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index deb71277a6..5515aa1af7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -220,8 +220,8 @@ possible SDists and wheels. The build procedure is as follows: #### 1. Building from the source directory When you invoke any `setup.py` command from the source directory, including -`pip wheel .` and `pip install .`, you will activate a full source build. This is made of the -following steps: +`pip wheel .` and `pip install .`, you will activate a full source build. This +is made of the following steps: 1. If the tool is PEP 518 compliant, it will create a temporary virtual environment and install the build requirements (mostly CMake) into it. (if @@ -238,12 +238,18 @@ following steps: files are valid as long as they stay in the correct relative position to the includes. `pybind11/share/cmake/pybind11` has the CMake files, and `pybind11/include` has the includes. The build directory is discarded. -4. Three files are temporarily substituted in: `tools/setup_main.py.in`, - `tools/pyproject.toml`, and `tools/_version.py.in`. The previously read - version is included in these files during the copy. -5. The package (such as the SDist) is created using the setup function in the - new setup.py. Since the SDist procedure just copies existing files. -6. Context managers clean up all changes (even if an error is thrown). +4. Three files are placed in the SDist: `tools/setup_main.py.in`, + `tools/pyproject.toml`, and `tools/_version.py.in`. +5. The package is created using the setup function in the new setup.py. +6. A context manager cleans up the temporary CMake install directory (even if + an error is thrown). + +### 2. Building from SDist + +Since the SDist has the rendered template files in `tools` along with the +includes and CMake files in the correct locations, the builds are completely +trivial and simple. No extra requirements are required. The version is baked +in, so there is not even a lookup performance penalty. diff --git a/setup.py b/setup.py index 623c8f072c..226b17cd49 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ # Setup script for PyPI; use CMakeFile.txt to build extension modules -import collections import contextlib import os import shutil @@ -12,57 +11,61 @@ import sys import tempfile +import setuptools.command.sdist -ToFrom = collections.namedtuple("ToFrom", ("to", "src")) +DIR = os.path.abspath(os.path.dirname(__file__)) # PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers # files, and the sys.prefix files (CMake and headers). alt_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) -version_py = ToFrom("pybind11/_version.py", "tools/_version.py.in") -pyproject_toml = ToFrom("pyproject.toml", "tools/pyproject.toml") -setup_py = ToFrom( - "setup.py", "tools/setup_global.py.in" if alt_sdist else "tools/setup_main.py.in" -) -# In a PEP 518 build, this will be in its own environment, so it will not -# create extra files in the source +version_py = "pybind11/_version.py" +setup_py = "tools/setup_global.py.in" if alt_sdist else "tools/setup_main.py.in" +extra_cmd = 'cmdclass["sdist"] = SDist\n' -DIR = os.path.abspath(os.path.dirname(__file__)) +to_src = ( + (version_py, "tools/_version.py.in"), + ("pyproject.toml", "tools/pyproject.toml"), + ("setup.py", setup_py), +) -with open(version_py.to) as f: - loc = {"__file__": version_py.to} - code = compile(f.read(), version_py.to, "exec") + +with open(version_py) as f: + loc = {"__file__": version_py} + code = compile(f.read(), version_py, "exec") exec(code, loc) version = loc["__version__"] -@contextlib.contextmanager -def monkey_patch_file(input_file, replacement_file, **template): - "Allow a file to be temporarily replaced" - inp_file = os.path.abspath(os.path.join(DIR, input_file)) - rep_file = os.path.abspath(os.path.join(DIR, replacement_file)) - - with open(inp_file, "rb") as f: +def get_and_replace(filename, binary=False, **opts): + with open(filename, "rb" if binary else "r") as f: contents = f.read() - with open(rep_file, "rb") as f: - replacement = f.read() + # Replacement has to be done on text in Python 3 (both work in Python 2) + if binary: + return string.Template(contents.decode()).substitute(opts).encode() + else: + return string.Template(contents).substitute(opts) - if template: - # We convert from/to binary, so that newline style is preserved. - replacement = replacement.decode() - replacement = string.Template(replacement).substitute(template) - replacement = replacement.encode() - try: - with open(inp_file, "wb") as f: - f.write(replacement) - yield - finally: - with open(inp_file, "wb") as f: - f.write(contents) +# Use our input files instead when making the SDist (and anything that depends +# on it, like a wheel) +class SDist(setuptools.command.sdist.sdist): + def make_release_tree(self, base_dir, files): + setuptools.command.sdist.sdist.make_release_tree(self, base_dir, files) + + for to, src in to_src: + txt = get_and_replace(src, binary=True, version=version, extra_cmd="") + + dest = os.path.join(base_dir, to) + + # This is normally linked, so unlink before writing! + os.unlink(dest) + with open(dest, "wb") as f: + f.write(txt) +# Backport from Python 3 @contextlib.contextmanager def TemporaryDirectory(): # noqa: N802 "Prepare a temporary directory, cleanup when done" @@ -73,6 +76,7 @@ def TemporaryDirectory(): # noqa: N802 shutil.rmtree(tmpdir) +# Remove the CMake install directory when done @contextlib.contextmanager def remove_output(*sources): try: @@ -94,9 +98,6 @@ def remove_output(*sources): subprocess.check_call(cmd, **cmake_opts) subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) - with monkey_patch_file(*pyproject_toml, version=version): - with monkey_patch_file(*setup_py, version=version): - with monkey_patch_file(*version_py, version=version): - with open(setup_py.to) as f: - code = compile(f.read(), setup_py.to, "exec") - exec(code, {}) + txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) + code = compile(txt, setup_py, "exec") + exec(code, {"SDist": SDist}) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 6e0f6f552d..ad3fa16e40 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -123,6 +123,11 @@ def test_build_sdist(monkeypatch, tmpdir): ) as f: setup_py = f.read() + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pybind11/_version.py")) + ) as f: + version_py = f.read() + with contextlib.closing( tar.extractfile(tar.getmember(start + "pyproject.toml")) ) as f: @@ -135,16 +140,22 @@ def test_build_sdist(monkeypatch, tmpdir): files.add("pybind11.egg-info/requires.txt") assert simpler == files - with open(os.path.join(MAIN_DIR, "tools", "setup_main.py"), "rb") as f: + with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: contents = ( - string.Template(f.read().decode()).substitute(version=version).encode() + string.Template(f.read().decode()) + .substitute(version=version, extra_cmd="") + .encode() ) assert setup_py == contents - with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + with open(os.path.join(MAIN_DIR, "tools", "_version.py.in"), "rb") as f: contents = ( string.Template(f.read().decode()).substitute(version=version).encode() ) + assert version_py == contents + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + contents = f.read() assert pyproject_toml == contents @@ -178,6 +189,11 @@ def test_build_alt_dist(monkeypatch, tmpdir): ) as f: setup_py = f.read() + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pybind11/_version.py")) + ) as f: + version_py = f.read() + with contextlib.closing( tar.extractfile(tar.getmember(start + "pyproject.toml")) ) as f: @@ -188,16 +204,22 @@ def test_build_alt_dist(monkeypatch, tmpdir): files |= set("pybind11_global{}".format(n) for n in local_sdist_files) assert simpler == files - with open(os.path.join(MAIN_DIR, "tools", "setup_global.py"), "rb") as f: + with open(os.path.join(MAIN_DIR, "tools", "setup_global.py.in"), "rb") as f: contents = ( - string.Template(f.read().decode()).substitute(version=version).encode() + string.Template(f.read().decode()) + .substitute(version=version, extra_cmd="") + .encode() ) assert setup_py == contents - with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + with open(os.path.join(MAIN_DIR, "tools", "_version.py.in"), "rb") as f: contents = ( string.Template(f.read().decode()).substitute(version=version).encode() ) + assert version_py == contents + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + contents = f.read() assert pyproject_toml == contents diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in index 5fb604ae67..b8ea101a36 100644 --- a/tools/setup_global.py.in +++ b/tools/setup_global.py.in @@ -35,16 +35,18 @@ detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") headers = main_headers + detail_headers +cmdclass = {"install_headers": InstallHeadersNested} +$extra_cmd setup( name="pybind11_global", version="$version", packages=[], headers=headers, - cmdclass={"install_headers": InstallHeadersNested}, data_files=[ ("share/cmake/pybind11", cmake_files), ("include/pybind11", main_headers), ("include/pybind11/detail", detail_headers), ], + cmdclass=cmdclass, ) diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 3bee529e3a..cd4f11c5bd 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -5,6 +5,9 @@ from setuptools import setup +cmdclass = {} +$extra_cmd + setup( name="pybind11", version="$version", @@ -27,5 +30,6 @@ setup( "console_scripts": [ "pybind11-config = pybind11.__main__:main", ] - } + }, + cmdclass=cmdclass ) From e907075ffcc7092592527f656de880bfe42c2d62 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 4 Sep 2020 01:08:02 -0400 Subject: [PATCH 17/28] docs: minor typos corrected --- .github/CONTRIBUTING.md | 6 +++--- .github/workflows/ci.yml | 4 +++- .github/workflows/configure.yml | 4 ++-- .pre-commit-config.yaml | 5 +++-- docs/changelog.rst | 6 +++--- docs/compiling.rst | 4 ++-- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5515aa1af7..567e248f83 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -225,7 +225,7 @@ is made of the following steps: 1. If the tool is PEP 518 compliant, it will create a temporary virtual environment and install the build requirements (mostly CMake) into it. (if - you are not on Windows, macOS, ora a manylinux compliant system, you can + you are not on Windows, macOS, or a manylinux compliant system, you can disable this with `--no-build-isolation` as long as you have CMake 3.15+ installed) 2. The environment variable `PYBIND11_GLOBAL_DIST` is checked - if it is set @@ -240,7 +240,7 @@ is made of the following steps: `pybind11/include` has the includes. The build directory is discarded. 4. Three files are placed in the SDist: `tools/setup_main.py.in`, `tools/pyproject.toml`, and `tools/_version.py.in`. -5. The package is created using the setup function in the new setup.py. +5. The package is created using the setup function in the `tools/setup_*.py`. 6. A context manager cleans up the temporary CMake install directory (even if an error is thrown). @@ -249,7 +249,7 @@ is made of the following steps: Since the SDist has the rendered template files in `tools` along with the includes and CMake files in the correct locations, the builds are completely trivial and simple. No extra requirements are required. The version is baked -in, so there is not even a lookup performance penalty. +in, so there is not even a lookup performance penalty / point of failure. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79ba2e6595..1749d07f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,8 @@ jobs: - name: Interface test run: cmake --build build2 --target test_cmake_build + # Eventually Microsoft might have an action for setting up + # MSVC, but for now, this action works: - name: Prepare compiler environment for Windows 🐍 2.7 if: matrix.python == 2.7 && runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -182,7 +184,7 @@ jobs: run: pytest tests/extra_setuptools - # Testing on clang using the sikleh clang docker images + # Testing on clang using the excellent silkeh clang docker images clang: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index ce5d7d0de3..eb2e3cc6b8 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -52,8 +52,8 @@ jobs: - name: Prepare env run: python -m pip install -r tests/requirements.txt - # An action for adding a specific version of CMake: - # https://github.com/jwlawson/actions-setup-cmake + # An action for adding a specific version of CMake: + # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} uses: jwlawson/actions-setup-cmake@v1.3 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bb36da56d..26f5e2f843 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,13 +4,14 @@ # # Or: # -# pre-commit install # (runs every time you commit in git) +# pre-commit install # (runs every time you commit in git) # # To update this file: # # pre-commit autoupdate # # See https://github.com/pre-commit/pre-commit + repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks @@ -28,7 +29,7 @@ repos: - id: trailing-whitespace - id: fix-encoding-pragma -# Black natively supports pre-commit +# Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black rev: 20.8b1 hooks: diff --git a/docs/changelog.rst b/docs/changelog.rst index cf00e0d84c..d1ecc10c88 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,9 +41,9 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. `#2433 `_ * :ref:`build-setuptools` is easier thanks to a new - ``pybind11.setup_helpers`` module, which provides utilities to easily use - setuptools with pybind11, and can be used via PEP 518 or by directly - copying ``setup_helpers.py`` into your project . + ``pybind11.setup_helpers`` module, which provides utilities to use + setuptools with pybind11, and can be used via PEP 518, ``setup_requires``, + or by directly copying ``setup_helpers.py`` into your project. * CMake configuration files are now included in the Python package. Use ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get diff --git a/docs/compiling.rst b/docs/compiling.rst index bf30cf1930..f52280c75e 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -22,8 +22,8 @@ ensure that ``pybind11`` is installed when you build your package. There are four possible ways to do this, and pybind11 supports all four: You can ask all users to install pybind11 beforehand (bad), you can use :ref:`setup_helpers-pep518` (good, but very new and requires Pip 10), -:ref:`setup_helpers-setup_requires` (discouraged py Python packagers now that -PEP 518 is available, but still works everywhere), or you can +:ref:`setup_helpers-setup_requires` (discouraged by Python packagers now that +PEP 518 is available, but it still works everywhere), or you can :ref:`setup_helpers-copy-manually` (always works but you have to manually sync your copy to get updates). From 6a0cd0fb696ddd7d3b2e6af8399494cc0260cc82 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 4 Sep 2020 15:18:36 -0400 Subject: [PATCH 18/28] fix: minor points from @YannickJadoul --- include/pybind11/detail/common.h | 2 +- pybind11/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 53262f74db..1f8390fbab 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,7 +11,7 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 6 -#define PYBIND11_VERSION_PATCH 1.dev0 +#define PYBIND11_VERSION_PATCH 0.dev1 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 3553726449..f4d5437836 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -34,7 +34,7 @@ def main(): parser.add_argument( "--cmakedir", action="store_true", - help="Print the CMake module directory, ideal for pybind11_ROOT.", + help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.", ) args = parser.parse_args() if not sys.argv[1:]: From 1768de56da35e0c8d37e7e27e4738c9f9d588089 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 7 Sep 2020 23:42:16 -0400 Subject: [PATCH 19/28] fix: typo on Windows C++ mode --- pybind11/setup_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index b3352ec8c1..7522864a74 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -54,7 +54,7 @@ WIN = sys.platform.startswith("win32") PY2 = sys.version_info[0] < 3 MACOS = sys.platform.startswith("darwin") -STD_TMPL = "/std:cxx{}" if WIN else "-std=c++{}" +STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" # It is recommended to use PEP 518 builds if using this module. However, this From f582e0e960fd9178041068a18f65625ceffbff45 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 7 Sep 2020 23:59:44 -0400 Subject: [PATCH 20/28] fix: MSVC 15 update 3+ have c++14 flag See --- pybind11/setup_helpers.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 7522864a74..2ede2c79ee 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -159,9 +159,13 @@ def cxx_std(self, level): if self._cxx_level: warnings.warn("You cannot safely change the cxx_level after setting it!") + # MSVC 2015 Update 3 and later only have 14 (and later 17) modes + if WIN and level == 11: + level = 14 + self._cxx_level = level - if not level or (WIN and level == 11): + if not level: return self.extra_compile_args.append(STD_TMPL.format(level)) @@ -229,8 +233,7 @@ def has_flag(compiler, flag): def auto_cpp_level(compiler): """ - Return the max supported C++ std level (17, 14, or 11). If neither 17 or 14 - is supported on Windows, "11" is returned. Caches result. + Return the max supported C++ std level (17, 14, or 11). """ global cpp_flag_cache @@ -248,11 +251,6 @@ def auto_cpp_level(compiler): cpp_flag_cache = level return level - if WIN: - with cpp_cache_lock: - cpp_flag_cache = 11 - return 11 - msg = "Unsupported compiler -- at least C++11 support is needed!" raise RuntimeError(msg) From 111ab6f0710ed20554b2bd1fd01dbe7758020216 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 8 Sep 2020 11:05:32 -0400 Subject: [PATCH 21/28] docs: discuss making SDists by hand --- .github/CONTRIBUTING.md | 53 ++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 567e248f83..ca221fbd44 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -210,12 +210,45 @@ cmake -S pybind11/ -B build cmake --build build ``` - ### Explanation of the SDist/wheel building design +> These details are _only_ for building from the Python sources from git. The +> SDists and wheels created do not have any extra requirements at all and are +> completely normal. + In order to support CMake output files (`pybind11Config.cmake` and helper -files), the `setup.py` _in the source_ has a few tricks to make the simplest -possible SDists and wheels. The build procedure is as follows: +files), the `[globals]` option, and a simple version file taken from the C++ +sources, the `setup.py` _in the source_ has a few tricks to make the simplest +possible SDists and wheels. If you want to build the SDist from the GitHub +source, it is best to have Pip 10 on a manylinux1, macOS, or Windows system. +You can then build the SDists, or run any procedure that makes SDists +internally, like making wheels or installing. Since Pip itself +does not have an `sdist` command (it does have `wheel` and `install`), you will +need to use the `pep517` package directly: + +```bash +# Normal package +python3 -m pep517.build -s . + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 -m pep517.build -s . +``` + +If you want to use the classic "direct" usage of `python setup.py`, you will +need CMake 3.15+ and either make or ninja preinstalled (possibly via `pip +install cmake ninja`), since directly running Python on `setup.py` cannot pick +up `pyproject.toml` requirements. As long as you have those two things, though, +everything works as normal: + +```bash +# Normal package +python setup.py sdist + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python setup.py sdist +``` + +A detailed explanation of the build procedure design is as follows: #### 1. Building from the source directory @@ -223,12 +256,12 @@ When you invoke any `setup.py` command from the source directory, including `pip wheel .` and `pip install .`, you will activate a full source build. This is made of the following steps: -1. If the tool is PEP 518 compliant, it will create a temporary virtual - environment and install the build requirements (mostly CMake) into it. (if - you are not on Windows, macOS, or a manylinux compliant system, you can - disable this with `--no-build-isolation` as long as you have CMake 3.15+ - installed) -2. The environment variable `PYBIND11_GLOBAL_DIST` is checked - if it is set +1. If the tool is PEP 518 compliant, like `pep517.build` or Pip 10+, it will + create a temporary virtual environment and install the build requirements + (mostly CMake) into it. (if you are not on Windows, macOS, or a manylinux + compliant system, you can disable this with `--no-build-isolation` as long + as you have CMake 3.15+ installed) +2. The environment variable `PYBIND11_GLOBAL_SDIST` is checked - if it is set and truthy, this will be make the accessory `pybind11-global` package, instead of the normal `pybind11` package. This package is used for installing the "global" headers and CMake files, using `pybind11[global]`. @@ -241,6 +274,8 @@ is made of the following steps: 4. Three files are placed in the SDist: `tools/setup_main.py.in`, `tools/pyproject.toml`, and `tools/_version.py.in`. 5. The package is created using the setup function in the `tools/setup_*.py`. + `setup_main.py` fills in Python packages, and `setup_global.py` fills in + only the data/header slots. 6. A context manager cleans up the temporary CMake install directory (even if an error is thrown). From 6eeda0094bc1cd412a155ca448a7e16b5ae10ec7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 8 Sep 2020 11:12:35 -0400 Subject: [PATCH 22/28] ci: use pep517.build instead of manual setup.py --- .github/workflows/configure.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index eb2e3cc6b8..2a9c81a7cf 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -120,28 +120,19 @@ jobs: python-version: 3.8 - name: Prepare env - run: python -m pip install -r tests/requirements.txt wheel twine --prefer-binary + run: python -m pip install -r tests/requirements.txt pep517 twine --prefer-binary - name: Python Packaging tests run: pytest tests/extra_python_package/ - - name: Build SDist + - name: Build SDist and wheels run: | - python setup.py sdist - PYBIND11_GLOBAL_SDIST=1 python setup.py sdist + python -m pep517.build -s -b . + PYBIND11_GLOBAL_SDIST=1 python -m pep517.build -s -b . - - uses: actions/upload-artifact@v2 - with: - path: dist/* - - - name: Build wheel - run: | - python -m pip wheel . -w wheels - PYBIND11_GLOBAL_SDIST=1 python -m pip wheel . -w wheels + - name: Check metadata + run: twine check dist/* - uses: actions/upload-artifact@v2 with: - path: wheels/pybind11*.whl - - - name: Check metadata - run: twine check dist/* wheels/* + path: dist/* From ce8dc875ece6318a21f0fc8cba093400e0ac5645 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 13 Sep 2020 23:27:01 -0400 Subject: [PATCH 23/28] refactor: more comments from @YannickJadoul --- docs/compiling.rst | 10 +-- docs/upgrade.rst | 4 +- pybind11/setup_helpers.py | 84 +++++++++++------------- setup.py | 4 +- tests/extra_python_package/test_files.py | 4 +- tools/setup_global.py.in | 3 +- tools/setup_main.py.in | 2 +- 7 files changed, 53 insertions(+), 58 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index f52280c75e..c3333c86ee 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -127,11 +127,11 @@ Copy manually ------------- You can also copy ``setup_helpers.py`` directly to your project; it was -designed to be usable standalone, like the old example ``setup.py``. The -``CppExtension`` class is identical to ``Pybind11Extension``, except it does -not include the pybind11 package headers, so you can use it with git submodules -and a specific git version. If you use this, you will need to import from a -local file in ``setup.py`` and ensure the helper file is part of your MANIFEST. +designed to be usable standalone, like the old example ``setup.py``. You can +set ``include_pybind11=False`` to skip including the pybind11 package headers, +so you can use it with git submodules and a specific git version. If you use +this, you will need to import from a local file in ``setup.py`` and ensure the +helper file is part of your MANIFEST. .. versionchanged:: 2.6 diff --git a/docs/upgrade.rst b/docs/upgrade.rst index f37c872d49..2164996915 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -82,8 +82,8 @@ The Python package now includes the headers as data in the package itself, as well as in the "headers" wheel slot. ``pybind11 --includes`` and ``pybind11.get_include()`` report the new location, which is always correct regardless of how pybind11 was installed, making the old ``user=`` argument -meaningless. If you are not using the helper, you are encouraged to switch to -the package location. +meaningless. If you are not using the function to get the location already, you +are encouraged to switch to the package location. v2.2 diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 2ede2c79ee..041e22689f 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -64,32 +64,11 @@ # directory into your path if it sits beside your setup.py. -# Just in case someone clever tries to multithread -tmp_chdir_lock = threading.Lock() -cpp_cache_lock = threading.Lock() - - -@contextlib.contextmanager -def tmp_chdir(): - "Prepare and enter a temporary directory, cleanup when done" - - # Threadsafe - with tmp_chdir_lock: - olddir = os.getcwd() - try: - tmpdir = tempfile.mkdtemp() - os.chdir(tmpdir) - yield tmpdir - finally: - os.chdir(olddir) - shutil.rmtree(tmpdir) - - -class CppExtension(_Extension): +class Pybind11Extension(_Extension): """ - Build a C++11+ Extension module. This automatically adds the recommended - flags when you init the extension and assumes C++ sources - you can further - modify the options yourself. + Build a C++11+ Extension module with pybind11. This automatically adds the + recommended flags when you init the extension and assumes C++ sources - you + can further modify the options yourself. The customizations are: @@ -105,6 +84,9 @@ class CppExtension(_Extension): property more than once, as flags are added when you set it. Set the property to None to disable the addition of C++ standard flags. + If you want to add pybind11 headers manually, for example for an exact + git checkout, then set ``include_pybind11=False``. + Warning: do not use property-based access to the instance on Python 2 - this is an ugly old-style class due to Distutils. """ @@ -127,12 +109,27 @@ def __init__(self, *args, **kwargs): if "language" not in kwargs: kwargs["language"] = "c++" + include_pybind11 = kwargs.pop("include_pybind11", True) + # Can't use super here because distutils has old-style classes in # Python 2! _Extension.__init__(self, *args, **kwargs) + # Include the installed package pybind11 headers + if include_pybind11: + # If using setup_requires, this fails the first time - that's okay + try: + import pybind11 + + pyinc = pybind11.get_include() + + if pyinc not in self.include_dirs: + self.include_dirs.append(pyinc) + except ImportError: + pass + # Have to use the accessor manually to support Python 2 distutils - CppExtension.cxx_std.__set__(self, cxx_std) + Pybind11Extension.cxx_std.__set__(self, cxx_std) if WIN: self._add_cflags("/EHsc", "/bigobj") @@ -183,28 +180,25 @@ def cxx_std(self, level): self.extra_compile_args.append("-Wno-deprecated-register") -class Pybind11Extension(CppExtension): - """ - A pybind11 Extension subclass. Includes the header directory from the - package. - """ - - def __init__(self, *args, **kwargs): - CppExtension.__init__(self, *args, **kwargs) - - # If using setup_requires, this fails the first time - that's okay - try: - import pybind11 - - pyinc = pybind11.get_include() +# Just in case someone clever tries to multithread +tmp_chdir_lock = threading.Lock() +cpp_cache_lock = threading.Lock() - if pyinc not in self.include_dirs: - self.include_dirs.append(pyinc) - except ImportError: - pass +@contextlib.contextmanager +def tmp_chdir(): + "Prepare and enter a temporary directory, cleanup when done" -Pybind11Extension.__doc__ += CppExtension.__doc__ + # Threadsafe + with tmp_chdir_lock: + olddir = os.getcwd() + try: + tmpdir = tempfile.mkdtemp() + os.chdir(tmpdir) + yield tmpdir + finally: + os.chdir(olddir) + shutil.rmtree(tmpdir) # cf http://bugs.python.org/issue26689 diff --git a/setup.py b/setup.py index 226b17cd49..36ccc6f107 100644 --- a/setup.py +++ b/setup.py @@ -18,10 +18,10 @@ # PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers # files, and the sys.prefix files (CMake and headers). -alt_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) +global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) version_py = "pybind11/_version.py" -setup_py = "tools/setup_global.py.in" if alt_sdist else "tools/setup_main.py.in" +setup_py = "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" extra_cmd = 'cmdclass["sdist"] = SDist\n' to_src = ( diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index ad3fa16e40..e0c4e3773f 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -159,7 +159,7 @@ def test_build_sdist(monkeypatch, tmpdir): assert pyproject_toml == contents -def test_build_alt_dist(monkeypatch, tmpdir): +def test_build_global_dist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") @@ -252,7 +252,7 @@ def tests_build_wheel(monkeypatch, tmpdir): assert files == trimmed -def tests_build_alt_wheel(monkeypatch, tmpdir): +def tests_build_global_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in index b8ea101a36..3325cd0ead 100644 --- a/tools/setup_global.py.in +++ b/tools/setup_global.py.in @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Setup script for PyPI; use CMakeFile.txt to build extension modules +# Setup script for pybind11-global (in the sdist or in tools/setup_global.py in the repository) +# This package is targeted for easy use from CMake. import contextlib import glob diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index cd4f11c5bd..c859c1f755 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Setup script for PyPI - placed in the sdist +# Setup script (in the sdist or in tools/setup_main.py in the repository) from setuptools import setup From 3b9d2f46800f513701cb74c175d0700da402c074 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 14 Sep 2020 11:38:15 -0400 Subject: [PATCH 24/28] docs: updates from @ktbarrett --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d1ecc10c88..dca8aaff20 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,7 +42,7 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. * :ref:`build-setuptools` is easier thanks to a new ``pybind11.setup_helpers`` module, which provides utilities to use - setuptools with pybind11, and can be used via PEP 518, ``setup_requires``, + setuptools with pybind11. It can be used via PEP 518, ``setup_requires``, or by directly copying ``setup_helpers.py`` into your project. * CMake configuration files are now included in the Python package. Use @@ -52,6 +52,9 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. new ``pybind11[global]`` option, which installs the CMake files and headers into your base environment in the standard location. + * ``pybind11-config`` is another way to write ``python -m pybind11`` if you + have your PATH set up. + * Minimum CMake required increased to 3.4. `#2338 `_ and `#2370 `_ From e541907f27b5b56eb82d12d81bc5ea076c1d8116 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 14 Sep 2020 16:13:59 -0400 Subject: [PATCH 25/28] fix: change to newly recommended tool instead of pep517.build This was intended as a proof of concept; build seems to be the correct replacement. See https://github.com/pypa/pep517/pull/83 --- .github/CONTRIBUTING.md | 8 ++++---- .github/workflows/configure.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ca221fbd44..58aa5b4df1 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -224,14 +224,14 @@ source, it is best to have Pip 10 on a manylinux1, macOS, or Windows system. You can then build the SDists, or run any procedure that makes SDists internally, like making wheels or installing. Since Pip itself does not have an `sdist` command (it does have `wheel` and `install`), you will -need to use the `pep517` package directly: +need to use the upcoming `build` package: ```bash # Normal package -python3 -m pep517.build -s . +python3 -m build -s . # Global extra -PYBIND11_GLOBAL_SDIST=1 python3 -m pep517.build -s . +PYBIND11_GLOBAL_SDIST=1 python3 -m build -s . ``` If you want to use the classic "direct" usage of `python setup.py`, you will @@ -256,7 +256,7 @@ When you invoke any `setup.py` command from the source directory, including `pip wheel .` and `pip install .`, you will activate a full source build. This is made of the following steps: -1. If the tool is PEP 518 compliant, like `pep517.build` or Pip 10+, it will +1. If the tool is PEP 518 compliant, like `build` or Pip 10+, it will create a temporary virtual environment and install the build requirements (mostly CMake) into it. (if you are not on Windows, macOS, or a manylinux compliant system, you can disable this with `--no-build-isolation` as long diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 2a9c81a7cf..3dd248e04a 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -120,15 +120,15 @@ jobs: python-version: 3.8 - name: Prepare env - run: python -m pip install -r tests/requirements.txt pep517 twine --prefer-binary + run: python -m pip install -r tests/requirements.txt build twine --prefer-binary - name: Python Packaging tests run: pytest tests/extra_python_package/ - name: Build SDist and wheels run: | - python -m pep517.build -s -b . - PYBIND11_GLOBAL_SDIST=1 python -m pep517.build -s -b . + python -m build -s -w . + PYBIND11_GLOBAL_SDIST=1 python -m build -s -w . - name: Check metadata run: twine check dist/* From a63e6fc459e5cfe06fbece1003873b65f2ae788c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 15 Sep 2020 17:43:35 -0400 Subject: [PATCH 26/28] docs: updates from @wjakob --- .github/CONTRIBUTING.md | 71 ++++++++++++++++++++++++++++------------- .pre-commit-config.yaml | 3 +- docs/changelog.rst | 5 +-- docs/compiling.rst | 28 +++++++++------- 4 files changed, 69 insertions(+), 38 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 58aa5b4df1..a8daeda249 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -212,21 +212,43 @@ cmake --build build ### Explanation of the SDist/wheel building design -> These details are _only_ for building from the Python sources from git. The +> These details below are _only_ for packaging the Python sources from git. The > SDists and wheels created do not have any extra requirements at all and are > completely normal. -In order to support CMake output files (`pybind11Config.cmake` and helper -files), the `[globals]` option, and a simple version file taken from the C++ -sources, the `setup.py` _in the source_ has a few tricks to make the simplest -possible SDists and wheels. If you want to build the SDist from the GitHub -source, it is best to have Pip 10 on a manylinux1, macOS, or Windows system. -You can then build the SDists, or run any procedure that makes SDists -internally, like making wheels or installing. Since Pip itself -does not have an `sdist` command (it does have `wheel` and `install`), you will -need to use the upcoming `build` package: +The main objective of the packaging system is to create SDists (Python's source +distribution packages) and wheels (Python's binary distribution packages) that +include everything that is needed to work with pybind11, and which can be +installed without any additional dependencies. This is more complex than it +appears: in order to support CMake as a first class language even when using +the PyPI package, they must include the _generated_ CMake files (so as not to +require CMake when installing the `pybind11` package itself). They should also +provide the option to install to the "standard" location +(`/include/pybind11` and `/share/cmake/pybind11`) so they are +easy to find with CMake, but this can cause problems if you are not an +environment or using ``pyproject.toml`` requirements. This was solved by having +two packages; the "nice" pybind11 package that stores the includes and CMake +files inside the package, that you get access to via functions in the package, +and a `pybind11-global` package that can be included via `pybind11[global]` if +you want the more invasive but discoverable file locations. + +If you want to install or package the GitHub source, it is best to have Pip 10 +or newer on Windows, macOS, or Linux (manylinux1 compatible, includes most +distributions). You can then build the SDists, or run any procedure that makes +SDists internally, like making wheels or installing. + + +```bash +# Editable development install example +python3 -m pip install -e . +``` + +Since Pip itself does not have an `sdist` command (it does have `wheel` and +`install`), you may want to use the upcoming `build` package: ```bash +python3 -m pip install build + # Normal package python3 -m build -s . @@ -235,20 +257,21 @@ PYBIND11_GLOBAL_SDIST=1 python3 -m build -s . ``` If you want to use the classic "direct" usage of `python setup.py`, you will -need CMake 3.15+ and either make or ninja preinstalled (possibly via `pip +need CMake 3.15+ and either `make` or `ninja` preinstalled (possibly via `pip install cmake ninja`), since directly running Python on `setup.py` cannot pick -up `pyproject.toml` requirements. As long as you have those two things, though, -everything works as normal: +up and install `pyproject.toml` requirements. As long as you have those two +things, though, everything works the way you would expect: ```bash # Normal package -python setup.py sdist +python3 setup.py sdist # Global extra -PYBIND11_GLOBAL_SDIST=1 python setup.py sdist +PYBIND11_GLOBAL_SDIST=1 python3 setup.py sdist ``` -A detailed explanation of the build procedure design is as follows: +A detailed explanation of the build procedure design for developers wanting to +work on or maintain the packaging system is as follows: #### 1. Building from the source directory @@ -256,15 +279,16 @@ When you invoke any `setup.py` command from the source directory, including `pip wheel .` and `pip install .`, you will activate a full source build. This is made of the following steps: -1. If the tool is PEP 518 compliant, like `build` or Pip 10+, it will - create a temporary virtual environment and install the build requirements - (mostly CMake) into it. (if you are not on Windows, macOS, or a manylinux - compliant system, you can disable this with `--no-build-isolation` as long - as you have CMake 3.15+ installed) +1. If the tool is PEP 518 compliant, like Pip 10+, it will create a temporary + virtual environment and install the build requirements (mostly CMake) into + it. (if you are not on Windows, macOS, or a manylinux compliant system, you + can disable this with `--no-build-isolation` as long as you have CMake 3.15+ + installed) 2. The environment variable `PYBIND11_GLOBAL_SDIST` is checked - if it is set and truthy, this will be make the accessory `pybind11-global` package, instead of the normal `pybind11` package. This package is used for - installing the "global" headers and CMake files, using `pybind11[global]`. + installing the files directly to your environment root directory, using + `pybind11[global]`. 2. `setup.py` reads the version from `includes/pybind11/detail/common.h`. 3. CMake is run with `-DCMAKE_INSTALL_PREIFX=pybind11`. Since the CMake install procedure uses only relative paths and is identical on all platforms, these @@ -284,7 +308,8 @@ is made of the following steps: Since the SDist has the rendered template files in `tools` along with the includes and CMake files in the correct locations, the builds are completely trivial and simple. No extra requirements are required. The version is baked -in, so there is not even a lookup performance penalty / point of failure. +in, so there is not even a lookup performance penalty / point of failure. You +can even use Pip 9 if you really want to. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26f5e2f843..71513c991c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,8 +51,7 @@ repos: additional_dependencies: [flake8-bugbear, pep8-naming] exclude: ^(docs/.*|tools/.*)$ -# CMake formatting (first party, but in a separate repo due to author's desire -# to use PyPI for pinning) +# CMake formatting - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.11 hooks: diff --git a/docs/changelog.rst b/docs/changelog.rst index dca8aaff20..92900a5b74 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -49,8 +49,9 @@ See :ref:`upgrade-guide-2.6` for help upgrading to the new version. ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get the directory with the CMake configuration files, or include the site-packages location in your ``CMAKE_MODULE_PATH``. Or you can use the - new ``pybind11[global]`` option, which installs the CMake files and headers - into your base environment in the standard location. + new ``pybind11[global]`` extra when you install ``pybind11``, which + installs the CMake files and headers into your base environment in the + standard location * ``pybind11-config`` is another way to write ``python -m pybind11`` if you have your PATH set up. diff --git a/docs/compiling.rst b/docs/compiling.rst index c3333c86ee..dc8b4e8837 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -74,17 +74,11 @@ PEP 518 requirements (Pip 10+ required) --------------------------------------- If you use `PEP 518's `_ -``pyproject.toml`` file, you can ensure that pybind11 is available during the -building of your project. When this file exists, Pip will make a new virtual -environment, download just the packages listed here in ``requires=``, and build -a wheel (binary Python package). It will then throw away the environment, and -install your wheel. - -The main drawback is that Pip 10+ is required to build from source; older -versions completely ignore this file. If you distribute wheels, using something -like `cibuildwheel `_, remember that -``setup.py`` and ``pyproject.toml`` are not even contained in the wheel, so -this high Pip requirement is only for source builds. +``pyproject.toml`` file, you can ensure that ``pybind11`` is available during +the compilation of your project. When this file exists, Pip will make a new +virtual environment, download just the packages listed here in ``requires=``, +and build a wheel (binary Python package). It will then throw away the +environment, and install your wheel. Your ``pyproject.toml`` file will likely look something like this: @@ -94,6 +88,18 @@ Your ``pyproject.toml`` file will likely look something like this: requires = ["setuptools", "wheel", "pybind11==2.6.0"] build-backend = "setuptools.build_meta" +.. note:: + + The main drawback to this method is that a `PEP 517`_ compiant build tool, + such as Pip 10+, is required for this approach to work; older versions of + Pip completely ignore this file. If you distribute binaries (called wheels + in Python) using something like `cibuildwheel`_, remember that ``setup.py`` + and ``pyproject.toml`` are not even contained in the wheel, so this high + Pip requirement is only for source builds, and will not affect users of + your binary wheels. + +.. _PEP 517: https://www.python.org/dev/peps/pep-0517/ +.. _cibuildwheel: https://cibuildwheel.readthedocs.io .. _setup_helpers-setup_requires: From 71e6420ea5abf282edcef565a8f3cc4f54602191 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 15 Sep 2020 14:24:03 -0400 Subject: [PATCH 27/28] refactor: dual version locations --- .github/CONTRIBUTING.md | 19 +++++++++--------- pybind11/_version.py | 25 +----------------------- setup.py | 24 +++++++++++++++++------ tests/extra_python_package/test_files.py | 22 --------------------- tools/_version.py.in | 12 ------------ 5 files changed, 28 insertions(+), 74 deletions(-) delete mode 100644 tools/_version.py.in diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index a8daeda249..4ced21baaa 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -289,17 +289,18 @@ is made of the following steps: instead of the normal `pybind11` package. This package is used for installing the files directly to your environment root directory, using `pybind11[global]`. -2. `setup.py` reads the version from `includes/pybind11/detail/common.h`. +2. `setup.py` reads the version from `pybind11/_version.py` and verifies it + matches `includes/pybind11/detail/common.h`. 3. CMake is run with `-DCMAKE_INSTALL_PREIFX=pybind11`. Since the CMake install procedure uses only relative paths and is identical on all platforms, these files are valid as long as they stay in the correct relative position to the includes. `pybind11/share/cmake/pybind11` has the CMake files, and `pybind11/include` has the includes. The build directory is discarded. -4. Three files are placed in the SDist: `tools/setup_main.py.in`, - `tools/pyproject.toml`, and `tools/_version.py.in`. -5. The package is created using the setup function in the `tools/setup_*.py`. - `setup_main.py` fills in Python packages, and `setup_global.py` fills in - only the data/header slots. +4. Simpler files are placed in the SDist: `tools/setup_*.py.in`, + `tools/pyproject.toml` (`main` or `global`) +5. The package is created by running the setup function in the + `tools/setup_*.py`. `setup_main.py` fills in Python packages, and + `setup_global.py` fills in only the data/header slots. 6. A context manager cleans up the temporary CMake install directory (even if an error is thrown). @@ -307,10 +308,8 @@ is made of the following steps: Since the SDist has the rendered template files in `tools` along with the includes and CMake files in the correct locations, the builds are completely -trivial and simple. No extra requirements are required. The version is baked -in, so there is not even a lookup performance penalty / point of failure. You -can even use Pip 9 if you really want to. - +trivial and simple. No extra requirements are required. You can even use Pip 9 +if you really want to. [pre-commit]: https://pre-commit.com diff --git a/pybind11/_version.py b/pybind11/_version.py index de8fb753b1..ca84c262c9 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -1,10 +1,5 @@ # -*- coding: utf-8 -*- -import os - - -DIR = os.path.abspath(os.path.dirname(__file__)) - def _to_int(s): try: @@ -13,23 +8,5 @@ def _to_int(s): return s -# Get the version from the C++ file (in-source) -versions = {} - -common_h = os.path.join( - os.path.dirname(DIR), "include", "pybind11", "detail", "common.h" -) - -with open(common_h) as f: - for line in f: - if "PYBIND11_VERSION_" in line: - _, name, vers = line.split() - versions[name[17:].lower()] = vers - if len(versions) >= 3: - break - else: - msg = "Version number not read correctly from {}: {}".format(common_h, versions) - raise RuntimeError(msg) - -__version__ = "{v[major]}.{v[minor]}.{v[patch]}".format(v=versions) +__version__ = "2.6.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/setup.py b/setup.py index 36ccc6f107..c9ba77d6d8 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ import contextlib import os +import re import shutil import string import subprocess @@ -14,29 +15,40 @@ import setuptools.command.sdist DIR = os.path.abspath(os.path.dirname(__file__)) +VERSION_REGEX = re.compile( + r"^\s*#\s*define\s+PYBIND11_VERSION_([A-Z]+)\s+(.*)$", re.MULTILINE +) # PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers # files, and the sys.prefix files (CMake and headers). global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) -version_py = "pybind11/_version.py" setup_py = "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" extra_cmd = 'cmdclass["sdist"] = SDist\n' to_src = ( - (version_py, "tools/_version.py.in"), ("pyproject.toml", "tools/pyproject.toml"), ("setup.py", setup_py), ) - -with open(version_py) as f: - loc = {"__file__": version_py} - code = compile(f.read(), version_py, "exec") +# Read the listed version +with open("pybind11/_version.py") as f: + code = compile(f.read(), "pybind11/_version.py", "exec") + loc = {} exec(code, loc) version = loc["__version__"] +# Verify that the version matches the one in C++ +with open("include/pybind11/detail/common.h") as f: + matches = dict(VERSION_REGEX.findall(f.read())) +cpp_version = "{MAJOR}.{MINOR}.{PATCH}".format(**matches) +if version != cpp_version: + msg = "Python version {} does not match C++ version {}!".format( + version, cpp_version + ) + raise RuntimeError(msg) + def get_and_replace(filename, binary=False, **opts): with open(filename, "rb" if binary else "r") as f: diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e0c4e3773f..ac8ca1f97b 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -123,11 +123,6 @@ def test_build_sdist(monkeypatch, tmpdir): ) as f: setup_py = f.read() - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pybind11/_version.py")) - ) as f: - version_py = f.read() - with contextlib.closing( tar.extractfile(tar.getmember(start + "pyproject.toml")) ) as f: @@ -148,12 +143,6 @@ def test_build_sdist(monkeypatch, tmpdir): ) assert setup_py == contents - with open(os.path.join(MAIN_DIR, "tools", "_version.py.in"), "rb") as f: - contents = ( - string.Template(f.read().decode()).substitute(version=version).encode() - ) - assert version_py == contents - with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: contents = f.read() assert pyproject_toml == contents @@ -189,11 +178,6 @@ def test_build_global_dist(monkeypatch, tmpdir): ) as f: setup_py = f.read() - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pybind11/_version.py")) - ) as f: - version_py = f.read() - with contextlib.closing( tar.extractfile(tar.getmember(start + "pyproject.toml")) ) as f: @@ -212,12 +196,6 @@ def test_build_global_dist(monkeypatch, tmpdir): ) assert setup_py == contents - with open(os.path.join(MAIN_DIR, "tools", "_version.py.in"), "rb") as f: - contents = ( - string.Template(f.read().decode()).substitute(version=version).encode() - ) - assert version_py == contents - with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: contents = f.read() assert pyproject_toml == contents diff --git a/tools/_version.py.in b/tools/_version.py.in deleted file mode 100644 index 8a5dea18ec..0000000000 --- a/tools/_version.py.in +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- - - -def _to_int(s): - try: - return int(s) - except ValueError: - return s - - -__version__ = "$version" -version_info = tuple(_to_int(s) for s in __version__.split(".")) From 6cdbd4be236cada6efbae4e9b55a1e2aee699571 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 16 Sep 2020 15:59:38 -0400 Subject: [PATCH 28/28] docs: typo spotted by @wjakob --- docs/compiling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index dc8b4e8837..344dac6988 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -90,7 +90,7 @@ Your ``pyproject.toml`` file will likely look something like this: .. note:: - The main drawback to this method is that a `PEP 517`_ compiant build tool, + The main drawback to this method is that a `PEP 517`_ compliant build tool, such as Pip 10+, is required for this approach to work; older versions of Pip completely ignore this file. If you distribute binaries (called wheels in Python) using something like `cibuildwheel`_, remember that ``setup.py``