diff --git a/.appveyor.yml b/.appveyor.yml index b33a4ccf4e..85445d41a2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,11 +19,11 @@ install: if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" python -W ignore -m pip install --upgrade pip wheel - python -W ignore -m pip install pytest numpy --no-warn-script-location + python -W ignore -m pip install pytest numpy --no-warn-script-location pytest-timeout - ps: | - Start-FileDownload 'http://bitbucket.org/eigen/eigen/get/3.3.3.zip' - 7z x 3.3.3.zip -y > $null - $env:CMAKE_INCLUDE_PATH = "eigen-eigen-67e894c6cd8f;$env:CMAKE_INCLUDE_PATH" + Start-FileDownload 'https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.zip' + 7z x eigen-3.3.7.zip -y > $null + $env:CMAKE_INCLUDE_PATH = "eigen-3.3.7;$env:CMAKE_INCLUDE_PATH" build_script: - cmake -G "%CMAKE_GENERATOR%" -A "%CMAKE_ARCH%" -DCMAKE_CXX_STANDARD=14 diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..8700fca84d --- /dev/null +++ b/.clang-format @@ -0,0 +1,21 @@ +--- +# See all possible options and defaults with: +# clang-format --style=llvm --dump-config +BasedOnStyle: LLVM +AccessModifierOffset: -4 +AlignConsecutiveAssignments: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BreakBeforeBinaryOperators: All +BreakConstructorInitializers: BeforeColon +ColumnLimit: 99 +IndentCaseLabels: true +IndentPPDirectives: AfterHash +IndentWidth: 4 +Language: Cpp +SpaceAfterCStyleCast: true +# SpaceInEmptyBlock: true # too new +Standard: Cpp11 +TabWidth: 4 +... diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000000..e29d929897 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,13 @@ +FormatStyle: file + +Checks: ' +llvm-namespace-comment, +modernize-use-override, +readability-container-size-empty, +modernize-use-using, +modernize-use-equals-default, +modernize-use-auto, +modernize-use-emplace, +' + +HeaderFilterRegex: 'pybind11/.*h' diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f61011d540..08d9e7cb93 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -126,7 +126,8 @@ cmake --build build --target check `--target` can be spelled `-t` in CMake 3.15+. You can also run individual tests with these targets: -* `pytest`: Python tests only +* `pytest`: Python tests only, using the +[pytest](https://docs.pytest.org/en/stable/) framework * `cpptest`: C++ tests only * `test_cmake_build`: Install / subdirectory tests @@ -134,6 +135,17 @@ If you want to build just a subset of tests, use `-DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp"`. If this is empty, all tests will be built. +You may also pass flags to the `pytest` target by editing `tests/pytest.ini` or +by using the `PYTEST_ADDOPTS` environment variable +(see [`pytest` docs](https://docs.pytest.org/en/2.7.3/customize.html#adding-default-options)). As an example: + +```bash +env PYTEST_ADDOPTS="--capture=no --exitfirst" \ + cmake --build build --target pytest +# Or using abbreviated flags +env PYTEST_ADDOPTS="-s -x" cmake --build build --target pytest +``` + ### Formatting All formatting is handled by pre-commit. @@ -164,7 +176,182 @@ name, pre-commit): pre-commit install ``` +### Clang-Format + +As of v2.6.2, pybind11 ships with a [`clang-format`][clang-format] +configuration file at the top level of the repo (the filename is +`.clang-format`). Currently, formatting is NOT applied automatically, but +manually using `clang-format` for newly developed files is highly encouraged. +To check if a file needs formatting: + +```bash +clang-format -style=file --dry-run some.cpp +``` + +The output will show things to be fixed, if any. To actually format the file: + +```bash +clang-format -style=file -i some.cpp +``` + +Note that the `-style-file` option searches the parent directories for the +`.clang-format` file, i.e. the commands above can be run in any subdirectory +of the pybind11 repo. + +### Clang-Tidy + +[`clang-tidy`][clang-tidy] performs deeper static code analyses and is +more complex to run, compared to `clang-format`, but support for `clang-tidy` +is built into the pybind11 CMake configuration. To run `clang-tidy`, the +following recipe should work. Files will be modified in place, so you can +use git to monitor the changes. + +```bash +docker run --rm -v $PWD:/pybind11 -it silkeh/clang:10 +apt-get update && apt-get install python3-dev python3-pytest +cmake -S pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);-fix" +cmake --build build +``` + +### Include what you use + +To run include what you use, install (`brew install include-what-you-use` on +macOS), then run: + +```bash +cmake -S . -B build-iwyu -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=$(which include-what-you-use) +cmake --build build +``` + +The report is sent to stderr; you can pipe it into a file if you wish. + +### Build recipes + +This builds with the Intel compiler (assuming it is in your path, along with a +recent CMake and Python 3): + +```bash +python3 -m venv venv +. venv/bin/activate +pip install pytest +cmake -S . -B build-intel -DCMAKE_CXX_COMPILER=$(which icpc) -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DPYBIND11_WERROR=ON +``` + +This will test the PGI compilers: + +```bash +docker run --rm -it -v $PWD:/pybind11 nvcr.io/hpc/pgi-compilers:ce +apt-get update && apt-get install -y python3-dev python3-pip python3-pytest +wget -qO- "https://cmake.org/files/v3.18/cmake-3.18.2-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local +cmake -S pybind11/ -B build +cmake --build build +``` + +### Explanation of the SDist/wheel building design + +> These details below are _only_ for packaging the Python sources from git. The +> SDists and wheels created do not have any extra requirements at all and are +> completely normal. + +The main objective of the packaging system is to create SDists (Python's source +distribution packages) and wheels (Python's binary distribution packages) that +include everything that is needed to work with pybind11, and which can be +installed without any additional dependencies. This is more complex than it +appears: in order to support CMake as a first class language even when using +the PyPI package, they must include the _generated_ CMake files (so as not to +require CMake when installing the `pybind11` package itself). They should also +provide the option to install to the "standard" location +(`/include/pybind11` and `/share/cmake/pybind11`) so they are +easy to find with CMake, but this can cause problems if you are not an +environment or using ``pyproject.toml`` requirements. This was solved by having +two packages; the "nice" pybind11 package that stores the includes and CMake +files inside the package, that you get access to via functions in the package, +and a `pybind11-global` package that can be included via `pybind11[global]` if +you want the more invasive but discoverable file locations. + +If you want to install or package the GitHub source, it is best to have Pip 10 +or newer on Windows, macOS, or Linux (manylinux1 compatible, includes most +distributions). You can then build the SDists, or run any procedure that makes +SDists internally, like making wheels or installing. + + +```bash +# Editable development install example +python3 -m pip install -e . +``` + +Since Pip itself does not have an `sdist` command (it does have `wheel` and +`install`), you may want to use the upcoming `build` package: + +```bash +python3 -m pip install build + +# Normal package +python3 -m build -s . + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 -m build -s . +``` + +If you want to use the classic "direct" usage of `python setup.py`, you will +need CMake 3.15+ and either `make` or `ninja` preinstalled (possibly via `pip +install cmake ninja`), since directly running Python on `setup.py` cannot pick +up and install `pyproject.toml` requirements. As long as you have those two +things, though, everything works the way you would expect: + +```bash +# Normal package +python3 setup.py sdist + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 setup.py sdist +``` + +A detailed explanation of the build procedure design for developers wanting to +work on or maintain the packaging system is as follows: + +#### 1. Building from the source directory + +When you invoke any `setup.py` command from the source directory, including +`pip wheel .` and `pip install .`, you will activate a full source build. This +is made of the following steps: + +1. If the tool is PEP 518 compliant, like Pip 10+, it will create a temporary + virtual environment and install the build requirements (mostly CMake) into + it. (if you are not on Windows, macOS, or a manylinux compliant system, you + can disable this with `--no-build-isolation` as long as you have CMake 3.15+ + installed) +2. The environment variable `PYBIND11_GLOBAL_SDIST` is checked - if it is set + and truthy, this will be make the accessory `pybind11-global` package, + instead of the normal `pybind11` package. This package is used for + installing the files directly to your environment root directory, using + `pybind11[global]`. +2. `setup.py` reads the version from `pybind11/_version.py` and verifies it + matches `includes/pybind11/detail/common.h`. +3. CMake is run with `-DCMAKE_INSTALL_PREIFX=pybind11`. Since the CMake install + procedure uses only relative paths and is identical on all platforms, these + files are valid as long as they stay in the correct relative position to the + includes. `pybind11/share/cmake/pybind11` has the CMake files, and + `pybind11/include` has the includes. The build directory is discarded. +4. Simpler files are placed in the SDist: `tools/setup_*.py.in`, + `tools/pyproject.toml` (`main` or `global`) +5. The package is created by running the setup function in the + `tools/setup_*.py`. `setup_main.py` fills in Python packages, and + `setup_global.py` fills in only the data/header slots. +6. A context manager cleans up the temporary CMake install directory (even if + an error is thrown). + +### 2. Building from SDist + +Since the SDist has the rendered template files in `tools` along with the +includes and CMake files in the correct locations, the builds are completely +trivial and simple. No extra requirements are required. You can even use Pip 9 +if you really want to. + + [pre-commit]: https://pre-commit.com +[clang-format]: https://clang.llvm.org/docs/ClangFormat.html +[clang-tidy]: https://clang.llvm.org/extra/clang-tidy/ [pybind11.readthedocs.org]: http://pybind11.readthedocs.org/en/latest [issue tracker]: https://github.com/pybind/pybind11/issues [gitter]: https://gitter.im/pybind/Lobby diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..73273365c0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + ignore: + # Official actions have moving tags like v1 + # that are used, so they don't need updates here + - dependency-name: "actions/checkout" + - dependency-name: "actions/setup-python" + - dependency-name: "actions/cache" + - dependency-name: "actions/upload-artifact" + - dependency-name: "actions/download-artifact" + - dependency-name: "actions/labeler" diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000..abb0d05aaa --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,8 @@ +docs: +- any: + - 'docs/**/*.rst' + - '!docs/changelog.rst' + - '!docs/upgrade.rst' + +ci: +- '.github/workflows/*.yml' diff --git a/.github/labeler_merged.yml b/.github/labeler_merged.yml new file mode 100644 index 0000000000..2374ad42e4 --- /dev/null +++ b/.github/labeler_merged.yml @@ -0,0 +1,3 @@ +needs changelog: +- all: + - '!docs/changelog.rst' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..97a6ff7dda --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ +## Description + + + + +## Suggested changelog entry: + + + +```rst + +``` + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 762fafca2f..8dfd9b4995 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,65 +8,53 @@ on: - drake jobs: + # This is the "main" test suite, which tests a large number of different + # versions of default compilers and Python versions in GitHub Actions. standard: strategy: fail-fast: false matrix: # runs-on: [ubuntu-latest, windows-latest, macos-latest] runs-on: [ubuntu-latest, macos-latest] - arch: [x64] - max-cxx-std: [17] python: # - 2.7 # - 3.5 - - 3.7 - - 3.8 - - 3.9-dev + - 3.6 + - 3.9 + - 3.10-dev # - pypy2 # - pypy3 + # Items in here will either be added to the build matrix (if not + # present), or add new keys to an existing matrix element if all the + # existing keys match. + # + # We support three optional keys: args (both build), args1 (first + # build), and args2 (second build). include: + # Just add a key - runs-on: ubuntu-latest python: 3.6 - arch: x64 - max-cxx-std: 17 - - runs-on: macos-latest - python: 3.7 - arch: x64 - max-cxx-std: 17 - # - runs-on: windows-2016 - # python: 3.7 - # arch: x86 - # max-cxx-std: 14 - - exclude: - # # Currently 32bit only, and we build 64bit + args: > + -DPYBIND11_FINDPYTHON=ON # - runs-on: windows-latest - # python: pypy2 - # arch: x64 - # max-cxx-std: 17 - # - runs-on: windows-latest - # python: pypy3 - # arch: x64 - # max-cxx-std: 17 - - # Currently can't build due to warning, fixed in CPython > 3.9b5 - - runs-on: macos-latest - python: 3.9-dev - arch: x64 - max-cxx-std: 17 - - # # Currently broken on embed_test - # - runs-on: windows-latest - # python: 3.8 - # arch: x64 - # max-cxx-std: 17 - # - runs-on: windows-latest - # python: 3.9-dev - # arch: x64 - # max-cxx-std: 17 - - name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • ${{ matrix.arch }}" + # python: 3.6 + # args: > + # -DPYBIND11_FINDPYTHON=ON + + # # These items will be removed from the build matrix, keys must match. + # exclude: + # # Currently 32bit only, and we build 64bit + # - runs-on: windows-latest + # python: pypy2 + # - runs-on: windows-latest + # python: pypy3 + + # # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine. + # - runs-on: ubuntu-latest + # python: pypy2 + + name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}" runs-on: ${{ matrix.runs-on }} steps: @@ -76,10 +64,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - architecture: ${{ matrix.arch }} + + - name: Setup Boost (Windows / Linux latest) + shell: bash + run: echo "BOOST_ROOT=$BOOST_ROOT_1_72_0" >> $GITHUB_ENV + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.8 - name: Cache wheels - if: startsWith(runner.os, 'macOS') + if: runner.os == 'macOS' uses: actions/cache@v2 with: # This path is specific to macOS - we really only need it for PyPy NumPy wheels @@ -87,55 +81,168 @@ jobs: # for ways to do this more generally path: ~/Library/Caches/pip # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ matrix.arch }}-${{ hashFiles('tests/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }} - name: Prepare env - run: python -m pip install -r tests/requirements.txt + run: python -m pip install -r tests/requirements.txt --prefer-binary - - name: Configure C++11 - shell: bash + - name: Setup annotations on Linux + if: runner.os == 'Linux' + run: python -m pip install pytest-github-actions-annotate-failures + + # First build - C++11 mode and inplace + - name: Configure C++11 ${{ matrix.args }} run: > - cmake -S . -B build + cmake -S . -B . -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 - -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + ${{ matrix.args }} - name: Build C++11 - run: cmake --build build -j 2 + run: cmake --build . -j 2 - name: Python tests C++11 - run: cmake --build build --target pytest -j 2 + run: cmake --build . --target pytest -j 2 - name: C++11 tests - run: cmake --build build --target cpptest -j 2 + # TODO: Figure out how to load the DLL on Python 3.8+ + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))" + run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 - run: cmake --build build --target test_cmake_build + run: cmake --build . --target test_cmake_build - - name: Configure C++${{ matrix.max-cxx-std }} - shell: bash + - name: Clean directory + run: git clean -fdx + + # Second build - C++17 mode and in a build directory + - name: Configure ${{ matrix.args2 }} run: > cmake -S . -B build2 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON - -DCMAKE_CXX_STANDARD=${{ matrix.max-cxx-std }} - -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + -DCMAKE_CXX_STANDARD=17 + ${{ matrix.args }} + ${{ matrix.args2 }} - - name: Build C++${{ matrix.max-cxx-std }} + - name: Build run: cmake --build build2 -j 2 - - name: Python tests C++${{ matrix.max-cxx-std }} + - name: Python tests run: cmake --build build2 --target pytest - - name: C++${{ matrix.max-cxx-std }} tests + - name: C++ tests + # TODO: Figure out how to load the DLL on Python 3.8+ + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))" run: cmake --build build2 --target cpptest - - name: Interface test C++${{ matrix.max-cxx-std }} + - name: Interface test run: cmake --build build2 --target test_cmake_build + # Eventually Microsoft might have an action for setting up + # MSVC, but for now, this action works: + - name: Prepare compiler environment for Windows 🐍 2.7 + if: matrix.python == 2.7 && runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + # This makes two environment variables available in the following step(s) + - name: Set Windows 🐍 2.7 environment variables + if: matrix.python == 2.7 && runner.os == 'Windows' + shell: bash + run: | + echo "DISTUTILS_USE_SDK=1" >> $GITHUB_ENV + echo "MSSdk=1" >> $GITHUB_ENV + + # This makes sure the setup_helpers module can build packages using + # setuptools + - name: Setuptools helpers test + run: pytest tests/extra_setuptools + + + # deadsnakes: + # strategy: + # fail-fast: false + # matrix: + # include: + # - python-version: 3.9 + # python-debug: true + # valgrind: true + # - python-version: 3.10-dev + # python-debug: false + + # name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v2 + + # - name: Setup Python ${{ matrix.python-version }} (deadsnakes) + # uses: deadsnakes/action@v2.1.1 + # with: + # python-version: ${{ matrix.python-version }} + # debug: ${{ matrix.python-debug }} + + # - name: Update CMake + # uses: jwlawson/actions-setup-cmake@v1.7 + + # - name: Valgrind cache + # if: matrix.valgrind + # uses: actions/cache@v2 + # id: cache-valgrind + # with: + # path: valgrind + # key: 3.16.1 # Valgrind version + + # - name: Compile Valgrind + # if: matrix.valgrind && steps.cache-valgrind.outputs.cache-hit != 'true' + # run: | + # VALGRIND_VERSION=3.16.1 + # curl https://sourceware.org/pub/valgrind/valgrind-$VALGRIND_VERSION.tar.bz2 -o - | tar xj + # mv valgrind-$VALGRIND_VERSION valgrind + # cd valgrind + # ./configure + # make -j 2 > /dev/null + + # - name: Install Valgrind + # if: matrix.valgrind + # working-directory: valgrind + # run: | + # sudo make install + # sudo apt-get update + # sudo apt-get install libc6-dbg # Needed by Valgrind + + # - name: Prepare env + # run: python -m pip install -r tests/requirements.txt --prefer-binary + + # - name: Configure + # run: > + # cmake -S . -B build + # -DCMAKE_BUILD_TYPE=Debug + # -DPYBIND11_WERROR=ON + # -DDOWNLOAD_CATCH=ON + # -DDOWNLOAD_EIGEN=ON + # -DCMAKE_CXX_STANDARD=17 + + # - name: Build + # run: cmake --build build -j 2 + + # - name: Python tests + # run: cmake --build build --target pytest + + # - name: C++ tests + # run: cmake --build build --target cpptest + + # - name: Run Valgrind on Python tests + # if: matrix.valgrind + # run: cmake --build build --target memcheck + + + # Testing on clang using the excellent silkeh clang docker images clang: runs-on: ubuntu-latest strategy: @@ -145,12 +252,20 @@ jobs: # - 3.6 # - 3.7 # - 3.9 - # - 5 - # - 7 + - 7 - 9 - dev - - name: "🐍 3 • Clang ${{ matrix.clang }} • x64" + std: + - 11 + include: + # - clang: 5 + # std: 14 + - clang: 10 + std: 20 + - clang: 10 + std: 17 + + name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" container: "silkeh/clang:${{ matrix.clang }}" steps: @@ -165,6 +280,7 @@ jobs: cmake -S . -B build -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") - name: Build @@ -179,6 +295,124 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + + # # Testing NVCC; forces sources to behave like .cu files + # cuda: + # runs-on: ubuntu-latest + # name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04" + # container: nvidia/cuda:11.0-devel-ubuntu20.04 + + # steps: + # - uses: actions/checkout@v2 + + # # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND + # - name: Install 🐍 3 + # run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy + + # - name: Configure + # run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + + # - name: Build + # run: cmake --build build -j2 --verbose + + # - name: Python tests + # run: cmake --build build --target pytest + + +# TODO: Internal compiler error - report to NVidia +# # Testing CentOS 8 + PGI compilers +# centos-nvhpc8: +# runs-on: ubuntu-latest +# name: "🐍 3 • CentOS8 / PGI 20.11 • x64" +# container: centos:8 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Add Python 3 and a few requirements +# run: yum update -y && yum install -y git python3-devel python3-numpy python3-pytest make environment-modules +# +# - name: Install CMake with pip +# run: | +# python3 -m pip install --upgrade pip +# python3 -m pip install cmake --prefer-binary +# +# - name: Install NVidia HPC SDK +# run: > +# yum -y install +# https://developer.download.nvidia.com/hpc-sdk/20.11/nvhpc-20-11-20.11-1.x86_64.rpm +# https://developer.download.nvidia.com/hpc-sdk/20.11/nvhpc-2020-20.11-1.x86_64.rpm +# +# - name: Configure +# shell: bash +# run: | +# source /etc/profile.d/modules.sh +# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.11 +# cmake -S . -B build -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") +# +# - name: Build +# run: cmake --build build -j 2 --verbose +# +# - name: Python tests +# run: cmake --build build --target pytest +# +# - name: C++ tests +# run: cmake --build build --target cpptest +# +# - name: Interface test +# run: cmake --build build --target test_cmake_build + + + # # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds + # centos-nvhpc7: + # runs-on: ubuntu-latest + # name: "🐍 3 • CentOS7 / PGI 20.9 • x64" + # container: centos:7 + + # steps: + # - uses: actions/checkout@v2 + + # - name: Add Python 3 and a few requirements + # run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 + + # - name: Install NVidia HPC SDK + # run: yum -y install https://developer.download.nvidia.com/hpc-sdk/20.9/nvhpc-20-9-20.9-1.x86_64.rpm https://developer.download.nvidia.com/hpc-sdk/20.9/nvhpc-2020-20.9-1.x86_64.rpm + + # # On CentOS 7, we have to filter a few tests (compiler internal error) + # # and allow deeper templete recursion (not needed on CentOS 8 with a newer + # # standard library). On some systems, you many need further workarounds: + # # https://github.com/pybind/pybind11/pull/2475 + # - name: Configure + # shell: bash + # run: | + # source /etc/profile.d/modules.sh + # module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.9 + # cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ + # -DCMAKE_CXX_STANDARD=11 \ + # -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ + # -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ + # -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + + # # Building before installing Pip should produce a warning but not an error + # - name: Build + # run: cmake3 --build build -j 2 --verbose + + # - name: Install CMake with pip + # run: | + # python3 -m pip install --upgrade pip + # python3 -m pip install pytest + + # - name: Python tests + # run: cmake3 --build build --target pytest + + # - name: C++ tests + # run: cmake3 --build build --target cpptest + + # - name: Interface test + # run: cmake3 --build build --target test_cmake_build + + + # Testing on GCC using the GCC docker images (only recent images supported) gcc: runs-on: ubuntu-latest strategy: @@ -187,8 +421,13 @@ jobs: gcc: - 7 - latest + std: + - 11 + include: + - gcc: 10 + std: 20 - name: "🐍 3 • GCC ${{ matrix.gcc }} • x64" + name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" container: "gcc:${{ matrix.gcc }}" steps: @@ -200,10 +439,8 @@ jobs: - name: Update pip run: python3 -m pip install --upgrade pip - - name: Setup CMake 3.18 - uses: jwlawson/actions-setup-cmake@v1.3 - with: - cmake-version: 3.18 + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.8 - name: Configure shell: bash @@ -211,7 +448,7 @@ jobs: cmake -S . -B build -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON - -DCMAKE_CXX_STANDARD=11 + -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") - name: Build @@ -226,6 +463,106 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + + # # Testing on ICC using the oneAPI apt repo + # icc: + # runs-on: ubuntu-20.04 + # strategy: + # fail-fast: false + + # name: "🐍 3 • ICC latest • x64" + + # steps: + # - uses: actions/checkout@v2 + + # - name: Add apt repo + # run: | + # sudo apt-get update + # sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg + # wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB + # sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB + # echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list + + # - name: Add ICC & Python 3 + # run: sudo apt-get update; sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic cmake python3-dev python3-numpy python3-pytest python3-pip + + # - name: Update pip + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # python3 -m pip install --upgrade pip + + # - name: Install dependencies + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # python3 -m pip install -r tests/requirements.txt --prefer-binary + + # - name: Configure C++11 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake -S . -B build-11 \ + # -DPYBIND11_WERROR=ON \ + # -DDOWNLOAD_CATCH=ON \ + # -DDOWNLOAD_EIGEN=OFF \ + # -DCMAKE_CXX_STANDARD=11 \ + # -DCMAKE_CXX_COMPILER=$(which icpc) \ + # -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + # - name: Build C++11 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake --build build-11 -j 2 -v + + # - name: Python tests C++11 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # sudo service apport stop + # cmake --build build-11 --target check + + # - name: C++ tests C++11 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake --build build-11 --target cpptest + + # - name: Interface test C++11 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake --build build-11 --target test_cmake_build + + # - name: Configure C++17 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake -S . -B build-17 \ + # -DPYBIND11_WERROR=ON \ + # -DDOWNLOAD_CATCH=ON \ + # -DDOWNLOAD_EIGEN=OFF \ + # -DCMAKE_CXX_STANDARD=17 \ + # -DCMAKE_CXX_COMPILER=$(which icpc) \ + # -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + # - name: Build C++17 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake --build build-17 -j 2 -v + + # - name: Python tests C++17 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # sudo service apport stop + # cmake --build build-17 --target check + + # - name: C++ tests C++17 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake --build build-17 --target cpptest + + # - name: Interface test C++17 + # run: | + # set +e; source /opt/intel/oneapi/setvars.sh; set -e + # cmake --build build-17 --target test_cmake_build + + + # # Testing on CentOS (manylinux uses a centos base, and this is an easy way + # # to get GCC 4.8, which is the manylinux1 compiler). # centos: # runs-on: ubuntu-latest # strategy: @@ -248,7 +585,7 @@ jobs: # run: python3 -m pip install --upgrade pip # - name: Install dependencies - # run: python3 -m pip install cmake -r tests/requirements.txt + # run: python3 -m pip install cmake -r tests/requirements.txt --prefer-binary # - name: Configure # shell: bash @@ -272,65 +609,237 @@ jobs: # - name: Interface test # run: cmake --build build --target test_cmake_build - install-classic: - name: "🐍 3.5 • Debian • x86 • Install" - runs-on: ubuntu-latest - container: i386/debian:stretch - steps: - - uses: actions/checkout@v1 + # # This tests an "install" with the CMake tools + # install-classic: + # name: "🐍 3.5 • Debian • x86 • Install" + # runs-on: ubuntu-latest + # container: i386/debian:stretch - - name: Install requirements - run: | - apt-get update - apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip python3-pytest + # steps: + # - uses: actions/checkout@v1 + + # - name: Install requirements + # run: | + # apt-get update + # apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip + # pip3 install "pytest==3.1.*" + + # - name: Configure for install + # run: > + # cmake . + # -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 + # -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") - - name: Configure for install - run: cmake -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 . + # - name: Make and install + # run: make install - - name: Make and install - run: make install + # - name: Copy tests to new directory + # run: cp -a tests /pybind11-tests - - name: Copy tests to new directory - run: cp -a tests /pybind11-tests + # - name: Make a new test directory + # run: mkdir /build-tests - - name: Make a new test directory - run: mkdir /build-tests + # - name: Configure tests + # run: > + # cmake ../pybind11-tests + # -DDOWNLOAD_CATCH=ON + # -DPYBIND11_WERROR=ON + # -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + # working-directory: /build-tests - - name: Configure tests - run: cmake -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") ../pybind11-tests -DPYBIND11_WERROR=ON - working-directory: /build-tests + # - name: Run tests + # run: make pytest -j 2 + # working-directory: /build-tests - - name: Run tests - run: make pytest -j 2 - working-directory: /build-tests + # # This verifies that the documentation is not horribly broken, and does a + # # basic sanity check on the SDist. + # doxygen: + # name: "Documentation build test" + # runs-on: ubuntu-latest - doxygen: - name: "Documentation build test" - runs-on: ubuntu-latest - container: alpine:3.12 + # steps: + # - uses: actions/checkout@v2 - steps: - - uses: actions/checkout@v2 + # - uses: actions/setup-python@v2 - - name: Install requirements - run: apk add doxygen python3-dev + # - name: Install Doxygen + # run: sudo apt-get install -y doxygen librsvg2-bin # Changed to rsvg-convert in 20.04 - - name: Ensure pip - run: python3 -m ensurepip + # - name: Install docs & setup requirements + # run: python3 -m pip install -r docs/requirements.txt - - name: Install python docs requirements - run: python3 -m pip install "sphinx<3" sphinx_rtd_theme breathe==4.13.1 pytest setuptools + # - name: Build docs + # run: python3 -m sphinx -W -b html docs docs/.build - - name: Build docs - run: python3 -m sphinx -W -b html docs docs/.build + # - name: Make SDist + # run: python3 setup.py sdist - - name: Make SDist - run: python3 setup.py sdist + # - run: git status --ignored - - name: Compare Dists (headers only) - run: | - python3 -m pip install --user -U ./dist/* - installed=$(python3 -c "import pybind11; print(pybind11.get_include(True) + '/pybind11')") - diff -rq $installed ./include/pybind11 + # - name: Check local include dir + # run: > + # ls pybind11; + # python3 -c "import pybind11, pathlib; assert (a := pybind11.get_include()) == (b := str(pathlib.Path('include').resolve())), f'{a} != {b}'" + + # - name: Compare Dists (headers only) + # working-directory: include + # run: | + # python3 -m pip install --user -U ../dist/* + # installed=$(python3 -c "import pybind11; print(pybind11.get_include() + '/pybind11')") + # diff -rq $installed ./pybind11 + + # win32: + # strategy: + # fail-fast: false + # matrix: + # python: + # - 3.5 + # - 3.6 + # - 3.7 + # - 3.8 + # - 3.9 + # - pypy3 + # # TODO: fix hang on pypy2 + + # include: + # - python: 3.9 + # args: -DCMAKE_CXX_STANDARD=20 -DDOWNLOAD_EIGEN=OFF + # - python: 3.8 + # args: -DCMAKE_CXX_STANDARD=17 + + # name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}" + # runs-on: windows-latest + + # steps: + # - uses: actions/checkout@v2 + + # - name: Setup Python ${{ matrix.python }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python }} + # architecture: x86 + + # - name: Update CMake + # uses: jwlawson/actions-setup-cmake@v1.7 + + # - name: Prepare MSVC + # uses: ilammy/msvc-dev-cmd@v1 + # with: + # arch: x86 + + # - name: Prepare env + # run: python -m pip install -r tests/requirements.txt --prefer-binary + + # # First build - C++11 mode and inplace + # - name: Configure ${{ matrix.args }} + # run: > + # cmake -S . -B build + # -G "Visual Studio 16 2019" -A Win32 + # -DPYBIND11_WERROR=ON + # -DDOWNLOAD_CATCH=ON + # -DDOWNLOAD_EIGEN=ON + # ${{ matrix.args }} + # - name: Build C++11 + # run: cmake --build build -j 2 + + # - name: Run tests + # run: cmake --build build -t pytest + + # win32-msvc2015: + # name: "🐍 ${{ matrix.python }} • MSVC 2015 • x64" + # runs-on: windows-latest + # strategy: + # fail-fast: false + # matrix: + # python: + # - 2.7 + # - 3.6 + # - 3.7 + # # todo: check/cpptest does not support 3.8+ yet + + # steps: + # - uses: actions/checkout@v2 + + # - name: Setup 🐍 ${{ matrix.python }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python }} + + # - name: Update CMake + # uses: jwlawson/actions-setup-cmake@v1.7 + + # - name: Prepare MSVC + # uses: ilammy/msvc-dev-cmd@v1 + # with: + # toolset: 14.0 + + # - name: Prepare env + # run: python -m pip install -r tests/requirements.txt --prefer-binary + + # # First build - C++11 mode and inplace + # - name: Configure + # run: > + # cmake -S . -B build + # -G "Visual Studio 14 2015" -A x64 + # -DPYBIND11_WERROR=ON + # -DDOWNLOAD_CATCH=ON + # -DDOWNLOAD_EIGEN=ON + + # - name: Build C++14 + # run: cmake --build build -j 2 + + # - name: Run all checks + # run: cmake --build build -t check + + + # win32-msvc2017: + # name: "🐍 ${{ matrix.python }} • MSVC 2017 • x64" + # runs-on: windows-2016 + # strategy: + # fail-fast: false + # matrix: + # python: + # - 2.7 + # - 3.5 + # - 3.7 + # std: + # - 14 + + # include: + # - python: 2.7 + # std: 17 + # args: > + # -DCMAKE_CXX_FLAGS="/permissive- /EHsc /GR" + + # steps: + # - uses: actions/checkout@v2 + + # - name: Setup 🐍 ${{ matrix.python }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python }} + + # - name: Update CMake + # uses: jwlawson/actions-setup-cmake@v1.7 + + # - name: Prepare env + # run: python -m pip install -r tests/requirements.txt --prefer-binary + + # # First build - C++11 mode and inplace + # - name: Configure + # run: > + # cmake -S . -B build + # -G "Visual Studio 15 2017" -A x64 + # -DPYBIND11_WERROR=ON + # -DDOWNLOAD_CATCH=ON + # -DDOWNLOAD_EIGEN=ON + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # ${{ matrix.args }} + + # - name: Build ${{ matrix.std }} + # run: cmake --build build -j 2 + + # - name: Run all checks + # run: cmake --build build -t check diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 96904341de..77ee8cad8c 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -1,78 +1,82 @@ -name: Configure +name: Config on: workflow_dispatch: pull_request: push: branches: - - master - - stable - - v* + - drake jobs: + # This tests various versions of CMake in various combinations, to make sure + # the configure step passes. cmake: strategy: fail-fast: false matrix: - python: - - 2.7 - - 3.8 + runs-on: [ubuntu-latest, macos-latest, windows-latest] + arch: [x64] + cmake: [3.18] - name: CMake ${{ matrix.cmake }} Python ${{ matrix.python }} on ubuntu - runs-on: ubuntu-latest + include: + - runs-on: ubuntu-latest + arch: x64 + cmake: 3.4 + + - runs-on: macos-latest + arch: x64 + cmake: 3.7 + + - runs-on: windows-2016 + arch: x86 + cmake: 3.8 + + - runs-on: windows-2016 + arch: x86 + cmake: 3.18 + + name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} + runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v2 - - name: Setup Python ${{ matrix.python }} + - name: Setup Python 3.7 uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python }} + python-version: 3.7 + architecture: ${{ matrix.arch }} - name: Prepare env run: python -m pip install -r tests/requirements.txt - - name: Make build directories - run: | - mkdir build3.7 - mkdir build3.11 - mkdir build3.18 - - - name: Setup CMake 3.7 - uses: jwlawson/actions-setup-cmake@v1.3 + # An action for adding a specific version of CMake: + # https://github.com/jwlawson/actions-setup-cmake + - name: Setup CMake ${{ matrix.cmake }} + uses: jwlawson/actions-setup-cmake@v1.8 with: - cmake-version: 3.7 - - - name: Configure 3.7 - working-directory: build3.7 - run: > - cmake .. - -DPYBIND11_WERROR=ON - -DDOWNLOAD_CATCH=ON - -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + cmake-version: ${{ matrix.cmake }} - - name: Setup CMake 3.11 - uses: jwlawson/actions-setup-cmake@v1.3 - with: - cmake-version: 3.11 + # These steps use a directory with a space in it intentionally + - name: Make build directories + run: mkdir "build dir" - - name: Configure 3.11 - working-directory: build3.11 + - name: Configure + working-directory: build dir + shell: bash run: > cmake .. -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") - - name: Setup CMake 3.18 - uses: jwlawson/actions-setup-cmake@v1.3 - with: - cmake-version: 3.18 + # Only build and test if this was manually triggered in the GitHub UI + - name: Build + working-directory: build dir + if: github.event_name == 'workflow_dispatch' + run: cmake --build . --config Release - - name: Configure 3.18 - working-directory: build3.18 - run: > - cmake .. - -DPYBIND11_WERROR=ON - -DDOWNLOAD_CATCH=ON - -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + - name: Test + working-directory: build dir + if: github.event_name == 'workflow_dispatch' + run: cmake --build . --config Release --target check diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index e92f96e6ef..1fb9a4e18a 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -1,19 +1,45 @@ -name: Format +# # This is a format job. Pre-commit has a first-party GitHub action, so we use +# # that: https://github.com/pre-commit/action -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - - stable - - "v*" +# name: Format -jobs: - pre-commit: - name: Format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 +# on: +# workflow_dispatch: +# pull_request: +# push: +# branches: +# - drake + +# jobs: +# pre-commit: +# name: Format +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# - uses: actions/setup-python@v2 +# - uses: pre-commit/action@v2.0.0 +# with: +# # Slow hooks are marked with manual - slow is okay here, run them too +# extra_args: --hook-stage manual --all-files + +# # TODO(eric.cousineau): This currently breaks current source code. +# clang-tidy: +# name: Clang-Tidy +# runs-on: ubuntu-latest +# container: silkeh/clang:10 +# steps: +# - uses: actions/checkout@v2 + +# - name: Install requirements +# run: apt-get update && apt-get install -y python3-dev python3-pytest + +# - name: Configure +# run: > +# cmake -S . -B build +# -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--warnings-as-errors=*" +# -DDOWNLOAD_EIGEN=ON +# -DDOWNLOAD_CATCH=ON +# -DCMAKE_CXX_STANDARD=17 + +# - name: Build +# run: cmake --build build -j 2 diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000..8362a18cfd --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,16 @@ +# name: Labeler +# on: +# pull_request_target: +# types: [closed] + +# jobs: +# label: +# name: Labeler +# runs-on: ubuntu-latest +# steps: + +# - uses: actions/labeler@main +# if: github.event.pull_request.merged == true +# with: +# repo-token: ${{ secrets.GITHUB_TOKEN }} +# configuration-path: .github/labeler_merged.yml diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml new file mode 100644 index 0000000000..5b2947dcb1 --- /dev/null +++ b/.github/workflows/pip.yml @@ -0,0 +1,103 @@ +# name: Pip + +# on: +# workflow_dispatch: +# pull_request: +# push: +# branches: [] +# # - master +# # - stable +# # - v* +# release: +# types: +# - published + +# # jobs: +# # # This builds the sdists and wheels and makes sure the files are exactly as +# # # expected. Using Windows and Python 2.7, since that is often the most +# # # challenging matrix element. +# # test-packaging: +# # name: 🐍 2.7 • 📦 tests • windows-latest +# # runs-on: windows-latest + +# # steps: +# # - uses: actions/checkout@v2 + +# # - name: Setup 🐍 2.7 +# # uses: actions/setup-python@v2 +# # with: +# # python-version: 2.7 + +# # - name: Prepare env +# # run: python -m pip install -r tests/requirements.txt --prefer-binary + +# # - name: Python Packaging tests +# # run: pytest tests/extra_python_package/ + + +# # # This runs the packaging tests and also builds and saves the packages as +# # # artifacts. +# # packaging: +# # name: 🐍 3.8 • 📦 & 📦 tests • ubuntu-latest +# # runs-on: ubuntu-latest + +# # steps: +# # - uses: actions/checkout@v2 + +# # - name: Setup 🐍 3.8 +# # uses: actions/setup-python@v2 +# # with: +# # python-version: 3.8 + +# # - name: Prepare env +# # run: python -m pip install -r tests/requirements.txt build twine --prefer-binary + +# # - name: Python Packaging tests +# # run: pytest tests/extra_python_package/ + +# # - name: Build SDist and wheels +# # run: | +# # python -m build +# # PYBIND11_GLOBAL_SDIST=1 python -m build + +# # - name: Check metadata +# # run: twine check dist/* + +# # - name: Save standard package +# # uses: actions/upload-artifact@v2 +# # with: +# # name: standard +# # path: dist/pybind11-* + +# # - name: Save global package +# # uses: actions/upload-artifact@v2 +# # with: +# # name: global +# # path: dist/pybind11_global-* + + + +# # # When a GitHub release is made, upload the artifacts to PyPI +# # upload: +# # name: Upload to PyPI +# # runs-on: ubuntu-latest +# # if: github.event_name == 'release' && github.event.action == 'published' +# # needs: [packaging] + +# # steps: +# # - uses: actions/setup-python@v2 + +# # # Downloads all to directories matching the artifact names +# # - uses: actions/download-artifact@v2 + +# - name: Publish standard package +# uses: pypa/gh-action-pypi-publish@v1.4.2 +# with: +# password: ${{ secrets.pypi_password }} +# packages_dir: standard/ + +# - name: Publish global package +# uses: pypa/gh-action-pypi-publish@v1.4.2 +# with: +# password: ${{ secrets.pypi_password_global }} +# packages_dir: global/ diff --git a/.gitignore b/.gitignore index 54a1f92601..3f36b89e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,9 +32,12 @@ MANIFEST .*.swp .DS_Store /dist -/build +/*build* .cache/ sosize-*.txt pybind11Config*.cmake pybind11Targets.cmake /*env* +/.vscode +/pybind11/include/* +/pybind11/share/* diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 4d698f93f8..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "tools/clang"] - path = tools/clang - url = ../../wjakob/clang-cindex-python3.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7fb3a13b70..6781ac4f11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,21 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://github.com/pre-commit/pre-commit + repos: +# Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.4.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -14,23 +29,66 @@ repos: - id: trailing-whitespace - id: fix-encoding-pragma +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + # By default, this ignores pyi files, though black supports them + types: [text] + files: \.pyi?$ + +# Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.9 hooks: - id: remove-tabs +# Flake8 also supports pre-commit natively (same author) - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: [flake8-bugbear, pep8-naming] exclude: ^(docs/.*|tools/.*)$ +# CMake formatting - repo: https://github.com/cheshirekow/cmake-format-precommit - rev: v0.6.10 + rev: v0.6.13 hooks: - id: cmake-format additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.800 + hooks: + - id: mypy + # The default Python type ignores .pyi files, so let's rerun if detected + types: [text] + files: ^pybind11.*\.pyi?$ + # Running per-file misbehaves a bit, so just run on all files, it's fast + pass_filenames: false + +# Checks the manifest for missing files (native support) +- repo: https://github.com/mgedmin/check-manifest + rev: "0.46" + hooks: + - id: check-manifest + # This is a slow hook, so only run this if --hook-stage manual is passed + stages: [manual] + additional_dependencies: [cmake, ninja] + +# The original pybind11 checks for a few C++ style items +- repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache + exclude: .pre-commit-config.yaml - repo: local hooks: diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aa8ed72c9..152763eb42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,11 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.4) -# VERSION 3.7...3.18, but some versions of MCVS have a patched CMake 3.11 -# that do not work properly with this syntax, so using the following workaround: +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: if(${CMAKE_VERSION} VERSION_LESS 3.18) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() @@ -25,10 +26,10 @@ foreach(ver ${pybind11_version_defines}) endif() endforeach() -if(PYBIND11_VERSION_PATCH MATCHES [[([a-zA-Z]+)]]) - set(PYBIND11_VERSION_TYPE "${CMAKE_MATCH_1}") +if(PYBIND11_VERSION_PATCH MATCHES [[\.([a-zA-Z0-9]+)$]]) + set(pybind11_VERSION_TYPE "${CMAKE_MATCH_1}") endif() -string(REGEX MATCH "[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") +string(REGEX MATCH "^[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") project( pybind11 @@ -40,30 +41,63 @@ include(GNUInstallDirs) include(CMakePackageConfigHelpers) include(CMakeDependentOption) -message(STATUS "pybind11 v${pybind11_VERSION} ${PYBIND11_VERSION_TYPE}") +if(NOT pybind11_FIND_QUIETLY) + message(STATUS "pybind11 v${pybind11_VERSION} ${pybind11_VERSION_TYPE}") +endif() + +# Avoid infinite recursion if tests include this as a subdirectory +if(DEFINED PYBIND11_MASTER_PROJECT) + set(PYBIND11_TEST OFF) +endif() # Check if pybind11 is being used directly or via add_subdirectory -if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR AND NOT DEFINED PYBIND11_MASTER_PROJECT) + ### Warn if not an out-of-source builds + if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + set(lines + "You are building in-place. If that is not what you intended to " + "do, you can clean the source directory with:\n" + "rm -r CMakeCache.txt CMakeFiles/ cmake_uninstall.cmake pybind11Config.cmake " + "pybind11ConfigVersion.cmake tests/CMakeFiles/\n") + message(AUTHOR_WARNING ${lines}) + endif() + set(PYBIND11_MASTER_PROJECT ON) + + if(OSX AND CMAKE_VERSION VERSION_LESS 3.7) + # Bug in macOS CMake < 3.7 is unable to download catch + message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended") + elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8) + # Only tested with 3.8+ in CI. + message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested") + endif() + message(STATUS "CMake ${CMAKE_VERSION}") if(CMAKE_CXX_STANDARD) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) endif() + + set(pybind11_system "") else() set(PYBIND11_MASTER_PROJECT OFF) set(pybind11_system SYSTEM) endif() +# Options option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) -option(PYBIND11_CLASSIC_LTO "Use the classic LTO flag algorithm, even on CMake 3.9+" OFF) +option(PYBIND11_NOPYTHON "Disable search for Python" OFF) + cmake_dependent_option( USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" OFF "PYBIND11_INSTALL" OFF) +cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF + "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) + # NB: when adding a header don't forget to also add it to setup.py set(PYBIND11_HEADERS include/pybind11/detail/class.h @@ -71,6 +105,7 @@ set(PYBIND11_HEADERS include/pybind11/detail/descr.h include/pybind11/detail/init.h include/pybind11/detail/internals.h + include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h include/pybind11/attr.h include/pybind11/buffer_info.h @@ -82,6 +117,7 @@ set(PYBIND11_HEADERS include/pybind11/eigen.h include/pybind11/embed.h include/pybind11/eval.h + include/pybind11/gil.h include/pybind11/iostream.h include/pybind11/functional.h include/pybind11/numpy.h @@ -115,112 +151,55 @@ endif() string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS "${PYBIND11_HEADERS}") -# Classic mode - -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools") -include(pybind11Tools) - -# Cache variables so pybind11_add_module can be used in parent projects -set(PYBIND11_INCLUDE_DIR +# Cache variable so this can be used in parent projects +set(pybind11_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/include" - CACHE INTERNAL "") -set(PYTHON_INCLUDE_DIRS - ${PYTHON_INCLUDE_DIRS} - CACHE INTERNAL "") -set(PYTHON_LIBRARIES - ${PYTHON_LIBRARIES} - CACHE INTERNAL "") -set(PYTHON_MODULE_PREFIX - ${PYTHON_MODULE_PREFIX} - CACHE INTERNAL "") -set(PYTHON_MODULE_EXTENSION - ${PYTHON_MODULE_EXTENSION} - CACHE INTERNAL "") -set(PYTHON_VERSION_MAJOR - ${PYTHON_VERSION_MAJOR} - CACHE INTERNAL "") -set(PYTHON_VERSION_MINOR - ${PYTHON_VERSION_MINOR} - CACHE INTERNAL "") -set(PYTHON_IS_DEBUG - "${PYTHON_IS_DEBUG}" - CACHE INTERNAL "") - -if(PYBIND11_TEST OR (BUILD_TESTING AND PYBIND11_MASTER_PROJECT)) - add_subdirectory(tests) -endif() + CACHE INTERNAL "Directory where pybind11 headers are located") -if(USE_PYTHON_INCLUDE_DIR) - file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) +# Backward compatible variable for add_subdirectory mode +if(NOT PYBIND11_MASTER_PROJECT) + set(PYBIND11_INCLUDE_DIR + "${pybind11_INCLUDE_DIR}" + CACHE INTERNAL "") endif() # Note: when creating targets, you cannot use if statements at configure time - # you need generator expressions, because those will be placed in the target file. # You can also place ifs *in* the Config.in, but not here. -# Build an interface library target: -add_library(pybind11 INTERFACE) -add_library(pybind11::pybind11 ALIAS pybind11) # to match exported target - -target_include_directories( - pybind11 ${pybind11_system} INTERFACE $ - $) -# Only add Python for build - must be added during the import for config since it has to be re-discovered. -target_include_directories(pybind11 SYSTEM INTERFACE $) - -if(CMAKE_VERSION VERSION_LESS 3.13) - target_compile_features(pybind11 INTERFACE cxx_inheriting_constructors cxx_user_literals - cxx_right_angle_brackets) +# This section builds targets, but does *not* touch Python +# Non-IMPORT targets cannot be defined twice +if(NOT TARGET pybind11_headers) + # Build the headers-only target (no Python included): + # (long name used here to keep this from clashing in subdirectory mode) + add_library(pybind11_headers INTERFACE) + add_library(pybind11::pybind11_headers ALIAS pybind11_headers) # to match exported target + add_library(pybind11::headers ALIAS pybind11_headers) # easier to use/remember + + target_include_directories( + pybind11_headers ${pybind11_system} INTERFACE $ + $) + + target_compile_features(pybind11_headers INTERFACE cxx_inheriting_constructors cxx_user_literals + cxx_right_angle_brackets) else() - # This was added in CMake 3.8, but we are keeping a consistent breaking - # point for the config file at 3.13. A config generated by CMake 3.13+ - # can only be read in 3.13+ due to the SHELL usage later, so this is safe to do. - target_compile_features(pybind11 INTERFACE cxx_std_11) + # It is invalid to install a target twice, too. + set(PYBIND11_INSTALL OFF) endif() -add_library(module INTERFACE) -add_library(pybind11::module ALIAS module) - -target_link_libraries(module INTERFACE pybind11::pybind11) - -# See https://github.com/Kitware/CMake/blob/master/Modules/CMakePlatformId.h.in for platform IDs -# Note: CMake 3.15 allows $ -target_link_libraries( - module - INTERFACE - "$<$,$>:$>") +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") -if(CMAKE_VERSION VERSION_LESS 3.13) - target_link_libraries(module INTERFACE "$<$:-undefined dynamic_lookup>") -else() - # SHELL (3.12+) forces this to remain together, and link_options was added in 3.13+ - # This is safer, because you are ensured the deduplication pass in CMake will not consider - # these separate and remove one but not the other. - target_link_options(module INTERFACE "$<$:SHELL:-undefined dynamic_lookup>") +# Relative directory setting +if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${Python_INCLUDE_DIRS}) +elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED PYTHON_INCLUDE_DIR) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) endif() -# Workaround for Python 2.7 and C++17 (C++14 as a warning) incompatibility -# This adds the flags -Wno-register and -Wno-deprecated-register if the compiler -# is Clang 3.9+ or AppleClang and the compile language is CXX, or /wd5033 for MSVC (all languages, -# since MSVC didn't recognize COMPILE_LANGUAGE until CMake 3.11+). -set(clang_4plus - "$,$,3.9>>>") -set(no_register "$>") -set(cxx_no_register "$,${no_register}>") -set(msvc "$") -target_compile_options( - pybind11 INTERFACE "$<${cxx_no_register}:-Wno-register;-Wno-deprecated-register>" - "$<${msvc}:/wd5033>") - -add_library(embed INTERFACE) -add_library(pybind11::embed ALIAS embed) -target_link_libraries(embed INTERFACE pybind11::pybind11 $) - if(PYBIND11_INSTALL) - install(DIRECTORY ${PYBIND11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - # GNUInstallDirs "DATADIR" wrong here; CMake search path wants "share". + install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) set(PYBIND11_CMAKECONFIG_INSTALL_DIR - "share/cmake/${PROJECT_NAME}" + "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}" CACHE STRING "install path for pybind11Config.cmake") configure_package_config_file( @@ -250,14 +229,17 @@ if(PYBIND11_INSTALL) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - tools/FindPythonLibsNew.cmake tools/pybind11Tools.cmake + tools/FindPythonLibsNew.cmake + tools/pybind11Common.cmake + tools/pybind11Tools.cmake + tools/pybind11NewTools.cmake DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) if(NOT PYBIND11_EXPORT_NAME) set(PYBIND11_EXPORT_NAME "${PROJECT_NAME}Targets") endif() - install(TARGETS pybind11 module embed EXPORT "${PYBIND11_EXPORT_NAME}") + install(TARGETS pybind11_headers EXPORT "${PYBIND11_EXPORT_NAME}") install( EXPORT "${PYBIND11_EXPORT_NAME}" @@ -273,3 +255,29 @@ if(PYBIND11_INSTALL) ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) endif() endif() + +# BUILD_TESTING takes priority, but only if this is the master project +if(PYBIND11_MASTER_PROJECT AND DEFINED BUILD_TESTING) + if(BUILD_TESTING) + if(_pybind11_nopython) + message(FATAL_ERROR "Cannot activate tests in NOPYTHON mode") + else() + add_subdirectory(tests) + endif() + endif() +else() + if(PYBIND11_TEST) + if(_pybind11_nopython) + message(FATAL_ERROR "Cannot activate tests in NOPYTHON mode") + else() + add_subdirectory(tests) + endif() + endif() +endif() + +# Better symmetry with find_package(pybind11 CONFIG) mode. +if(NOT PYBIND11_MASTER_PROJECT) + set(pybind11_FOUND + TRUE + CACHE INTERNAL "True if pybind11 and all required components found on the system") +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 9dd3bd6f6c..0000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,53 +0,0 @@ -Thank you for your interest in this project! Please refer to the following -sections on how to contribute code and bug reports. - -### Reporting bugs - -At the moment, this project is run in the spare time of a single person -([Wenzel Jakob](http://rgl.epfl.ch/people/wjakob)) with very limited resources -for issue tracker tickets. Thus, before submitting a question or bug report, -please take a moment of your time and ensure that your issue isn't already -discussed in the project documentation provided at -[http://pybind11.readthedocs.org/en/latest](http://pybind11.readthedocs.org/en/latest). - -Assuming that you have identified a previously unknown problem or an important -question, it's essential that you submit a self-contained and minimal piece of -code that reproduces the problem. In other words: no external dependencies, -isolate the function(s) that cause breakage, submit matched and complete C++ -and Python snippets that can be easily compiled and run on my end. - -## Pull requests -Contributions are submitted, reviewed, and accepted using Github pull requests. -Please refer to [this -article](https://help.github.com/articles/using-pull-requests) for details and -adhere to the following rules to make the process as smooth as possible: - -* Make a new branch for every feature you're working on. -* Make small and clean pull requests that are easy to review but make sure they - do add value by themselves. -* Add tests for any new functionality and run the test suite (``make pytest``) - to ensure that no existing features break. -* Please run [``pre-commit``][pre-commit] to check your code matches the - project style. (Note that ``gawk`` is required.) Use `pre-commit run - --all-files` before committing (or use installed-mode, check pre-commit docs) - to verify your code passes before pushing to save time. -* This project has a strong focus on providing general solutions using a - minimal amount of code, thus small pull requests are greatly preferred. - -[pre-commit]: https://pre-commit.com - -### Licensing of contributions - -pybind11 is provided under a BSD-style license that can be found in the -``LICENSE`` file. By using, distributing, or contributing to this project, you -agree to the terms and conditions of this license. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to the author of this software, without -imposing a separate written license agreement for such Enhancements, then you -hereby grant the following license: a non-exclusive, royalty-free perpetual -license to install, use, modify, prepare derivative works, incorporate into -other computer software, distribute, and sublicense such enhancements or -derivative works thereof, in binary and source code form. diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md deleted file mode 100644 index 75df39981a..0000000000 --- a/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,17 +0,0 @@ -Make sure you've completed the following steps before submitting your issue -- thank you! - -1. Check if your question has already been answered in the [FAQ](http://pybind11.readthedocs.io/en/latest/faq.html) section. -2. Make sure you've read the [documentation](http://pybind11.readthedocs.io/en/latest/). Your issue may be addressed there. -3. If those resources didn't help and you only have a short question (not a bug report), consider asking in the [Gitter chat room](https://gitter.im/pybind/Lobby). -4. If you have a genuine bug report or a more complex question which is not answered in the previous items (or not suitable for chat), please fill in the details below. -5. Include a self-contained and minimal piece of code that reproduces the problem. If that's not possible, try to make the description as clear as possible. - -*After reading, remove this checklist and the template text in parentheses below.* - -## Issue description - -(Provide a short description, state the expected behavior and what actually happens.) - -## Reproducible example code - -(The code should be minimal, have no external dependencies, isolate the function(s) that cause breakage. Submit matched and complete C++ and Python snippets that can be easily compiled and run to diagnose the issue.) diff --git a/LICENSE b/LICENSE index 6f15578cc4..e466b0dfda 100644 --- a/LICENSE +++ b/LICENSE @@ -25,5 +25,5 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Please also refer to the file CONTRIBUTING.md, which clarifies licensing of +Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of external contributions to this project including patches, pull requests, etc. diff --git a/MANIFEST.in b/MANIFEST.in index 6e57baeeef..e43ab37711 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,6 @@ -recursive-include include/pybind11 *.h -include LICENSE README.md CONTRIBUTING.md +recursive-include pybind11/include/pybind11 *.h +recursive-include pybind11 *.py +recursive-include pybind11 py.typed +recursive-include pybind11 *.pyi +include pybind11/share/cmake/pybind11/*.cmake +include LICENSE README.rst README_DRAKE.md pyproject.toml setup.py setup.cfg diff --git a/README.md b/README.md deleted file mode 100644 index ebc57a95f1..0000000000 --- a/README.md +++ /dev/null @@ -1,145 +0,0 @@ -![pybind11 logo](https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png) - -## RobotLocomotion Fork for Drake - -This is a fork of the [official `pybind/pybind11` repository](https://github.com/pybind/pybind11/). For information specific to this fork, please see -[`README_DRAKE.md`](./README_DRAKE.md). - -All the links and badges within this document may link back to the official -repository. - -This fork's CI badge: - -[![Build Status](https://travis-ci.com/RobotLocomotion/pybind11.svg?branch=drake)](https://travis-ci.com/github/RobotLocomotion/pybind11) - -# pybind11 — Seamless operability between C++11 and Python - - - -**pybind11** is a lightweight header-only library that exposes C++ types in Python -and vice versa, mainly to create Python bindings of existing C++ code. Its -goals and syntax are similar to the excellent -[Boost.Python](http://www.boost.org/doc/libs/1_58_0/libs/python/doc/) library -by David Abrahams: to minimize boilerplate code in traditional extension -modules by inferring type information using compile-time introspection. - -The main issue with Boost.Python—and the reason for creating such a similar -project—is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. - -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations. - -Tutorial and reference documentation is provided at -[http://pybind11.readthedocs.org/en/master](http://pybind11.readthedocs.org/en/master). -A PDF version of the manual is available -[here](https://media.readthedocs.org/pdf/pybind11/master/pybind11.pdf). - -## Core features -pybind11 can map the following core C++ features to Python - -- Functions accepting and returning custom data structures per value, reference, or pointer -- Instance methods and static methods -- Overloaded functions -- Instance attributes and static attributes -- Arbitrary exception types -- Enumerations -- Callbacks -- Iterators and ranges -- Custom operators -- Single and multiple inheritance -- STL data structures -- Smart pointers with reference counting like ``std::shared_ptr`` -- Internal references with correct reference counting -- C++ classes with virtual (and pure virtual) methods can be extended in Python - -## Goodies -In addition to the core functionality, pybind11 provides some extra goodies: - -- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.7) are supported with an - implementation-agnostic interface. - -- It is possible to bind C++11 lambda functions with captured variables. The - lambda capture data is stored inside the resulting Python function object. - -- pybind11 uses C++11 move constructors and move assignment operators whenever - possible to efficiently transfer custom data types. - -- It's easy to expose the internal storage of custom data types through - Pythons' buffer protocols. This is handy e.g. for fast conversion between - C++ matrix classes like Eigen and NumPy without expensive copy operations. - -- pybind11 can automatically vectorize functions so that they are transparently - applied to all entries of one or more NumPy array arguments. - -- Python's slice-based access and assignment operations can be supported with - just a few lines of code. - -- Everything is contained in just a few header files; there is no need to link - against any additional libraries. - -- Binaries are generally smaller by a factor of at least 2 compared to - equivalent bindings generated by Boost.Python. A recent pybind11 conversion - of PyRosetta, an enormous Boost.Python binding project, - [reported](http://graylab.jhu.edu/RosettaCon2016/PyRosetta-4.pdf) a binary - size reduction of **5.4x** and compile time reduction by **5.8x**. - -- Function signatures are precomputed at compile time (using ``constexpr``), - leading to smaller binaries. - -- With little extra effort, C++ types can be pickled and unpickled similar to - regular Python objects. - -## Supported compilers - -1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) -2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 Update 3 or newer -4. Intel C++ compiler 17 or newer (16 with pybind11 v2.0 and 15 with pybind11 v2.0 and a [workaround](https://github.com/pybind/pybind11/issues/276)) -5. Cygwin/GCC (tested on 2.5.1) - -## About - -This project was created by [Wenzel Jakob](http://rgl.epfl.ch/people/wjakob). -Significant features and/or improvements to the code were contributed by -Jonas Adler, -Lori A. Burns, -Sylvain Corlay, -Trent Houliston, -Axel Huebl, -@hulucc, -Sergey Lyskov -Johan Mabille, -Tomasz Miąsko, -Dean Moldovan, -Ben Pritchard, -Jason Rhinelander, -Boris Schäling, -Pim Schellart, -Henry Schreiner, -Ivan Smirnov, and -Patrick Stewart. - -### License - -pybind11 is provided under a BSD-style license that can be found in the -``LICENSE`` file. By using, distributing, or contributing to this project, -you agree to the terms and conditions of this license. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000000..ec64a547f8 --- /dev/null +++ b/README.rst @@ -0,0 +1,211 @@ +RobotLocomotion Fork for Drake +============================== + +|DrakeCI| + +This is a fork of the +`official pybind/pybind11 repository `_. +For information specific to this fork, please see +`README_DRAKE.md <./README_DRAKE.md>`_. + +Some links within this fork's documentation may link back to the official +upstream repository. + +.. |DrakeCI| image:: https://github.com/RobotLocomotion/pybind11/workflows/CI/badge.svg + :target: https://github.com/RobotLocomotion/pybind11/actions + +---------- + +.. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png + :alt: pybind11 logo + +**pybind11 — Seamless operability between C++11 and Python** + +.. + # Hide upstream's badges. + |Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |CI| |Build status| + +|Repology| |PyPI package| |Conda-forge| |Python Versions| + +`Setuptools example `_ +• `Scikit-build example `_ +• `CMake example `_ + +.. start + +.. warning:: + + Combining older versions of pybind11 (< 2.6.0) with Python 3.9.0 will + trigger undefined behavior that typically manifests as crashes during + interpreter shutdown (but could also destroy your data. **You have been + warned.**) + + We recommend that you update to the latest patch release of Python (3.9.1), + which includes a `fix `_ + that resolves this problem. If you do use Python 3.9.0, please update to + the latest version of pybind11 (2.6.0 or newer), which includes a temporary + workaround specifically when Python 3.9.0 is detected at runtime. + + +**pybind11** is a lightweight header-only library that exposes C++ types +in Python and vice versa, mainly to create Python bindings of existing +C++ code. Its goals and syntax are similar to the excellent +`Boost.Python `_ +library by David Abrahams: to minimize boilerplate code in traditional +extension modules by inferring type information using compile-time +introspection. + +The main issue with Boost.Python—and the reason for creating such a +similar project—is Boost. Boost is an enormously large and complex suite +of utility libraries that works with almost every C++ compiler in +existence. This compatibility has its cost: arcane template tricks and +workarounds are necessary to support the oldest and buggiest of compiler +specimens. Now that C++11-compatible compilers are widely available, +this heavy machinery has become an excessively large and unnecessary +dependency. + +Think of this library as a tiny self-contained version of Boost.Python +with everything stripped away that isn’t relevant for binding +generation. Without comments, the core header files only require ~4K +lines of code and depend on Python (2.7 or 3.5+, or PyPy) and the C++ +standard library. This compact implementation was possible thanks to +some of the new C++11 language features (specifically: tuples, lambda +functions and variadic templates). Since its creation, this library has +grown beyond Boost.Python in many ways, leading to dramatically simpler +binding code in many common situations. + +Tutorial and reference documentation is provided at +`pybind11.readthedocs.io `_. +A PDF version of the manual is available +`here `_. +And the source code is always available at +`github.com/pybind/pybind11 `_. + + +Core features +------------- + + +pybind11 can map the following core C++ features to Python: + +- Functions accepting and returning custom data structures per value, + reference, or pointer +- Instance methods and static methods +- Overloaded functions +- Instance attributes and static attributes +- Arbitrary exception types +- Enumerations +- Callbacks +- Iterators and ranges +- Custom operators +- Single and multiple inheritance +- STL data structures +- Smart pointers with reference counting like ``std::shared_ptr`` +- Internal references with correct reference counting +- C++ classes with virtual (and pure virtual) methods can be extended + in Python + +Goodies +------- + +In addition to the core functionality, pybind11 provides some extra +goodies: + +- Python 2.7, 3.5+, and PyPy/PyPy3 7.3 are supported with an + implementation-agnostic interface. + +- It is possible to bind C++11 lambda functions with captured + variables. The lambda capture data is stored inside the resulting + Python function object. + +- pybind11 uses C++11 move constructors and move assignment operators + whenever possible to efficiently transfer custom data types. + +- It’s easy to expose the internal storage of custom data types through + Pythons’ buffer protocols. This is handy e.g. for fast conversion + between C++ matrix classes like Eigen and NumPy without expensive + copy operations. + +- pybind11 can automatically vectorize functions so that they are + transparently applied to all entries of one or more NumPy array + arguments. + +- Python’s slice-based access and assignment operations can be + supported with just a few lines of code. + +- Everything is contained in just a few header files; there is no need + to link against any additional libraries. + +- Binaries are generally smaller by a factor of at least 2 compared to + equivalent bindings generated by Boost.Python. A recent pybind11 + conversion of PyRosetta, an enormous Boost.Python binding project, + `reported `_ + a binary size reduction of **5.4x** and compile time reduction by + **5.8x**. + +- Function signatures are precomputed at compile time (using + ``constexpr``), leading to smaller binaries. + +- With little extra effort, C++ types can be pickled and unpickled + similar to regular Python objects. + +Supported compilers +------------------- + +1. Clang/LLVM 3.3 or newer (for Apple Xcode’s clang, this is 5.0.0 or + newer) +2. GCC 4.8 or newer +3. Microsoft Visual Studio 2015 Update 3 or newer +4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) +5. Cygwin/GCC (previously tested on 2.5.1) +6. NVCC (CUDA 11.0 tested in CI) +7. NVIDIA PGI (20.9 tested in CI) + +About +----- + +This project was created by `Wenzel +Jakob `_. Significant features and/or +improvements to the code were contributed by Jonas Adler, Lori A. Burns, +Sylvain Corlay, Eric Cousineau, Ralf Grosse-Kunstleve, Trent Houliston, Axel +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, +Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim +Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. + +We thank Google for a generous financial contribution to the continuous +integration infrastructure used by this project. + + +Contributing +~~~~~~~~~~~~ + +See the `contributing +guide `_ +for information on building and contributing to pybind11. + +License +~~~~~~~ + +pybind11 is provided under a BSD-style license that can be found in the +`LICENSE `_ +file. By using, distributing, or contributing to this project, you agree +to the terms and conditions of this license. + +.. |Latest Documentation Status| image:: https://readthedocs.org/projects/pybind11/badge?version=latest + :target: http://pybind11.readthedocs.org/en/latest +.. |Stable Documentation Status| image:: https://img.shields.io/badge/docs-stable-blue.svg + :target: http://pybind11.readthedocs.org/en/stable +.. |Gitter chat| image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg + :target: https://gitter.im/pybind/Lobby +.. |CI| image:: https://github.com/pybind/pybind11/workflows/CI/badge.svg + :target: https://github.com/pybind/pybind11/actions +.. |Build status| image:: https://ci.appveyor.com/api/projects/status/riaj54pn4h08xy40?svg=true + :target: https://ci.appveyor.com/project/wjakob/pybind11 +.. |PyPI package| image:: https://img.shields.io/pypi/v/pybind11.svg + :target: https://pypi.org/project/pybind11/ +.. |Conda-forge| image:: https://img.shields.io/conda/vn/conda-forge/pybind11.svg + :target: https://github.com/conda-forge/pybind11-feedstock +.. |Repology| image:: https://repology.org/badge/latest-versions/python:pybind11.svg + :target: https://repology.org/project/python:pybind11/versions +.. |Python Versions| image:: https://img.shields.io/pypi/pyversions/pybind11.svg + :target: https://pypi.org/project/pybind11/ diff --git a/docs/Doxyfile b/docs/Doxyfile index 24ece0d8db..c8562952ef 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -19,4 +19,5 @@ QUIET = YES WARNINGS = YES WARN_IF_UNDOCUMENTED = NO PREDEFINED = DOXYGEN_SHOULD_SKIP_THIS \ - PY_MAJOR_VERSION=3 + PY_MAJOR_VERSION=3 \ + PYBIND11_NOINLINE diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index e4f99ac5b0..a779444c24 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -29,9 +29,9 @@ The following Python snippet demonstrates the intended usage from the Python sid from example import print print(A()) -To register the necessary conversion routines, it is necessary to add -a partial overload to the ``pybind11::detail::type_caster`` template. -Although this is an implementation detail, adding partial overloads to this +To register the necessary conversion routines, it is necessary to add an +instantiation of the ``pybind11::detail::type_caster`` template. +Although this is an implementation detail, adding an instantiation of this type is explicitly allowed. .. code-block:: cpp diff --git a/docs/advanced/cast/eigen.rst b/docs/advanced/cast/eigen.rst index 59ba08c3c4..e01472d5ae 100644 --- a/docs/advanced/cast/eigen.rst +++ b/docs/advanced/cast/eigen.rst @@ -274,7 +274,7 @@ Vectors versus column/row matrices Eigen and numpy have fundamentally different notions of a vector. In Eigen, a vector is simply a matrix with the number of columns or rows set to 1 at -compile time (for a column vector or row vector, respectively). Numpy, in +compile time (for a column vector or row vector, respectively). NumPy, in contrast, has comparable 2-dimensional 1xN and Nx1 arrays, but *also* has 1-dimensional arrays of size N. diff --git a/docs/advanced/cast/index.rst b/docs/advanced/cast/index.rst index 724585c920..3ce9ea0286 100644 --- a/docs/advanced/cast/index.rst +++ b/docs/advanced/cast/index.rst @@ -1,3 +1,5 @@ +.. _type-conversions: + Type conversions ################ diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index e48409f025..b8622ee095 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -5,7 +5,7 @@ Automatic conversion ==================== When including the additional header file :file:`pybind11/stl.h`, conversions -between ``std::vector<>``/``std::deque<>``/``std::list<>``/``std::array<>``, +between ``std::vector<>``/``std::deque<>``/``std::list<>``/``std::array<>``/``std::valarray<>``, ``std::set<>``/``std::unordered_set<>``, and ``std::map<>``/``std::unordered_map<>`` and the Python ``list``, ``set`` and ``dict`` data structures are automatically enabled. The types ``std::pair<>`` @@ -72,6 +72,17 @@ The ``visit_helper`` specialization is not required if your ``name::variant`` pr a ``name::visit()`` function. For any other function name, the specialization must be included to tell pybind11 how to visit the variant. +.. warning:: + + When converting a ``variant`` type, pybind11 follows the same rules as when + determining which function overload to call (:ref:`overload_resolution`), and + so the same caveats hold. In particular, the order in which the ``variant``'s + alternatives are listed is important, since pybind11 will try conversions in + this order. This means that, for example, when converting ``variant``, + the ``bool`` variant will never be selected, as any Python ``bool`` is already + an ``int`` and is convertible to a C++ ``int``. Changing the order of alternatives + (and using ``variant``, in this example) provides a solution. + .. note:: pybind11 only supports the modern implementation of ``boost::variant`` @@ -157,7 +168,7 @@ the declaration before any binding code (e.g. invocations to ``class_::def()``, etc.). This macro must be specified at the top level (and outside of any namespaces), since -it instantiates a partial template overload. If your binding code consists of +it adds a template instantiation of ``type_caster``. If your binding code consists of multiple compilation units, it must be present in every file (typically via a common header) preceding any usage of ``std::vector``. Opaque types must also have a corresponding ``class_`` declaration to associate them with a name diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index b2d963dc4f..17b095f5a6 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -71,7 +71,7 @@ helper class that is defined as follows: /* Trampoline (need one for each virtual function) */ std::string go(int n_times) override { - PYBIND11_OVERLOAD_PURE( + PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ go, /* Name of function in C++ (must match Python name) */ @@ -80,10 +80,10 @@ helper class that is defined as follows: } }; -The macro :c:macro:`PYBIND11_OVERLOAD_PURE` should be used for pure virtual -functions, and :c:macro:`PYBIND11_OVERLOAD` should be used for functions which have +The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual +functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have a default implementation. There are also two alternate macros -:c:macro:`PYBIND11_OVERLOAD_PURE_NAME` and :c:macro:`PYBIND11_OVERLOAD_NAME` which +:c:macro:`PYBIND11_OVERRIDE_PURE_NAME` and :c:macro:`PYBIND11_OVERRIDE_NAME` which take a string-valued name argument between the *Parent class* and *Name of the function* slots, which defines the name of function in Python. This is required when the C++ and Python versions of the @@ -122,7 +122,7 @@ Bindings should be made against the actual class, not the trampoline helper clas Note, however, that the above is sufficient for allowing python classes to extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the -necessary steps required to providing proper overload support for inherited +necessary steps required to providing proper overriding support for inherited classes. The wrapper ``py::wrapper<>`` is optional, but is recommended as it @@ -159,8 +159,7 @@ memory for the C++ portion of the instance will be left uninitialized, which will generally leave the C++ instance in an invalid state and cause undefined behavior if the C++ instance is subsequently used. -.. versionadded:: 2.5.1 - +.. versionchanged:: 2.6 The default pybind11 metaclass will throw a ``TypeError`` when it detects that ``__init__`` was not called by a derived class. @@ -170,7 +169,7 @@ Here is an example: class Dachshund(Dog): def __init__(self, name): - Dog.__init__(self) # Without this, undefined behavior may occur if the C++ portions are referenced. + Dog.__init__(self) # Without this, a TypeError is raised. self.name = name def bark(self): return "yap!" @@ -192,15 +191,24 @@ Please take a look at the :ref:`macro_notes` before using this feature. - because in these cases there is no C++ variable to reference (the value is stored in the referenced Python variable), pybind11 provides one in - the PYBIND11_OVERLOAD macros (when needed) with static storage duration. - Note that this means that invoking the overloaded method on *any* + the PYBIND11_OVERRIDE macros (when needed) with static storage duration. + Note that this means that invoking the overridden method on *any* instance will change the referenced value stored in *all* instances of that type. - Attempts to modify a non-const reference will not have the desired effect: it will change only the static cache variable, but this change will not propagate to underlying Python instance, and the change will be - replaced the next time the overload is invoked. + replaced the next time the override is invoked. + +.. warning:: + + The :c:macro:`PYBIND11_OVERRIDE` and accompanying macros used to be called + ``PYBIND11_OVERLOAD`` up until pybind11 v2.5.0, and :func:`get_override` + used to be called ``get_overload``. This naming was corrected and the older + macro and function names may soon be deprecated, in order to reduce + confusion with overloaded functions and methods and ``py::overload_cast`` + (see :ref:`classes`). .. seealso:: @@ -248,20 +256,20 @@ override the ``name()`` method): class PyAnimal : public Animal { public: using Animal::Animal; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Animal, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, Animal, name, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Animal, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, Animal, name, ); } }; class PyDog : public Dog { public: using Dog::Dog; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD(std::string, Dog, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, Dog, name, ); } - std::string bark() override { PYBIND11_OVERLOAD(std::string, Dog, bark, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE(std::string, Dog, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, Dog, name, ); } + std::string bark() override { PYBIND11_OVERRIDE(std::string, Dog, bark, ); } }; .. note:: - Note the trailing commas in the ``PYBIND11_OVERLOAD`` calls to ``name()`` + Note the trailing commas in the ``PYBIND11_OVERIDE`` calls to ``name()`` and ``bark()``. These are needed to portably implement a trampoline for a function that does not take any arguments. For functions that take a nonzero number of arguments, the trailing comma must be omitted. @@ -276,9 +284,9 @@ declare or override any virtual methods itself: class PyHusky : public Husky { public: using Husky::Husky; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Husky, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, Husky, name, ); } - std::string bark() override { PYBIND11_OVERLOAD(std::string, Husky, bark, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Husky, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, Husky, name, ); } + std::string bark() override { PYBIND11_OVERRIDE(std::string, Husky, bark, ); } }; There is, however, a technique that can be used to avoid this duplication @@ -291,15 +299,15 @@ follows: template class PyAnimal : public AnimalBase { public: using AnimalBase::AnimalBase; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, AnimalBase, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, AnimalBase, name, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, AnimalBase, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, AnimalBase, name, ); } }; template class PyDog : public PyAnimal { public: using PyAnimal::PyAnimal; // Inherit constructors // Override PyAnimal's pure virtual go() with a non-pure one: - std::string go(int n_times) override { PYBIND11_OVERLOAD(std::string, DogBase, go, n_times); } - std::string bark() override { PYBIND11_OVERLOAD(std::string, DogBase, bark, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE(std::string, DogBase, go, n_times); } + std::string bark() override { PYBIND11_OVERRIDE(std::string, DogBase, bark, ); } }; This technique has the advantage of requiring just one trampoline method to be @@ -358,7 +366,7 @@ valid for the trampoline class but not the registered class. This is primarily for performance reasons: when the trampoline class is not needed for anything except virtual method dispatching, not initializing the trampoline class improves performance by avoiding needing to do a run-time check to see if the -inheriting python instance has an overloaded method. +inheriting python instance has an overridden method. Sometimes, however, it is useful to always initialize a trampoline class as an intermediate class that does more than just handle virtual method dispatching. @@ -389,7 +397,7 @@ references (See also :ref:`faq_reference_arguments`). Another way of solving this is to use the method body of the trampoline class to do conversions to the input and return of the Python method. -The main building block to do so is the :func:`get_overload`, this function +The main building block to do so is the :func:`get_override`, this function allows retrieving a method implemented in Python from within the trampoline's methods. Consider for example a C++ method which has the signature ``bool myMethod(int32_t& value)``, where the return indicates whether @@ -401,10 +409,10 @@ Python side by allowing the Python function to return ``None`` or an ``int``: bool MyClass::myMethod(int32_t& value) { pybind11::gil_scoped_acquire gil; // Acquire the GIL while in this scope. - // Try to look up the overloaded method on the Python side. - pybind11::function overload = pybind11::get_overload(this, "myMethod"); - if (overload) { // method is found - auto obj = overload(value); // Call the Python function. + // Try to look up the overridden method on the Python side. + pybind11::function override = pybind11::get_override(this, "myMethod"); + if (override) { // method is found + auto obj = override(value); // Call the Python function. if (py::isinstance(obj)) { // check if it returned a Python integer type value = obj.cast(); // Cast it and assign it to the value. return true; // Return true; value should be used. @@ -575,6 +583,46 @@ crucial that instances are deallocated on the C++ side to avoid memory leaks. py::class_>(m, "MyClass") .def(py::init<>()) +.. _destructors_that_call_python: + +Destructors that call Python +============================ + +If a Python function is invoked from a C++ destructor, an exception may be thrown +of type :class:`error_already_set`. If this error is thrown out of a class destructor, +``std::terminate()`` will be called, terminating the process. Class destructors +must catch all exceptions of type :class:`error_already_set` to discard the Python +exception using :func:`error_already_set::discard_as_unraisable`. + +Every Python function should be treated as *possibly throwing*. When a Python generator +stops yielding items, Python will throw a ``StopIteration`` exception, which can pass +though C++ destructors if the generator's stack frame holds the last reference to C++ +objects. + +For more information, see :ref:`the documentation on exceptions `. + +.. code-block:: cpp + + class MyClass { + public: + ~MyClass() { + try { + py::print("Even printing is dangerous in a destructor"); + py::exec("raise ValueError('This is an unraisable exception')"); + } catch (py::error_already_set &e) { + // error_context should be information about where/why the occurred, + // e.g. use __func__ to get the name of the current function + e.discard_as_unraisable(__func__); + } + } + }; + +.. note:: + + pybind11 does not support C++ destructors marked ``noexcept(false)``. + +.. versionadded:: 2.6 + .. _implicit_conversions: Implicit conversions @@ -1081,7 +1129,7 @@ described trampoline: class Trampoline : public A { public: - int foo() const override { PYBIND11_OVERLOAD(int, A, foo, ); } + int foo() const override { PYBIND11_OVERRIDE(int, A, foo, ); } }; class Publicist : public A { @@ -1125,6 +1173,8 @@ error: .. note:: This attribute is currently ignored on PyPy +.. versionadded:: 2.6 + Custom automatic downcasters ============================ @@ -1324,3 +1374,21 @@ as the `Cat` portion would have been destroyed and no longer visible for the tra meow! meow! Note that both the C++ and Python portion of ``cat`` will be destroyed once ``cage`` is destroyed. + +Accessing the type object +========================= + +You can get the type object from a C++ class that has already been registered using: + +.. code-block:: python + + py::type T_py = py::type::of(); + +You can directly use ``py::type::of(ob)`` to get the type object from any python +object, just like ``type(ob)`` in Python. + +.. note:: + + Other types, like ``py::type::of()``, do not work, see :ref:`type-conversions`. + +.. versionadded:: 2.6 diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 3930316032..dfdaad2d7f 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.0) + cmake_minimum_required(VERSION 3.4) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` @@ -108,11 +108,11 @@ The two approaches can also be combined: Importing modules ================= -Python modules can be imported using `module::import()`: +Python modules can be imported using `module_::import()`: .. code-block:: cpp - py::module sys = py::module::import("sys"); + py::module_ sys = py::module_::import("sys"); py::print(sys.attr("path")); For convenience, the current working directory is included in ``sys.path`` when @@ -128,12 +128,12 @@ embedding the interpreter. This makes it easy to import local Python files: .. code-block:: cpp - py::module calc = py::module::import("calc"); + py::module_ calc = py::module_::import("calc"); py::object result = calc.attr("add")(1, 2); int n = result.cast(); assert(n == 3); -Modules can be reloaded using `module::reload()` if the source is modified e.g. +Modules can be reloaded using `module_::reload()` if the source is modified e.g. by an external process. This can be useful in scenarios where the application imports a user defined data processing script which needs to be updated after changes by the user. Note that this function does not reload modules recursively. @@ -153,7 +153,7 @@ like any other module. namespace py = pybind11; PYBIND11_EMBEDDED_MODULE(fast_calc, m) { - // `m` is a `py::module` which is used to bind functions and classes + // `m` is a `py::module_` which is used to bind functions and classes m.def("add", [](int i, int j) { return i + j; }); @@ -162,7 +162,7 @@ like any other module. int main() { py::scoped_interpreter guard{}; - auto fast_calc = py::module::import("fast_calc"); + auto fast_calc = py::module_::import("fast_calc"); auto result = fast_calc.attr("add")(1, 2).cast(); assert(result == 3); } @@ -196,7 +196,7 @@ naturally: int main() { py::scoped_interpreter guard{}; - auto py_module = py::module::import("py_module"); + auto py_module = py::module_::import("py_module"); auto locals = py::dict("fmt"_a="{} + {} = {}", **py_module.attr("__dict__")); assert(locals["a"].cast() == 1); diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 75ad7f7f4a..7a4d6cbc6e 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -1,18 +1,24 @@ Exceptions ########## -Built-in exception translation -============================== +Built-in C++ to Python exception translation +============================================ + +When Python calls C++ code through pybind11, pybind11 provides a C++ exception handler +that will trap C++ exceptions, translate them to the corresponding Python exception, +and raise them so that Python code can handle them. -When C++ code invoked from Python throws an ``std::exception``, it is -automatically converted into a Python ``Exception``. pybind11 defines multiple -special exception classes that will map to different types of Python -exceptions: +pybind11 defines translations for ``std::exception`` and its standard +subclasses, and several special exception classes that translate to specific +Python exceptions. Note that these are not actually Python exceptions, so they +cannot be examined using the Python C API. Instead, they are pure C++ objects +that pybind11 will translate the corresponding Python exception when they arrive +at its exception handler. .. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}| +--------------------------------------+--------------------------------------+ -| C++ exception type | Python exception type | +| Exception thrown by C++ | Translated to Python exception type | +======================================+======================================+ | :class:`std::exception` | ``RuntimeError`` | +--------------------------------------+--------------------------------------+ @@ -37,25 +43,28 @@ exceptions: | | of bounds access in ``__getitem__``, | | | ``__setitem__``, etc.) | +--------------------------------------+--------------------------------------+ -| :class:`pybind11::value_error` | ``ValueError`` (used to indicate | -| | wrong value passed in | -| | ``container.remove(...)``) | -+--------------------------------------+--------------------------------------+ | :class:`pybind11::key_error` | ``KeyError`` (used to indicate out | | | of bounds access in ``__getitem__``, | | | ``__setitem__`` in dict-like | | | objects, etc.) | +--------------------------------------+--------------------------------------+ -| :class:`pybind11::error_already_set` | Indicates that the Python exception | -| | flag has already been set via Python | -| | API calls from C++ code; this C++ | -| | exception is used to propagate such | -| | a Python exception back to Python. | +| :class:`pybind11::value_error` | ``ValueError`` (used to indicate | +| | wrong value passed in | +| | ``container.remove(...)``) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::type_error` | ``TypeError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::buffer_error` | ``BufferError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::import_error` | ``import_error`` | ++--------------------------------------+--------------------------------------+ +| Any other exception | ``RuntimeError`` | +--------------------------------------+--------------------------------------+ -When a Python function invoked from C++ throws an exception, it is converted -into a C++ exception of type :class:`error_already_set` whose string payload -contains a textual summary. +Exception translation is not bidirectional. That is, *catching* the C++ +exceptions defined above above will not trap exceptions that originate from +Python. For that, catch :class:`pybind11::error_already_set`. See :ref:`below +` for further details. There is also a special exception :class:`cast_error` that is thrown by :func:`handle::call` when the input arguments cannot be converted to Python @@ -78,6 +87,19 @@ This call creates a Python exception class with the name ``PyExp`` in the given module and automatically converts any encountered exceptions of type ``CppExp`` into Python exceptions of type ``PyExp``. +It is possible to specify base class for the exception using the third +parameter, a `handle`: + +.. code-block:: cpp + + py::register_exception(module, "PyExp", PyExc_RuntimeError); + +Then `PyExp` can be caught both as `PyExp` and `RuntimeError`. + +The class objects of the built-in Python exceptions are listed in the Python +documentation on `Standard Exceptions `_. +The default base class is `PyExc_Exception`. + When more advanced exception translation is needed, the function ``py::register_exception_translator(translator)`` can be used to register functions that can translate arbitrary exception types (and which may include @@ -100,7 +122,6 @@ and use this in the associated exception translator (note: it is often useful to make this a static declaration when using it inside a lambda expression without requiring capturing). - The following example demonstrates this for a hypothetical exception classes ``MyCustomException`` and ``OtherException``: the first is translated to a custom python exception ``MyCustomError``, while the second is translated to a @@ -134,7 +155,7 @@ section. .. note:: - You must call either ``PyErr_SetString`` or a custom exception's call + Call either ``PyErr_SetString`` or a custom exception's call operator (``exc(string)``) for every exception caught in a custom exception translator. Failure to do so will cause Python to crash with ``SystemError: error return without exception set``. @@ -142,3 +163,144 @@ section. Exceptions that you do not plan to handle should simply not be caught, or may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. + +.. _handling_python_exceptions_cpp: + +Handling exceptions from Python in C++ +====================================== + +When C++ calls Python functions, such as in a callback function or when +manipulating Python objects, and Python raises an ``Exception``, pybind11 +converts the Python exception into a C++ exception of type +:class:`pybind11::error_already_set` whose payload contains a C++ string textual +summary and the actual Python exception. ``error_already_set`` is used to +propagate Python exception back to Python (or possibly, handle them in C++). + +.. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}| + ++--------------------------------------+--------------------------------------+ +| Exception raised in Python | Thrown as C++ exception type | ++======================================+======================================+ +| Any Python ``Exception`` | :class:`pybind11::error_already_set` | ++--------------------------------------+--------------------------------------+ + +For example: + +.. code-block:: cpp + + try { + // open("missing.txt", "r") + auto file = py::module_::import("io").attr("open")("missing.txt", "r"); + auto text = file.attr("read")(); + file.attr("close")(); + } catch (py::error_already_set &e) { + if (e.matches(PyExc_FileNotFoundError)) { + py::print("missing.txt not found"); + } else if (e.matches(PyExc_PermissionError)) { + py::print("missing.txt found but not accessible"); + } else { + throw; + } + } + +Note that C++ to Python exception translation does not apply here, since that is +a method for translating C++ exceptions to Python, not vice versa. The error raised +from Python is always ``error_already_set``. + +This example illustrates this behavior: + +.. code-block:: cpp + + try { + py::eval("raise ValueError('The Ring')"); + } catch (py::value_error &boromir) { + // Boromir never gets the ring + assert(false); + } catch (py::error_already_set &frodo) { + // Frodo gets the ring + py::print("I will take the ring"); + } + + try { + // py::value_error is a request for pybind11 to raise a Python exception + throw py::value_error("The ball"); + } catch (py::error_already_set &cat) { + // cat won't catch the ball since + // py::value_error is not a Python exception + assert(false); + } catch (py::value_error &dog) { + // dog will catch the ball + py::print("Run Spot run"); + throw; // Throw it again (pybind11 will raise ValueError) + } + +Handling errors from the Python C API +===================================== + +Where possible, use :ref:`pybind11 wrappers ` instead of calling +the Python C API directly. When calling the Python C API directly, in +addition to manually managing reference counts, one must follow the pybind11 +error protocol, which is outlined here. + +After calling the Python C API, if Python returns an error, +``throw py::error_already_set();``, which allows pybind11 to deal with the +exception and pass it back to the Python interpreter. This includes calls to +the error setting functions such as ``PyErr_SetString``. + +.. code-block:: cpp + + PyErr_SetString(PyExc_TypeError, "C API type error demo"); + throw py::error_already_set(); + + // But it would be easier to simply... + throw py::type_error("pybind11 wrapper type error"); + +Alternately, to ignore the error, call `PyErr_Clear +`_. + +Any Python error must be thrown or cleared, or Python/pybind11 will be left in +an invalid state. + +.. _unraisable_exceptions: + +Handling unraisable exceptions +============================== + +If a Python function invoked from a C++ destructor or any function marked +``noexcept(true)`` (collectively, "noexcept functions") throws an exception, there +is no way to propagate the exception, as such functions may not throw. +Should they throw or fail to catch any exceptions in their call graph, +the C++ runtime calls ``std::terminate()`` to abort immediately. + +Similarly, Python exceptions raised in a class's ``__del__`` method do not +propagate, but are logged by Python as an unraisable error. In Python 3.8+, a +`system hook is triggered +`_ +and an auditing event is logged. + +Any noexcept function should have a try-catch block that traps +class:`error_already_set` (or any other exception that can occur). Note that +pybind11 wrappers around Python exceptions such as +:class:`pybind11::value_error` are *not* Python exceptions; they are C++ +exceptions that pybind11 catches and converts to Python exceptions. Noexcept +functions cannot propagate these exceptions either. A useful approach is to +convert them to Python exceptions and then ``discard_as_unraisable`` as shown +below. + +.. code-block:: cpp + + void nonthrowing_func() noexcept(true) { + try { + // ... + } catch (py::error_already_set &eas) { + // Discard the Python error using Python APIs, using the C++ magic + // variable __func__. Python already knows the type and value and of the + // exception object. + eas.discard_as_unraisable(__func__); + } catch (const std::exception &e) { + // Log and discard C++ exceptions. + third_party::log(e); + } + } + +.. versionadded:: 2.6 diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index 984b046efa..6a93ab9da8 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -17,7 +17,7 @@ bindings for functions that return a non-trivial type. Just by looking at the type information, it is not clear whether Python should take charge of the returned value and eventually free its resources, or if this is handled on the C++ side. For this reason, pybind11 provides a several *return value policy* -annotations that can be passed to the :func:`module::def` and +annotations that can be passed to the :func:`module_::def` and :func:`class_::def` functions. The default policy is :enum:`return_value_policy::automatic`. @@ -50,7 +50,7 @@ implied transfer of ownership, i.e.: .. code-block:: cpp - m.def("get_data", &get_data, return_value_policy::reference); + m.def("get_data", &get_data, py::return_value_policy::reference); On the other hand, this is not the right policy for many other situations, where ignoring ownership could lead to resource leaks. @@ -360,7 +360,7 @@ like so: .. code-block:: cpp py::class_("MyClass") - .def("myFunction", py::arg("arg") = (SomeType *) nullptr); + .def("myFunction", py::arg("arg") = static_cast(nullptr)); Keyword-only arguments ====================== @@ -378,17 +378,37 @@ argument in a function definition: f(1, b=2) # good f(1, 2) # TypeError: f() takes 1 positional argument but 2 were given -Pybind11 provides a ``py::kwonly`` object that allows you to implement +Pybind11 provides a ``py::kw_only`` object that allows you to implement the same behaviour by specifying the object between positional and keyword-only argument annotations when registering the function: .. code-block:: cpp m.def("f", [](int a, int b) { /* ... */ }, - py::arg("a"), py::kwonly(), py::arg("b")); + py::arg("a"), py::kw_only(), py::arg("b")); -Note that, as in Python, you cannot combine this with a ``py::args`` argument. -This feature does *not* require Python 3 to work. +Note that you currently cannot combine this with a ``py::args`` argument. This +feature does *not* require Python 3 to work. + +.. versionadded:: 2.6 + +Positional-only arguments +========================= + +Python 3.8 introduced a new positional-only argument syntax, using ``/`` in the +function definition (note that this has been a convention for CPython +positional arguments, such as in ``pow()``, since Python 2). You can +do the same thing in any version of Python using ``py::pos_only()``: + +.. code-block:: cpp + + m.def("f", [](int a, int b) { /* ... */ }, + py::arg("a"), py::pos_only(), py::arg("b")); + +You now cannot give argument ``a`` by keyword. This can be combined with +keyword-only arguments, as well. + +.. versionadded:: 2.6 .. _nonconverting_arguments: @@ -504,6 +524,8 @@ The default behaviour when the tag is unspecified is to allow ``None``. not allow ``None`` as argument. To pass optional argument of these copied types consider using ``std::optional`` +.. _overload_resolution: + Overload resolution order ========================= @@ -520,11 +542,13 @@ an explicit ``py::arg().noconvert()`` attribute in the function definition). If the second pass also fails a ``TypeError`` is raised. Within each pass, overloads are tried in the order they were registered with -pybind11. +pybind11. If the ``py::prepend()`` tag is added to the definition, a function +can be placed at the beginning of the overload sequence instead, allowing user +overloads to proceed built in functions. What this means in practice is that pybind11 will prefer any overload that does -not require conversion of arguments to an overload that does, but otherwise prefers -earlier-defined overloads to later-defined ones. +not require conversion of arguments to an overload that does, but otherwise +prefers earlier-defined overloads to later-defined ones. .. note:: @@ -533,3 +557,7 @@ earlier-defined overloads to later-defined ones. requiring one conversion over one requiring three, but only prioritizes overloads requiring no conversion at all to overloads that require conversion of at least one argument. + +.. versionadded:: 2.6 + + The ``py::prepend()`` tag. diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 7798462df7..b3f3b2265a 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -7,14 +7,14 @@ General notes regarding convenience macros ========================================== pybind11 provides a few convenience macros such as -:func:`PYBIND11_DECLARE_HOLDER_TYPE` and ``PYBIND11_OVERLOAD_*``. Since these +:func:`PYBIND11_DECLARE_HOLDER_TYPE` and ``PYBIND11_OVERRIDE_*``. Since these are "just" macros that are evaluated in the preprocessor (which has no concept of types), they *will* get confused by commas in a template argument; for example, consider: .. code-block:: cpp - PYBIND11_OVERLOAD(MyReturnType, Class, func) + PYBIND11_OVERRIDE(MyReturnType, Class, func) The limitation of the C preprocessor interprets this as five arguments (with new arguments beginning after each comma) rather than three. To get around this, @@ -26,10 +26,10 @@ using the ``PYBIND11_TYPE`` macro: // Version 1: using a type alias using ReturnType = MyReturnType; using ClassType = Class; - PYBIND11_OVERLOAD(ReturnType, ClassType, func); + PYBIND11_OVERRIDE(ReturnType, ClassType, func); // Version 2: using the PYBIND11_TYPE macro: - PYBIND11_OVERLOAD(PYBIND11_TYPE(MyReturnType), + PYBIND11_OVERRIDE(PYBIND11_TYPE(MyReturnType), PYBIND11_TYPE(Class), func) The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds. @@ -59,7 +59,7 @@ could be realized as follows (important changes highlighted): /* Acquire GIL before calling Python code */ py::gil_scoped_acquire acquire; - PYBIND11_OVERLOAD_PURE( + PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ go, /* Name of function */ @@ -132,7 +132,7 @@ However, it can be acquired as follows: .. code-block:: cpp - py::object pet = (py::object) py::module::import("basic").attr("Pet"); + py::object pet = (py::object) py::module_::import("basic").attr("Pet"); py::class_(m, "Dog", pet) .def(py::init()) @@ -146,7 +146,7 @@ has been executed: .. code-block:: cpp - py::module::import("basic"); + py::module_::import("basic"); py::class_(m, "Dog") .def(py::init()) @@ -176,9 +176,9 @@ pybind11 version. Consider the following example: .. code-block:: cpp - auto data = (MyData *) py::get_shared_data("mydata"); + auto data = reinterpret_cast(py::get_shared_data("mydata")); if (!data) - data = (MyData *) py::set_shared_data("mydata", new MyData(42)); + data = static_cast(py::set_shared_data("mydata", new MyData(42))); If the above snippet was used in several separately compiled extension modules, the first one to be imported would create a ``MyData`` instance and associate @@ -223,7 +223,7 @@ avoids this issue involves weak reference with a cleanup callback: .. code-block:: cpp - // Register a callback function that is invoked when the BaseClass object is colelcted + // Register a callback function that is invoked when the BaseClass object is collected py::cpp_function cleanup_callback( [](py::handle weakref) { // perform cleanup here -- this function is called with the GIL held @@ -237,13 +237,13 @@ avoids this issue involves weak reference with a cleanup callback: .. note:: - PyPy (at least version 5.9) does not garbage collect objects when the - interpreter exits. An alternative approach (which also works on CPython) is to use - the :py:mod:`atexit` module [#f7]_, for example: + PyPy does not garbage collect objects when the interpreter exits. An alternative + approach (which also works on CPython) is to use the :py:mod:`atexit` module [#f7]_, + for example: .. code-block:: cpp - auto atexit = py::module::import("atexit"); + auto atexit = py::module_::import("atexit"); atexit.attr("register")(py::cpp_function([]() { // perform cleanup here -- this function is called with the GIL held })); @@ -284,7 +284,7 @@ work, it is important that all lines are indented consistently, i.e.: )mydelimiter"); By default, pybind11 automatically generates and prepends a signature to the docstring of a function -registered with ``module::def()`` and ``class_::def()``. Sometimes this +registered with ``module_::def()`` and ``class_::def()``. Sometimes this behavior is not desirable, because you want to provide your own signature or remove the docstring completely to exclude the function from the Sphinx documentation. The class ``options`` allows you to selectively suppress auto-generated signatures: @@ -304,3 +304,34 @@ the default settings are restored to prevent unwanted side effects. .. [#f4] http://www.sphinx-doc.org .. [#f5] http://github.com/pybind/python_example + +.. _avoiding-cpp-types-in-docstrings: + +Avoiding C++ types in docstrings +================================ + +Docstrings are generated at the time of the declaration, e.g. when ``.def(...)`` is called. +At this point parameter and return types should be known to pybind11. +If a custom type is not exposed yet through a ``py::class_`` constructor or a custom type caster, +its C++ type name will be used instead to generate the signature in the docstring: + +.. code-block:: text + + | __init__(...) + | __init__(self: example.Foo, arg0: ns::Bar) -> None + ^^^^^^^ + + +This limitation can be circumvented by ensuring that C++ classes are registered with pybind11 +before they are used as a parameter or return type of a function: + +.. code-block:: cpp + + PYBIND11_MODULE(example, m) { + + auto pyFoo = py::class_(m, "Foo"); + auto pyBar = py::class_(m, "Bar"); + + pyFoo.def(py::init()); + pyBar.def(py::init()); + } diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index f1941392fa..0a81aa8fd5 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -57,11 +57,11 @@ specification. struct buffer_info { void *ptr; - ssize_t itemsize; + py::ssize_t itemsize; std::string format; - ssize_t ndim; - std::vector shape; - std::vector strides; + py::ssize_t ndim; + std::vector shape; + std::vector strides; }; To create a C++ function that can take a Python buffer object as an argument, @@ -81,7 +81,7 @@ buffer objects (e.g. a NumPy matrix). constexpr bool rowMajor = Matrix::Flags & Eigen::RowMajorBit; py::class_(m, "Matrix", py::buffer_protocol()) - .def("__init__", [](py::buffer b) { + .def(py::init([](py::buffer b) { typedef Eigen::Stride Strides; /* Request a buffer descriptor from Python */ @@ -101,8 +101,8 @@ buffer objects (e.g. a NumPy matrix). auto map = Eigen::Map( static_cast(info.ptr), info.shape[0], info.shape[1], strides); - return Matrix(m); - }); + return Matrix(map); + })); For reference, the ``def_buffer()`` call for this Eigen data type should look as follows: @@ -150,8 +150,10 @@ NumPy array containing double precision values. When it is invoked with a different type (e.g. an integer or a list of integers), the binding code will attempt to cast the input into a NumPy array -of the requested type. Note that this feature requires the -:file:`pybind11/numpy.h` header to be included. +of the requested type. This feature requires the :file:`pybind11/numpy.h` +header to be included. Note that :file:`pybind11/numpy.h` does not depend on +the NumPy headers, and thus can be used without declaring a build-time +dependency on NumPy; NumPy>=1.7.0 is a runtime dependency. Data in NumPy arrays is not guaranteed to packed in a dense manner; furthermore, entries can be separated by arbitrary column and row strides. @@ -274,9 +276,9 @@ simply using ``vectorize``). py::buffer_info buf3 = result.request(); - double *ptr1 = (double *) buf1.ptr, - *ptr2 = (double *) buf2.ptr, - *ptr3 = (double *) buf3.ptr; + double *ptr1 = static_cast(buf1.ptr); + double *ptr2 = static_cast(buf2.ptr); + double *ptr3 = static_cast(buf3.ptr); for (size_t idx = 0; idx < buf1.shape[0]; idx++) ptr3[idx] = ptr1[idx] + ptr2[idx]; @@ -309,17 +311,17 @@ where ``N`` gives the required dimensionality of the array: m.def("sum_3d", [](py::array_t x) { auto r = x.unchecked<3>(); // x must have ndim = 3; can be non-writeable double sum = 0; - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t k = 0; k < r.shape(2); k++) sum += r(i, j, k); return sum; }); m.def("increment_3d", [](py::array_t x) { auto r = x.mutable_unchecked<3>(); // Will throw if ndim != 3 or flags.writeable is false - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t k = 0; k < r.shape(2); k++) r(i, j, k) += 1.0; }, py::arg().noconvert()); @@ -371,6 +373,8 @@ Ellipsis Python 3 provides a convenient ``...`` ellipsis notation that is often used to slice multidimensional arrays. For instance, the following snippet extracts the middle dimensions of a tensor with the first and last index set to zero. +In Python 2, the syntactic sugar ``...`` is not available, but the singleton +``Ellipsis`` (of type ``ellipsis``) can still be used directly. .. code-block:: python @@ -385,6 +389,9 @@ operation on the C++ side: py::array a = /* A NumPy array */; py::array b = a[py::make_tuple(0, py::ellipsis(), 0)]; +.. versionchanged:: 2.6 + ``py::ellipsis()`` is now also avaliable in Python 2. + Memory view =========== @@ -426,3 +433,6 @@ We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer: .. note:: ``memoryview::from_memory`` is not available in Python 2. + +.. versionchanged:: 2.6 + ``memoryview::from_memory`` added. diff --git a/docs/advanced/pycpp/object.rst b/docs/advanced/pycpp/object.rst index 19a226a876..6c7525ceaf 100644 --- a/docs/advanced/pycpp/object.rst +++ b/docs/advanced/pycpp/object.rst @@ -1,6 +1,8 @@ Python types ############ +.. _wrappers: + Available wrappers ================== @@ -13,6 +15,13 @@ Available types include :class:`handle`, :class:`object`, :class:`bool_`, :class:`iterable`, :class:`iterator`, :class:`function`, :class:`buffer`, :class:`array`, and :class:`array_t`. +.. warning:: + + Be sure to review the :ref:`pytypes_gotchas` before using this heavily in + your C++ API. + +.. _casting_back_and_forth: + Casting back and forth ====================== @@ -47,14 +56,15 @@ This example obtains a reference to the Python ``Decimal`` class. .. code-block:: cpp // Equivalent to "from decimal import Decimal" - py::object Decimal = py::module::import("decimal").attr("Decimal"); + py::object Decimal = py::module_::import("decimal").attr("Decimal"); .. code-block:: cpp // Try to import scipy - py::object scipy = py::module::import("scipy"); + py::object scipy = py::module_::import("scipy"); return scipy.attr("__version__"); + .. _calling_python_functions: Calling Python functions @@ -71,7 +81,7 @@ via ``operator()``. .. code-block:: cpp // Use Python to make our directories - py::object os = py::module::import("os"); + py::object os = py::module_::import("os"); py::object makedirs = os.attr("makedirs"); makedirs("/tmp/path/to/somewhere"); @@ -168,3 +178,74 @@ Generalized unpacking according to PEP448_ is also supported: Python functions from C++, including keywords arguments and unpacking. .. _PEP448: https://www.python.org/dev/peps/pep-0448/ + +.. _implicit_casting: + +Implicit casting +================ + +When using the C++ interface for Python types, or calling Python functions, +objects of type :class:`object` are returned. It is possible to invoke implicit +conversions to subclasses like :class:`dict`. The same holds for the proxy objects +returned by ``operator[]`` or ``obj.attr()``. +Casting to subtypes improves code readability and allows values to be passed to +C++ functions that require a specific subtype rather than a generic :class:`object`. + +.. code-block:: cpp + + #include + using namespace pybind11::literals; + + py::module_ os = py::module_::import("os"); + py::module_ path = py::module_::import("os.path"); // like 'import os.path as path' + py::module_ np = py::module_::import("numpy"); // like 'import numpy as np' + + py::str curdir_abs = path.attr("abspath")(path.attr("curdir")); + py::print(py::str("Current directory: ") + curdir_abs); + py::dict environ = os.attr("environ"); + py::print(environ["HOME"]); + py::array_t arr = np.attr("ones")(3, "dtype"_a="float32"); + py::print(py::repr(arr + py::int_(1))); + +These implicit conversions are available for subclasses of :class:`object`; there +is no need to call ``obj.cast()`` explicitly as for custom classes, see +:ref:`casting_back_and_forth`. + +.. note:: + If a trivial conversion via move constructor is not possible, both implicit and + explicit casting (calling ``obj.cast()``) will attempt a "rich" conversion. + For instance, ``py::list env = os.attr("environ");`` will succeed and is + equivalent to the Python code ``env = list(os.environ)`` that produces a + list of the dict keys. + +.. TODO: Adapt text once PR #2349 has landed + +Handling exceptions +=================== + +Python exceptions from wrapper classes will be thrown as a ``py::error_already_set``. +See :ref:`Handling exceptions from Python in C++ +` for more information on handling exceptions +raised when calling C++ wrapper classes. + +.. _pytypes_gotchas: + +Gotchas +======= + +Default-Constructed Wrappers +---------------------------- + +When a wrapper type is default-constructed, it is **not** a valid Python object (i.e. it is not ``py::none()``). It is simply the same as +``PyObject*`` null pointer. To check for this, use +``static_cast(my_wrapper)``. + +Assigning py::none() to wrappers +-------------------------------- + +You may be tempted to use types like ``py::str`` and ``py::dict`` in C++ +signatures (either pure C++, or in bound signatures), and assign them default +values of ``py::none()``. However, in a best case scenario, it will fail fast +because ``None`` is not convertible to that type (e.g. ``py::dict``), or in a +worse case scenario, it will silently work but corrupt the types you want to +work with (e.g. ``py::str(py::none())`` will yield ``"None"`` in Python). diff --git a/docs/advanced/pycpp/utilities.rst b/docs/advanced/pycpp/utilities.rst index 369e7c94db..c15051fb96 100644 --- a/docs/advanced/pycpp/utilities.rst +++ b/docs/advanced/pycpp/utilities.rst @@ -42,7 +42,7 @@ redirects output to the corresponding Python streams: m.def("noisy_func", []() { py::scoped_ostream_redirect stream( std::cout, // std::ostream& - py::module::import("sys").attr("stdout") // Python output + py::module_::import("sys").attr("stdout") // Python output ); call_noisy_func(); }); @@ -104,7 +104,7 @@ can be used. ... // Evaluate in scope of main module - py::object scope = py::module::import("__main__").attr("__dict__"); + py::object scope = py::module_::import("__main__").attr("__dict__"); // Evaluate an isolated expression int result = py::eval("my_variable + 10", scope).cast(); diff --git a/docs/basics.rst b/docs/basics.rst index 7bf4d426d3..0b1d85cfd3 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -11,11 +11,11 @@ included set of test cases. Compiling the test cases ======================== -Linux/MacOS +Linux/macOS ----------- On Linux you'll need to install the **python-dev** or **python3-dev** packages as -well as **cmake**. On Mac OS, the included python version works out of the box, +well as **cmake**. On macOS, the included python version works out of the box, but **cmake** must still be installed. After installing the prerequisites, run @@ -35,6 +35,14 @@ Windows On Windows, only **Visual Studio 2015** and newer are supported since pybind11 relies on various C++11 language features that break older versions of Visual Studio. +.. Note:: + + To use the C++17 in Visual Studio 2017 (MSVC 14.1), pybind11 requires the flag + ``/permissive-`` to be passed to the compiler `to enforce standard conformance`_. When + building with Visual Studio 2019, this is not strictly necessary, but still advised. + +.. _`to enforce standard conformance`: https://docs.microsoft.com/en-us/cpp/build/reference/permissive-standards-conformance?view=vs-2017 + To compile and run the tests: .. code-block:: batch @@ -110,8 +118,8 @@ a file named :file:`example.cpp` with the following contents: The :func:`PYBIND11_MODULE` macro creates a function that will be called when an ``import`` statement is issued from within Python. The module name (``example``) is given as the first macro argument (it should not be in quotes). The second -argument (``m``) defines a variable of type :class:`py::module ` which -is the main interface for creating bindings. The method :func:`module::def` +argument (``m``) defines a variable of type :class:`py::module_ ` which +is the main interface for creating bindings. The method :func:`module_::def` generates binding code that exposes the ``add()`` function to Python. .. note:: @@ -128,9 +136,16 @@ On Linux, the above example can be compiled using the following command: .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` + $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) + +.. note:: + + If you used :ref:`include_as_a_submodule` to get the pybind11 source, then + use ``$(python3-config --includes) -Iextern/pybind11/include`` instead of + ``$(python3 -m pybind11 --includes)`` in the above compilation, as + explained in :ref:`building_manually`. -For more details on the required compiler flags on Linux and MacOS, see +For more details on the required compiler flags on Linux and macOS, see :ref:`building_manually`. For complete cross-platform compilation instructions, refer to the :ref:`compiling` page. @@ -173,7 +188,7 @@ names of the arguments ("i" and "j" in this case). py::arg("i"), py::arg("j")); :class:`arg` is one of several special tag classes which can be used to pass -metadata into :func:`module::def`. With this modified binding code, we can now +metadata into :func:`module_::def`. With this modified binding code, we can now call the function using keyword arguments, which is a more readable alternative particularly for functions taking many parameters: diff --git a/docs/benchmark.py b/docs/benchmark.py index 023477212e..33d78fb4e6 100644 --- a/docs/benchmark.py +++ b/docs/benchmark.py @@ -14,7 +14,7 @@ def generate_dummy_code_pybind11(nclasses=10): for cl in range(nclasses): decl += "class cl%03i;\n" % cl - decl += '\n' + decl += "\n" for cl in range(nclasses): decl += "class cl%03i {\n" % cl @@ -22,18 +22,17 @@ def generate_dummy_code_pybind11(nclasses=10): bindings += ' py::class_(m, "cl%03i")\n' % (cl, cl) for fn in range(nfns): ret = random.randint(0, nclasses - 1) - params = [random.randint(0, nclasses - 1) for i in range(nargs)] + params = [random.randint(0, nclasses - 1) for i in range(nargs)] decl += " cl%03i *fn_%03i(" % (ret, fn) decl += ", ".join("cl%03i *" % p for p in params) decl += ");\n" - bindings += ' .def("fn_%03i", &cl%03i::fn_%03i)\n' % \ - (fn, cl, fn) + bindings += ' .def("fn_%03i", &cl%03i::fn_%03i)\n' % (fn, cl, fn) decl += "};\n\n" - bindings += ' ;\n' + bindings += " ;\n" result = "#include \n\n" result += "namespace py = pybind11;\n\n" - result += decl + '\n' + result += decl + "\n" result += "PYBIND11_MODULE(example, m) {\n" result += bindings result += "}" @@ -46,7 +45,7 @@ def generate_dummy_code_boost(nclasses=10): for cl in range(nclasses): decl += "class cl%03i;\n" % cl - decl += '\n' + decl += "\n" for cl in range(nclasses): decl += "class cl%03i {\n" % cl @@ -54,18 +53,20 @@ def generate_dummy_code_boost(nclasses=10): bindings += ' py::class_("cl%03i")\n' % (cl, cl) for fn in range(nfns): ret = random.randint(0, nclasses - 1) - params = [random.randint(0, nclasses - 1) for i in range(nargs)] + params = [random.randint(0, nclasses - 1) for i in range(nargs)] decl += " cl%03i *fn_%03i(" % (ret, fn) decl += ", ".join("cl%03i *" % p for p in params) decl += ");\n" - bindings += ' .def("fn_%03i", &cl%03i::fn_%03i, py::return_value_policy())\n' % \ - (fn, cl, fn) + bindings += ( + ' .def("fn_%03i", &cl%03i::fn_%03i, py::return_value_policy())\n' + % (fn, cl, fn) + ) decl += "};\n\n" - bindings += ' ;\n' + bindings += " ;\n" result = "#include \n\n" result += "namespace py = boost::python;\n\n" - result += decl + '\n' + result += decl + "\n" result += "BOOST_PYTHON_MODULE(example) {\n" result += bindings result += "}" @@ -73,17 +74,19 @@ def generate_dummy_code_boost(nclasses=10): for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: - print ("{") + print("{") for i in range(0, 10): nclasses = 2 ** i with open("test.cpp", "w") as f: f.write(codegen(nclasses)) n1 = dt.datetime.now() - os.system("g++ -Os -shared -rdynamic -undefined dynamic_lookup " + os.system( + "g++ -Os -shared -rdynamic -undefined dynamic_lookup " "-fvisibility=hidden -std=c++14 test.cpp -I include " - "-I /System/Library/Frameworks/Python.framework/Headers -o test.so") + "-I /System/Library/Frameworks/Python.framework/Headers -o test.so" + ) n2 = dt.datetime.now() elapsed = (n2 - n1).total_seconds() - size = os.stat('test.so').st_size + size = os.stat("test.so").st_size print(" {%i, %f, %i}," % (nclasses * nfns, elapsed, size)) - print ("}") + print("}") diff --git a/docs/changelog.rst b/docs/changelog.rst index 2def2b0719..84f1352fb9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,408 @@ Changelog Starting with version 1.8.0, pybind11 releases use a `semantic versioning `_ policy. +v2.7.0 (TBA, not yet released) +------------------------------ + +* ``py::str`` changed to exclusively hold `PyUnicodeObject`. Previously + ``py::str`` could also hold `bytes`, which is probably surprising, was + never documented, and can mask bugs (e.g. accidental use of ``py::str`` + instead of ``py::bytes``). + `#2409 `_ + + +v2.6.2 (Jan 26, 2021) +--------------------- + +Minor missing functionality added: + +* enum: add missing Enum.value property. + `#2739 `_ + +* Allow thread termination to be avoided during shutdown for CPython 3.7+ via + ``.disarm`` for ``gil_scoped_acquire``/``gil_scoped_release``. + `#2657 `_ + +Fixed or improved behavior in a few special cases: + +* Fix bug where the constructor of ``object`` subclasses would not throw on + being passed a Python object of the wrong type. + `#2701 `_ + +* The ``type_caster`` for integers does not convert Python objects with + ``__int__`` anymore with ``noconvert`` or during the first round of trying + overloads. + `#2698 `_ + +* When casting to a C++ integer, ``__index__`` is always called and not + considered as conversion, consistent with Python 3.8+. + `#2801 `_ + +Build improvements: + +* Setup helpers: ``extra_compile_args`` and ``extra_link_args`` automatically set by + Pybind11Extension are now prepended, which allows them to be overridden + by user-set ``extra_compile_args`` and ``extra_link_args``. + `#2808 `_ + +* Setup helpers: Don't trigger unused parameter warning. + `#2735 `_ + +* CMake: Support running with ``--warn-uninitialized`` active. + `#2806 `_ + +* CMake: Avoid error if included from two submodule directories. + `#2804 `_ + +* CMake: Fix ``STATIC`` / ``SHARED`` being ignored in FindPython mode. + `#2796 `_ + +* CMake: Respect the setting for ``CMAKE_CXX_VISIBILITY_PRESET`` if defined. + `#2793 `_ + +* CMake: Fix issue with FindPython2/FindPython3 not working with ``pybind11::embed``. + `#2662 `_ + +* CMake: mixing local and installed pybind11's would prioritize the installed + one over the local one (regression in 2.6.0). + `#2716 `_ + + +Bug fixes: + +* Fixed segfault in multithreaded environments when using + ``scoped_ostream_redirect``. + `#2675 `_ + +* Leave docstring unset when all docstring-related options are disabled, rather + than set an empty string. + `#2745 `_ + +* The module key in builtins that pybind11 uses to store its internals changed + from std::string to a python str type (more natural on Python 2, no change on + Python 3). + `#2814 `_ + +* Fixed assertion error related to unhandled (later overwritten) exception in + CPython 3.8 and 3.9 debug builds. + `#2685 `_ + +* Fix ``py::gil_scoped_acquire`` assert with CPython 3.9 debug build. + `#2683 `_ + +* Fix issue with a test failing on PyTest 6.2. + `#2741 `_ + +Warning fixes: + +* Fix warning modifying constructor parameter 'flag' that shadows a field of + 'set_flag' ``[-Wshadow-field-in-constructor-modified]``. + `#2780 `_ + +* Suppressed some deprecation warnings about old-style + ``__init__``/``__setstate__`` in the tests. + `#2759 `_ + +Valgrind work: + +* Fix invalid access when calling a pybind11 ``__init__`` on a non-pybind11 + class instance. + `#2755 `_ + +* Fixed various minor memory leaks in pybind11's test suite. + `#2758 `_ + +* Resolved memory leak in cpp_function initialization when exceptions occurred. + `#2756 `_ + +* Added a Valgrind build, checking for leaks and memory-related UB, to CI. + `#2746 `_ + +Compiler support: + +* Intel compiler was not activating C++14 support due to a broken define. + `#2679 `_ + +* Support ICC and NVIDIA HPC SDK in C++17 mode. + `#2729 `_ + +* Support Intel OneAPI compiler (ICC 20.2) and add to CI. + `#2573 `_ + + + +v2.6.1 (Nov 11, 2020) +--------------------- + +* ``py::exec``, ``py::eval``, and ``py::eval_file`` now add the builtins module + as ``"__builtins__"`` to their ``globals`` argument, better matching ``exec`` + and ``eval`` in pure Python. + `#2616 `_ + +* ``setup_helpers`` will no longer set a minimum macOS version higher than the + current version. + `#2622 `_ + +* Allow deleting static properties. + `#2629 `_ + +* Seal a leak in ``def_buffer``, cleaning up the ``capture`` object after the + ``class_`` object goes out of scope. + `#2634 `_ + +* ``pybind11_INCLUDE_DIRS`` was incorrect, potentially causing a regression if + it was expected to include ``PYTHON_INCLUDE_DIRS`` (please use targets + instead). + `#2636 `_ + +* Added parameter names to the ``py::enum_`` constructor and methods, avoiding + ``arg0`` in the generated docstrings. + `#2637 `_ + +* Added ``needs_recompile`` optional function to the ``ParallelCompiler`` + helper, to allow a recompile to be skipped based on a user-defined function. + `#2643 `_ + + +v2.6.0 (Oct 21, 2020) +--------------------- + +See :ref:`upgrade-guide-2.6` for help upgrading to the new version. + +New features: + +* Keyword-only arguments supported in Python 2 or 3 with ``py::kw_only()``. + `#2100 `_ + +* Positional-only arguments supported in Python 2 or 3 with ``py::pos_only()``. + `#2459 `_ + +* ``py::is_final()`` class modifier to block subclassing (CPython only). + `#2151 `_ + +* Added ``py::prepend()``, allowing a function to be placed at the beginning of + the overload chain. + `#1131 `_ + +* Access to the type object now provided with ``py::type::of()`` and + ``py::type::of(h)``. + `#2364 `_ + +* Perfect forwarding support for methods. + `#2048 `_ + +* Added ``py::error_already_set::discard_as_unraisable()``. + `#2372 `_ + +* ``py::hash`` is now public. + `#2217 `_ + +* ``py::class_`` is now supported. Note that writing to one data + member of the union and reading another (type punning) is UB in C++. Thus + pybind11-bound enums should never be used for such conversions. + `#2320 `_. + +* Classes now check local scope when registering members, allowing a subclass + to have a member with the same name as a parent (such as an enum). + `#2335 `_ + +Code correctness features: + +* Error now thrown when ``__init__`` is forgotten on subclasses. + `#2152 `_ + +* Throw error if conversion to a pybind11 type if the Python object isn't a + valid instance of that type, such as ``py::bytes(o)`` when ``py::object o`` + isn't a bytes instance. + `#2349 `_ + +* Throw if conversion to ``str`` fails. + `#2477 `_ + + +API changes: + +* ``py::module`` was renamed ``py::module_`` to avoid issues with C++20 when + used unqualified, but an alias ``py::module`` is provided for backward + compatibility. + `#2489 `_ + +* Public constructors for ``py::module_`` have been deprecated; please use + ``pybind11::module_::create_extension_module`` if you were using the public + constructor (fairly rare after ``PYBIND11_MODULE`` was introduced). + `#2552 `_ + +* ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function replaced by + correctly-named ``PYBIND11_OVERRIDE*`` and ``get_override``, fixing + inconsistencies in the presence of a closing ``;`` in these macros. + ``get_type_overload`` is deprecated. + `#2325 `_ + +Packaging / building improvements: + +* The Python package was reworked to be more powerful and useful. + `#2433 `_ + + * :ref:`build-setuptools` is easier thanks to a new + ``pybind11.setup_helpers`` module, which provides utilities to use + setuptools with pybind11. It can be used via PEP 518, ``setup_requires``, + or by directly importing or copying ``setup_helpers.py`` into your project. + + * CMake configuration files are now included in the Python package. Use + ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get + the directory with the CMake configuration files, or include the + site-packages location in your ``CMAKE_MODULE_PATH``. Or you can use the + new ``pybind11[global]`` extra when you install ``pybind11``, which + installs the CMake files and headers into your base environment in the + standard location. + + * ``pybind11-config`` is another way to write ``python -m pybind11`` if you + have your PATH set up. + + * Added external typing support to the helper module, code from + ``import pybind11`` can now be type checked. + `#2588 `_ + +* Minimum CMake required increased to 3.4. + `#2338 `_ and + `#2370 `_ + + * Full integration with CMake’s C++ standard system and compile features + replaces ``PYBIND11_CPP_STANDARD``. + + * Generated config file is now portable to different Python/compiler/CMake + versions. + + * Virtual environments prioritized if ``PYTHON_EXECUTABLE`` is not set + (``venv``, ``virtualenv``, and ``conda``) (similar to the new FindPython + mode). + + * Other CMake features now natively supported, like + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION``, ``set(CMAKE_CXX_VISIBILITY_PRESET + hidden)``. + + * ``CUDA`` as a language is now supported. + + * Helper functions ``pybind11_strip``, ``pybind11_extension``, + ``pybind11_find_import`` added, see :doc:`cmake/index`. + + * Optional :ref:`find-python-mode` and :ref:`nopython-mode` with CMake. + `#2370 `_ + +* Uninstall target added. + `#2265 `_ and + `#2346 `_ + +* ``pybind11_add_module()`` now accepts an optional ``OPT_SIZE`` flag that + switches the binding target to size-based optimization if the global build + type can not always be fixed to ``MinSizeRel`` (except in debug mode, where + optimizations remain disabled). ``MinSizeRel`` or this flag reduces binary + size quite substantially (~25% on some platforms). + `#2463 `_ + +Smaller or developer focused features and fixes: + +* Moved ``mkdoc.py`` to a new repo, `pybind11-mkdoc`_. There are no longer + submodules in the main repo. + +* ``py::memoryview`` segfault fix and update, with new + ``py::memoryview::from_memory`` in Python 3, and documentation. + `#2223 `_ + +* Fix for ``buffer_info`` on Python 2. + `#2503 `_ + +* If ``__eq__`` defined but not ``__hash__``, ``__hash__`` is now set to + ``None``. + `#2291 `_ + +* ``py::ellipsis`` now also works on Python 2. + `#2360 `_ + +* Pointer to ``std::tuple`` & ``std::pair`` supported in cast. + `#2334 `_ + +* Small fixes in NumPy support. ``py::array`` now uses ``py::ssize_t`` as first + argument type. + `#2293 `_ + +* Added missing signature for ``py::array``. + `#2363 `_ + +* ``unchecked_mutable_reference`` has access to operator ``()`` and ``[]`` when + const. + `#2514 `_ + +* ``py::vectorize`` is now supported on functions that return void. + `#1969 `_ + +* ``py::capsule`` supports ``get_pointer`` and ``set_pointer``. + `#1131 `_ + +* Fix crash when different instances share the same pointer of the same type. + `#2252 `_ + +* Fix for ``py::len`` not clearing Python's error state when it fails and throws. + `#2575 `_ + +* Bugfixes related to more extensive testing, new GitHub Actions CI. + `#2321 `_ + +* Bug in timezone issue in Eastern hemisphere midnight fixed. + `#2438 `_ + +* ``std::chrono::time_point`` now works when the resolution is not the same as + the system. + `#2481 `_ + +* Bug fixed where ``py::array_t`` could accept arrays that did not match the + requested ordering. + `#2484 `_ + +* Avoid a segfault on some compilers when types are removed in Python. + `#2564 `_ + +* ``py::arg::none()`` is now also respected when passing keyword arguments. + `#2611 `_ + +* PyPy fixes, PyPy 7.3.x now supported, including PyPy3. (Known issue with + PyPy2 and Windows `#2596 `_). + `#2146 `_ + +* CPython 3.9.0 workaround for undefined behavior (macOS segfault). + `#2576 `_ + +* CPython 3.9 warning fixes. + `#2253 `_ + +* Improved C++20 support, now tested in CI. + `#2489 `_ + `#2599 `_ + +* Improved but still incomplete debug Python interpreter support. + `#2025 `_ + +* NVCC (CUDA 11) now supported and tested in CI. + `#2461 `_ + +* NVIDIA PGI compilers now supported and tested in CI. + `#2475 `_ + +* At least Intel 18 now explicitly required when compiling with Intel. + `#2577 `_ + +* Extensive style checking in CI, with `pre-commit`_ support. Code + modernization, checked by clang-tidy. + +* Expanded docs, including new main page, new installing section, and CMake + helpers page, along with over a dozen new sections on existing pages. + +* In GitHub, new docs for contributing and new issue templates. + +.. _pre-commit: https://pre-commit.com + +.. _pybind11-mkdoc: https://github.com/pybind/pybind11-mkdoc + v2.5.0 (Mar 31, 2020) ----------------------------------------------------- @@ -361,7 +763,7 @@ v2.2.2 (February 7, 2018) v2.2.1 (September 14, 2017) ----------------------------------------------------- -* Added ``py::module::reload()`` member function for reloading a module. +* Added ``py::module_::reload()`` member function for reloading a module. `#1040 `_. * Fixed a reference leak in the number converter. @@ -536,7 +938,7 @@ v2.2.0 (August 31, 2017) in reference cycles. `#856 `_. -* Numpy and buffer protocol related improvements: +* NumPy and buffer protocol related improvements: 1. Support for negative strides in Python buffer objects/numpy arrays. This required changing integers from unsigned to signed for the related C++ APIs. @@ -1267,7 +1669,7 @@ Happy Christmas! * Improved support for ``std::shared_ptr<>`` conversions * Initial support for ``std::set<>`` conversions * Fixed type resolution issue for types defined in a separate plugin module -* Cmake build system improvements +* CMake build system improvements * Factored out generic functionality to non-templated code (smaller code size) * Added a code size / compile time benchmark vs Boost.Python * Added an appveyor CI script diff --git a/docs/classes.rst b/docs/classes.rst index a63f6a1969..f3610ef367 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -74,7 +74,7 @@ Note how ``print(p)`` produced a rather useless summary of our data structure in >>> print(p) -To address this, we could bind an utility function that returns a human-readable +To address this, we could bind a utility function that returns a human-readable summary to the special method slot named ``__repr__``. Unfortunately, there is no suitable functionality in the ``Pet`` data structure, and it would be nice if we did not have to change it. This can easily be accomplished by binding a @@ -373,8 +373,8 @@ sequence. py::class_(m, "Pet") .def(py::init()) - .def("set", (void (Pet::*)(int)) &Pet::set, "Set the pet's age") - .def("set", (void (Pet::*)(const std::string &)) &Pet::set, "Set the pet's name"); + .def("set", static_cast(&Pet::set), "Set the pet's age") + .def("set", static_cast(&Pet::set), "Set the pet's name"); The overload signatures are also visible in the method's docstring: diff --git a/docs/cmake/index.rst b/docs/cmake/index.rst new file mode 100644 index 0000000000..eaf66d70f3 --- /dev/null +++ b/docs/cmake/index.rst @@ -0,0 +1,8 @@ +CMake helpers +------------- + +Pybind11 can be used with ``add_subdirectory(extern/pybind11)``, or from an +install with ``find_package(pybind11 CONFIG)``. The interface provided in +either case is functionally identical. + +.. cmake-module:: ../../tools/pybind11Config.cmake.in diff --git a/docs/compiling.rst b/docs/compiling.rst index 07f93e7cc5..3a8a270d58 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -3,6 +3,8 @@ Build systems ############# +.. _build-setuptools: + Building with setuptools ======================== @@ -13,6 +15,206 @@ the [python_example]_ repository. .. [python_example] https://github.com/pybind/python_example +A helper file is provided with pybind11 that can simplify usage with setuptools. + +To use pybind11 inside your ``setup.py``, you have to have some system to +ensure that ``pybind11`` is installed when you build your package. There are +four possible ways to do this, and pybind11 supports all four: You can ask all +users to install pybind11 beforehand (bad), you can use +:ref:`setup_helpers-pep518` (good, but very new and requires Pip 10), +:ref:`setup_helpers-setup_requires` (discouraged by Python packagers now that +PEP 518 is available, but it still works everywhere), or you can +:ref:`setup_helpers-copy-manually` (always works but you have to manually sync +your copy to get updates). + +An example of a ``setup.py`` using pybind11's helpers: + +.. code-block:: python + + from glob import glob + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension + + ext_modules = [ + Pybind11Extension( + "python_example", + sorted(glob("src/*.cpp")), # Sort source files for reproducibility + ), + ] + + setup( + ..., + ext_modules=ext_modules + ) + +If you want to do an automatic search for the highest supported C++ standard, +that is supported via a ``build_ext`` command override; it will only affect +``Pybind11Extensions``: + +.. code-block:: python + + from glob import glob + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension, build_ext + + ext_modules = [ + Pybind11Extension( + "python_example", + sorted(glob("src/*.cpp")), + ), + ] + + setup( + ..., + cmdclass={"build_ext": build_ext}, + ext_modules=ext_modules + ) + +Since pybind11 does not require NumPy when building, a light-weight replacement +for NumPy's parallel compilation distutils tool is included. Use it like this: + +.. code-block:: python + + from pybind11.setup_helpers import ParallelCompile + + # Optional multithreaded build + ParallelCompile("NPY_NUM_BUILD_JOBS").install() + + setup(...) + +The argument is the name of an environment variable to control the number of +threads, such as ``NPY_NUM_BUILD_JOBS`` (as used by NumPy), though you can set +something different if you want; ``CMAKE_BUILD_PARALLEL_LEVEL`` is another choice +a user might expect. You can also pass ``default=N`` to set the default number +of threads (0 will take the number of threads available) and ``max=N``, the +maximum number of threads; if you have a large extension you may want set this +to a memory dependent number. + +If you are developing rapidly and have a lot of C++ files, you may want to +avoid rebuilding files that have not changed. For simple cases were you are +using ``pip install -e .`` and do not have local headers, you can skip the +rebuild if a object file is newer than it's source (headers are not checked!) +with the following: + +.. code-block:: python + + from pybind11.setup_helpers import ParallelCompile, naive_recompile + + SmartCompile("NPY_NUM_BUILD_JOBS", needs_recompile=naive_recompile).install() + + +If you have a more complex build, you can implement a smarter function and pass +it to ``needs_recompile``, or you can use [Ccache]_ instead. ``CXX="cache g++" +pip install -e .`` would be the way to use it with GCC, for example. Unlike the +simple solution, this even works even when not compiling in editable mode, but +it does require Ccache to be installed. + +Keep in mind that Pip will not even attempt to rebuild if it thinks it has +already built a copy of your code, which it deduces from the version number. +One way to avoid this is to use [setuptools_scm]_, which will generate a +version number that includes the number of commits since your last tag and a +hash for a dirty directory. Another way to force a rebuild is purge your cache +or use Pip's ``--no-cache-dir`` option. + +.. [Ccache] https://ccache.dev + +.. [setuptools_scm] https://github.com/pypa/setuptools_scm + +.. _setup_helpers-pep518: + +PEP 518 requirements (Pip 10+ required) +--------------------------------------- + +If you use `PEP 518's `_ +``pyproject.toml`` file, you can ensure that ``pybind11`` is available during +the compilation of your project. When this file exists, Pip will make a new +virtual environment, download just the packages listed here in ``requires=``, +and build a wheel (binary Python package). It will then throw away the +environment, and install your wheel. + +Your ``pyproject.toml`` file will likely look something like this: + +.. code-block:: toml + + [build-system] + requires = ["setuptools>=42", "wheel", "pybind11~=2.6.1"] + build-backend = "setuptools.build_meta" + +.. note:: + + The main drawback to this method is that a `PEP 517`_ compliant build tool, + such as Pip 10+, is required for this approach to work; older versions of + Pip completely ignore this file. If you distribute binaries (called wheels + in Python) using something like `cibuildwheel`_, remember that ``setup.py`` + and ``pyproject.toml`` are not even contained in the wheel, so this high + Pip requirement is only for source builds, and will not affect users of + your binary wheels. If you are building SDists and wheels, then + `pypa-build`_ is the recommended offical tool. + +.. _PEP 517: https://www.python.org/dev/peps/pep-0517/ +.. _cibuildwheel: https://cibuildwheel.readthedocs.io +.. _pypa-build: https://pypa-build.readthedocs.io/en/latest/ + +.. _setup_helpers-setup_requires: + +Classic ``setup_requires`` +-------------------------- + +If you want to support old versions of Pip with the classic +``setup_requires=["pybind11"]`` keyword argument to setup, which triggers a +two-phase ``setup.py`` run, then you will need to use something like this to +ensure the first pass works (which has not yet installed the ``setup_requires`` +packages, since it can't install something it does not know about): + +.. code-block:: python + + try: + from pybind11.setup_helpers import Pybind11Extension + except ImportError: + from setuptools import Extension as Pybind11Extension + + +It doesn't matter that the Extension class is not the enhanced subclass for the +first pass run; and the second pass will have the ``setup_requires`` +requirements. + +This is obviously more of a hack than the PEP 518 method, but it supports +ancient versions of Pip. + +.. _setup_helpers-copy-manually: + +Copy manually +------------- + +You can also copy ``setup_helpers.py`` directly to your project; it was +designed to be usable standalone, like the old example ``setup.py``. You can +set ``include_pybind11=False`` to skip including the pybind11 package headers, +so you can use it with git submodules and a specific git version. If you use +this, you will need to import from a local file in ``setup.py`` and ensure the +helper file is part of your MANIFEST. + + +Closely related, if you include pybind11 as a subproject, you can run the +``setup_helpers.py`` inplace. If loaded correctly, this should even pick up +the correct include for pybind11, though you can turn it off as shown above if +you want to input it manually. + +Suggested usage if you have pybind11 as a submodule in ``extern/pybind11``: + +.. code-block:: python + + DIR = os.path.abspath(os.path.dirname(__file__)) + + sys.path.append(os.path.join(DIR, "extern", "pybind11")) + from pybind11.setup_helpers import Pybind11Extension # noqa: E402 + + del sys.path[-1] + + +.. versionchanged:: 2.6 + + Added ``setup_helpers`` file. + Building with cppimport ======================== @@ -33,8 +235,8 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.7) - project(example) + cmake_minimum_required(VERSION 3.4...3.18) + project(example LANGUAGES CXX) add_subdirectory(pybind11) pybind11_add_module(example example.cpp) @@ -50,6 +252,11 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. [cmake_example] https://github.com/pybind/cmake_example +.. versionchanged:: 2.6 + CMake 3.4+ is required. + +Further information can be found at :doc:`cmake/index`. + pybind11_add_module ------------------- @@ -59,7 +266,7 @@ function with the following signature: .. code-block:: cmake pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] - [NO_EXTRAS] [THIN_LTO] source1 [source2 ...]) + [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] source1 [source2 ...]) This function behaves very much like CMake's builtin ``add_library`` (in fact, it's a wrapper function around that command). It will add a library target @@ -89,7 +296,29 @@ will result in code bloat and is generally not recommended. As stated above, LTO is enabled by default. Some newer compilers also support different flavors of LTO such as `ThinLTO`_. Setting ``THIN_LTO`` will cause the function to prefer this flavor if available. The function falls back to -regular LTO if ``-flto=thin`` is not available. +regular LTO if ``-flto=thin`` is not available. If +``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is set (either ``ON`` or ``OFF``), then +that will be respected instead of the built-in flag search. + +.. note:: + + If you want to set the property form on targets or the + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION_`` versions of this, you should + still use ``set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF)`` (otherwise a + no-op) to disable pybind11's ipo flags. + +The ``OPT_SIZE`` flag enables size-based optimization equivalent to the +standard ``/Os`` or ``-Os`` compiler flags and the ``MinSizeRel`` build type, +which avoid optimizations that that can substantially increase the size of the +resulting binary. This flag is particularly useful in projects that are split +into performance-critical parts and associated bindings. In this case, we can +compile the project in release mode (and hence, optimize performance globally), +and specify ``OPT_SIZE`` for the binding target, where size might be the main +concern as performance is often less critical here. A ~25% size reduction has +been observed in practice. This flag only changes the optimization behavior at +a per-target level and takes precedence over the global CMake build type +(``Release``, ``RelWithDebInfo``) except for ``Debug`` builds, where +optimizations remain disabled. .. _ThinLTO: http://clang.llvm.org/docs/ThinLTO.html @@ -103,19 +332,18 @@ standard explicitly with .. code-block:: cmake - set(CMAKE_CXX_STANDARD 14) # or 11, 14, 17, 20 + set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ version selection") # or 11, 14, 17, 20 set(CMAKE_CXX_STANDARD_REQUIRED ON) # optional, ensure standard is supported set(CMAKE_CXX_EXTENSIONS OFF) # optional, keep compiler extensionsn off - The variables can also be set when calling CMake from the command line using the ``-D=`` flag. You can also manually set ``CXX_STANDARD`` on a target or use ``target_compile_features`` on your targets - anything that CMake supports. -The target Python version can be selected by setting ``PYBIND11_PYTHON_VERSION`` -or an exact Python installation can be specified with ``PYTHON_EXECUTABLE``. -For example: +Classic Python support: The target Python version can be selected by setting +``PYBIND11_PYTHON_VERSION`` or an exact Python installation can be specified +with ``PYTHON_EXECUTABLE``. For example: .. code-block:: bash @@ -127,6 +355,7 @@ For example: # This often is a good way to get the current Python, works in environments: cmake -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") .. + find_package vs. add_subdirectory --------------------------------- @@ -136,8 +365,8 @@ See the `Config file`_ docstring for details of relevant CMake variables. .. code-block:: cmake - cmake_minimum_required(VERSION 3.7) - project(example) + cmake_minimum_required(VERSION 3.4...3.18) + project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) pybind11_add_module(example example.cpp) @@ -169,52 +398,143 @@ can refer to the same [cmake_example]_ repository for a full sample project .. _Config file: https://github.com/pybind/pybind11/blob/master/tools/pybind11Config.cmake.in -Advanced: interface library target ----------------------------------- -When using a version of CMake greater than 3.0, pybind11 can additionally -be used as a special *interface library* . The target ``pybind11::module`` -is available with pybind11 headers, Python headers and libraries as needed, -and C++ compile features attached. This target is suitable for linking -to an independently constructed (through ``add_library``, not -``pybind11_add_module``) target in the consuming project. +.. _find-python-mode: + +FindPython mode +--------------- + +CMake 3.12+ (3.15+ recommended, 3.18.2+ ideal) added a new module called +FindPython that had a highly improved search algorithm and modern targets +and tools. If you use FindPython, pybind11 will detect this and use the +existing targets instead: + +.. code-block:: cmake + + cmake_minumum_required(VERSION 3.15...3.19) + project(example LANGUAGES CXX) + + find_package(Python COMPONENTS Interpreter Development REQUIRED) + find_package(pybind11 CONFIG REQUIRED) + # or add_subdirectory(pybind11) + + pybind11_add_module(example example.cpp) + +You can also use the targets (as listed below) with FindPython. If you define +``PYBIND11_FINDPYTHON``, pybind11 will perform the FindPython step for you +(mostly useful when building pybind11's own tests, or as a way to change search +algorithms from the CMake invocation, with ``-DPYBIND11_FINDPYTHON=ON``. + +.. warning:: + + If you use FindPython2 and FindPython3 to dual-target Python, use the + individual targets listed below, and avoid targets that directly include + Python parts. + +There are `many ways to hint or force a discovery of a specific Python +installation `_), +setting ``Python_ROOT_DIR`` may be the most common one (though with +virtualenv/venv support, and Conda support, this tends to find the correct +Python version more often than the old system did). + +.. warning:: + + When the Python libraries (i.e. ``libpythonXX.a`` and ``libpythonXX.so`` + on Unix) are not available, as is the case on a manylinux image, the + ``Development`` component will not be resolved by ``FindPython``. When not + using the embedding functionality, CMake 3.18+ allows you to specify + ``Development.Module`` instead of ``Development`` to resolve this issue. + +.. versionadded:: 2.6 + +Advanced: interface library targets +----------------------------------- + +Pybind11 supports modern CMake usage patterns with a set of interface targets, +available in all modes. The targets provided are: + + ``pybind11::headers`` + Just the pybind11 headers and minimum compile requirements + + ``pybind11::python2_no_register`` + Quiets the warning/error when mixing C++14 or higher and Python 2 + + ``pybind11::pybind11`` + Python headers + ``pybind11::headers`` + ``pybind11::python2_no_register`` (Python 2 only) + + ``pybind11::python_link_helper`` + Just the "linking" part of pybind11:module + + ``pybind11::module`` + Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython CMake 3.15+) or ``pybind11::python_link_helper`` + + ``pybind11::embed`` + Everything for embedding the Python interpreter - ``pybind11::pybind11`` + ``Python::Embed`` (FindPython) or Python libs + + ``pybind11::lto`` / ``pybind11::thin_lto`` + An alternative to `INTERPROCEDURAL_OPTIMIZATION` for adding link-time optimization. + + ``pybind11::windows_extras`` + ``/bigobj`` and ``/mp`` for MSVC. + + ``pybind11::opt_size`` + ``/Os`` for MSVC, ``-Os`` for other compilers. Does nothing for debug builds. + +Two helper functions are also provided: + + ``pybind11_strip(target)`` + Strips a target (uses ``CMAKE_STRIP`` after the target is built) + + ``pybind11_extension(target)`` + Sets the correct extension (with SOABI) for a target. + +You can use these targets to build complex applications. For example, the +``add_python_module`` function is identical to: .. code-block:: cmake - cmake_minimum_required(VERSION 3.7) - project(example) + cmake_minimum_required(VERSION 3.4) + project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) add_library(example MODULE main.cpp) - target_link_libraries(example PRIVATE pybind11::module) - set_target_properties(example PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") + + target_link_libraries(example PRIVATE pybind11::module pybind11::lto pybind11::windows_extras) + + pybind11_extension(example) + pybind11_strip(example) + + set_target_properties(example PROPERTIES CXX_VISIBILITY_PRESET "hidden" + CUDA_VISIBILITY_PRESET "hidden") + +Instead of setting properties, you can set ``CMAKE_*`` variables to initialize these correctly. .. warning:: Since pybind11 is a metatemplate library, it is crucial that certain compiler flags are provided to ensure high quality code generation. In contrast to the ``pybind11_add_module()`` command, the CMake interface - library only provides the *minimal* set of parameters to ensure that the - code using pybind11 compiles, but it does **not** pass these extra compiler - flags (i.e. this is up to you). + provides a *composable* set of targets to ensure that you retain flexibility. + It can be expecially important to provide or set these properties; the + :ref:`FAQ ` contains an explanation on why these are needed. - These include Link Time Optimization (``-flto`` on GCC/Clang/ICPC, ``/GL`` - and ``/LTCG`` on Visual Studio) and .OBJ files with many sections on Visual - Studio (``/bigobj``). The :ref:`FAQ ` contains an - explanation on why these are needed. +.. versionadded:: 2.6 - If you want to add these in yourself, you can use: +.. _nopython-mode: - .. code-block:: cmake +Advanced: NOPYTHON mode +----------------------- - set(CMAKE_CXX_VISIBILITY_PRESET hidden) - set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) # CMake 3.9+ required +If you want complete control, you can set ``PYBIND11_NOPYTHON`` to completely +disable Python integration (this also happens if you run ``FindPython2`` and +``FindPython3`` without running ``FindPython``). This gives you complete +freedom to integrate into an existing system (like `Scikit-Build's +`_ ``PythonExtensions``). +``pybind11_add_module`` and ``pybind11_extension`` will be unavailable, and the +targets will be missing any Python specific behavior. - or set the corresponding property (without the ``CMAKE_``) on the targets - manually. +.. versionadded:: 2.6 Embedding the Python interpreter -------------------------------- @@ -228,8 +548,8 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.7) - project(example) + cmake_minimum_required(VERSION 3.4...3.18) + project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -249,7 +569,7 @@ On Linux, you can compile an example such as the one given in .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` + $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) The flags given here assume that you're using Python 3. For Python 2, just change the executable appropriately (to ``python`` or ``python2``). @@ -261,18 +581,18 @@ using ``pip`` or ``conda``. If it hasn't, you can also manually specify ``python3-config --includes``. Note that Python 2.7 modules don't use a special suffix, so you should simply -use ``example.so`` instead of ``example`python3-config --extension-suffix```. +use ``example.so`` instead of ``example$(python3-config --extension-suffix)``. Besides, the ``--extension-suffix`` option may or may not be available, depending on the distribution; in the latter case, the module extension can be manually set to ``.so``. -On Mac OS: the build command is almost the same but it also requires passing +On macOS: the build command is almost the same but it also requires passing the ``-undefined dynamic_lookup`` flag so as to ignore missing symbols when building the module: .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -undefined dynamic_lookup `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` + $ c++ -O3 -Wall -shared -std=c++11 -undefined dynamic_lookup $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) In general, it is advisable to include several additional build parameters that can considerably reduce the size of the created binary. Refer to section @@ -290,6 +610,13 @@ build system that works on all platforms including Windows. of possibly importing a second Python library into a process that already contains one (which will lead to a segfault). + +Building with Bazel +=================== + +You can build with the Bazel build system using the `pybind11_bazel +`_ repository. + Generating binding code automatically ===================================== @@ -306,3 +633,10 @@ extensible, and applies to very complex C++ libraries, composed of thousands of classes or incorporating modern meta-programming constructs. .. [AutoWIG] https://github.com/StatisKit/AutoWIG + +[robotpy-build]_ is a is a pure python, cross platform build tool that aims to +simplify creation of python wheels for pybind11 projects, and provide +cross-project dependency management. Additionally, it is able to autogenerate +customizable pybind11-based wrappers by parsing C++ header files. + +.. [robotpy-build] https://robotpy-build.readthedocs.io diff --git a/docs/conf.py b/docs/conf.py index 0946f30e2e..6ac054c6b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,53 +17,65 @@ import os import shlex import subprocess +from pathlib import Path +import re + +DIR = Path(__file__).parent.resolve() # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['breathe'] +extensions = [ + "breathe", + "sphinxcontrib.rsvgconverter", + "sphinxcontrib.moderncmakedomain", +] -breathe_projects = {'pybind11': '.build/doxygenxml/'} -breathe_default_project = 'pybind11' -breathe_domain_by_extension = {'h': 'cpp'} +breathe_projects = {"pybind11": ".build/doxygenxml/"} +breathe_default_project = "pybind11" +breathe_domain_by_extension = {"h": "cpp"} # Add any paths that contain templates here, relative to this directory. -templates_path = ['.templates'] +templates_path = [".templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'pybind11' -copyright = '2017, Wenzel Jakob' -author = 'Wenzel Jakob' +project = "pybind11" +copyright = "2017, Wenzel Jakob" +author = "Wenzel Jakob" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -# -# The short X.Y version. -version = '2.5' + +# Read the listed version +with open("../pybind11/_version.py") as f: + code = compile(f.read(), "../pybind11/_version.py", "exec") +loc = {} +exec(code, loc) + # The full version, including alpha/beta/rc tags. -release = '2.5.dev1' +version = loc["__version__"] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -74,37 +86,37 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['.build', 'release.rst'] +exclude_patterns = [".build", "release.rst"] # The reST default role (used for this markup: `text`) to use for all # documents. -default_role = 'any' +default_role = "any" # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -#pygments_style = 'monokai' +# pygments_style = 'monokai' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -115,141 +127,150 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' + + html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - html_context = { - 'css_files': [ - '_static/theme_overrides.css' - ] - } + html_context = {"css_files": ["_static/theme_overrides.css"]} else: html_context = { - 'css_files': [ - '//media.readthedocs.org/css/sphinx_rtd_theme.css', - '//media.readthedocs.org/css/readthedocs-doc-embed.css', - '_static/theme_overrides.css' + "css_files": [ + "//media.readthedocs.org/css/sphinx_rtd_theme.css", + "//media.readthedocs.org/css/readthedocs-doc-embed.css", + "_static/theme_overrides.css", ] } # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None +# " v documentation". +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'pybind11doc' +htmlhelp_basename = "pybind11doc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -'preamble': r'\DeclareUnicodeCharacter{00A0}{}', +latex_engine = "pdflatex" -# Latex figure (float) alignment -#'figure_align': 'htbp', +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # + # Additional stuff for the LaTeX preamble. + # remove blank pages (between the title page and the TOC, etc.) + "classoptions": ",openany,oneside", + "preamble": r""" +\usepackage{fontawesome} +\usepackage{textgreek} +\DeclareUnicodeCharacter{00A0}{} +\DeclareUnicodeCharacter{2194}{\faArrowsH} +\DeclareUnicodeCharacter{1F382}{\faBirthdayCake} +\DeclareUnicodeCharacter{1F355}{\faAdjust} +\DeclareUnicodeCharacter{0301}{'} +\DeclareUnicodeCharacter{03C0}{\textpi} + +""", + # Latex figure (float) alignment + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pybind11.tex', 'pybind11 Documentation', - 'Wenzel Jakob', 'manual'), + (master_doc, "pybind11.tex", "pybind11 Documentation", "Wenzel Jakob", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -258,32 +279,29 @@ # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pybind11', 'pybind11 Documentation', - [author], 1) -] +man_pages = [(master_doc, "pybind11", "pybind11 Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -292,41 +310,73 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pybind11', 'pybind11 Documentation', - author, 'pybind11', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pybind11", + "pybind11 Documentation", + author, + "pybind11", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False -primary_domain = 'cpp' -highlight_language = 'cpp' +primary_domain = "cpp" +highlight_language = "cpp" def generate_doxygen_xml(app): - build_dir = os.path.join(app.confdir, '.build') + build_dir = os.path.join(app.confdir, ".build") if not os.path.exists(build_dir): os.mkdir(build_dir) try: - subprocess.call(['doxygen', '--version']) - retcode = subprocess.call(['doxygen'], cwd=app.confdir) + subprocess.call(["doxygen", "--version"]) + retcode = subprocess.call(["doxygen"], cwd=app.confdir) if retcode < 0: sys.stderr.write("doxygen error code: {}\n".format(-retcode)) except OSError as e: sys.stderr.write("doxygen execution failed: {}\n".format(e)) +def prepare(app): + with open(DIR.parent / "README.rst") as f: + contents = f.read() + + if app.builder.name == "latex": + # Remove badges and stuff from start + contents = contents[contents.find(r".. start") :] + + # Filter out section titles for index.rst for LaTeX + contents = re.sub(r"^(.*)\n[-~]{3,}$", r"**\1**", contents, flags=re.MULTILINE) + + with open(DIR / "readme.rst", "w") as f: + f.write(contents) + + +def clean_up(app, exception): + (DIR / "readme.rst").unlink() + + def setup(app): - """Add hook for building doxygen xml when needed""" + + # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) + + # Copy the readme in + app.connect("builder-inited", prepare) + + # Clean up the generated readme + app.connect("build-finished", clean_up) diff --git a/docs/faq.rst b/docs/faq.rst index b68562910a..8c0b0f9fc2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -27,18 +27,6 @@ The Python interpreter immediately crashes when importing my module See the first answer. -CMake doesn't detect the right Python version -============================================= - -The CMake-based build system will try to automatically detect the installed -version of Python and link against that. When this fails, or when there are -multiple versions of Python and it finds the wrong one, delete -``CMakeCache.txt`` and then invoke CMake as follows: - -.. code-block:: bash - - cmake -DPYTHON_EXECUTABLE:FILEPATH= . - .. _faq_reference_arguments: Limitations involving reference arguments @@ -100,8 +88,8 @@ following example: .. code-block:: cpp - void init_ex1(py::module &); - void init_ex2(py::module &); + void init_ex1(py::module_ &); + void init_ex2(py::module_ &); /* ... */ PYBIND11_MODULE(example, m) { @@ -114,7 +102,7 @@ following example: .. code-block:: cpp - void init_ex1(py::module &m) { + void init_ex1(py::module_ &m) { m.def("add", [](int a, int b) { return a + b; }); } @@ -122,7 +110,7 @@ following example: .. code-block:: cpp - void init_ex2(py::module &m) { + void init_ex2(py::module_ &m) { m.def("sub", [](int a, int b) { return a - b; }); } @@ -181,7 +169,7 @@ can be changed, but even if it isn't it is not always enough to guarantee complete independence of the symbols involved when not using ``-fvisibility=hidden``. -Additionally, ``-fvisiblity=hidden`` can deliver considerably binary size +Additionally, ``-fvisibility=hidden`` can deliver considerably binary size savings. (See the following section for more details). @@ -275,17 +263,34 @@ been received, you must either explicitly interrupt execution by throwing }); } +CMake doesn't detect the right Python version +============================================= + +The CMake-based build system will try to automatically detect the installed +version of Python and link against that. When this fails, or when there are +multiple versions of Python and it finds the wrong one, delete +``CMakeCache.txt`` and then add ``-DPYTHON_EXECUTABLE=$(which python)`` to your +CMake configure line. (Replace ``$(which python)`` with a path to python if +your prefer.) + +You can alternatively try ``-DPYBIND11_FINDPYTHON=ON``, which will activate the +new CMake FindPython support instead of pybind11's custom search. Requires +CMake 3.12+, and 3.15+ or 3.18.2+ are even better. You can set this in your +``CMakeLists.txt`` before adding or finding pybind11, as well. + Inconsistent detection of Python version in CMake and pybind11 ============================================================== -The functions ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` provided by CMake -for Python version detection are not used by pybind11 due to unreliability and limitations that make -them unsuitable for pybind11's needs. Instead pybind provides its own, more reliable Python detection -CMake code. Conflicts can arise, however, when using pybind11 in a project that *also* uses the CMake -Python detection in a system with several Python versions installed. +The functions ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` +provided by CMake for Python version detection are modified by pybind11 due to +unreliability and limitations that make them unsuitable for pybind11's needs. +Instead pybind11 provides its own, more reliable Python detection CMake code. +Conflicts can arise, however, when using pybind11 in a project that *also* uses +the CMake Python detection in a system with several Python versions installed. -This difference may cause inconsistencies and errors if *both* mechanisms are used in the same project. Consider the following -Cmake code executed in a system with Python 2.7 and 3.x installed: +This difference may cause inconsistencies and errors if *both* mechanisms are +used in the same project. Consider the following CMake code executed in a +system with Python 2.7 and 3.x installed: .. code-block:: cmake @@ -303,10 +308,24 @@ In contrast this code: find_package(PythonInterp) find_package(PythonLibs) -will detect Python 3.x for pybind11 and may crash on ``find_package(PythonLibs)`` afterwards. - -It is advised to avoid using ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` from CMake and rely -on pybind11 in detecting Python version. If this is not possible CMake machinery should be called *before* including pybind11. +will detect Python 3.x for pybind11 and may crash on +``find_package(PythonLibs)`` afterwards. + +There are three possible solutions: + +1. Avoid using ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` + from CMake and rely on pybind11 in detecting Python version. If this is not + possible, the CMake machinery should be called *before* including pybind11. +2. Set ``PYBIND11_FINDPYTHON`` to ``True`` or use ``find_package(Python + COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, + 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead + of the old, deprecated search tools, and these modules are much better at + finding the correct Python. +3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. + However, you will have to use the target-based system, and do more setup + yourself, because it does not know about or include things that depend on + Python, like ``pybind11_add_module``. This might be ideal for integrating + into an existing system, like scikit-build's Python helpers. How to cite this project? ========================= diff --git a/docs/index.rst b/docs/index.rst index d236611b72..4e2e8ca3a0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,18 +1,17 @@ -.. only: not latex +.. only:: latex - .. image:: pybind11-logo.png + Intro + ===== -pybind11 --- Seamless operability between C++11 and Python -========================================================== +.. include:: readme.rst -.. only: not latex +.. only:: not latex Contents: .. toctree:: :maxdepth: 1 - intro changelog upgrade @@ -20,6 +19,7 @@ pybind11 --- Seamless operability between C++11 and Python :caption: The Basics :maxdepth: 2 + installing basics classes compiling @@ -45,3 +45,4 @@ pybind11 --- Seamless operability between C++11 and Python benchmark limitations reference + cmake/index diff --git a/docs/installing.rst b/docs/installing.rst new file mode 100644 index 0000000000..30b9f1853d --- /dev/null +++ b/docs/installing.rst @@ -0,0 +1,105 @@ +.. _installing: + +Installing the library +###################### + +There are several ways to get the pybind11 source, which lives at +`pybind/pybind11 on GitHub `_. The pybind11 +developers recommend one of the first three ways listed here, submodule, PyPI, +or conda-forge, for obtaining pybind11. + +.. _include_as_a_submodule: + +Include as a submodule +====================== + +When you are working on a project in Git, you can use the pybind11 repository +as a submodule. From your git repository, use: + +.. code-block:: bash + + git submodule add -b stable ../../pybind/pybind11 extern/pybind11 + git submodule update --init + +This assumes you are placing your dependencies in ``extern/``, and that you are +using GitHub; if you are not using GitHub, use the full https or ssh URL +instead of the relative URL ``../../pybind/pybind11`` above. Some other servers +also require the ``.git`` extension (GitHub does not). + +From here, you can now include ``extern/pybind11/include``, or you can use +the various integration tools (see :ref:`compiling`) pybind11 provides directly +from the local folder. + +Include with PyPI +================= + +You can download the sources and CMake files as a Python package from PyPI +using Pip. Just use: + +.. code-block:: bash + + pip install pybind11 + +This will provide pybind11 in a standard Python package format. If you want +pybind11 available directly in your environment root, you can use: + +.. code-block:: bash + + pip install "pybind11[global]" + +This is not recommended if you are installing with your system Python, as it +will add files to ``/usr/local/include/pybind11`` and +``/usr/local/share/cmake/pybind11``, so unless that is what you want, it is +recommended only for use in virtual environments or your ``pyproject.toml`` +file (see :ref:`compiling`). + +Include with conda-forge +======================== + +You can use pybind11 with conda packaging via `conda-forge +`_: + +.. code-block:: bash + + conda install -c conda-forge pybind11 + + +Include with vcpkg +================== +You can download and install pybind11 using the Microsoft `vcpkg +`_ dependency manager: + +.. code-block:: bash + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + vcpkg install pybind11 + +The pybind11 port in vcpkg is kept up to date by Microsoft team members and +community contributors. If the version is out of date, please `create an issue +or pull request `_ on the vcpkg +repository. + +Global install with brew +======================== + +The brew package manager (Homebrew on macOS, or Linuxbrew on Linux) has a +`pybind11 package +`_. +To install: + +.. code-block:: bash + + brew install pybind11 + +.. We should list Conan, and possibly a few other C++ package managers (hunter, +.. perhaps). Conan has a very clean CMake integration that would be good to show. + +Other options +============= + +Other locations you can find pybind11 are `listed here +`_; these are maintained +by various packagers and the community. diff --git a/docs/intro.rst b/docs/intro.rst deleted file mode 100644 index 10e1799a19..0000000000 --- a/docs/intro.rst +++ /dev/null @@ -1,93 +0,0 @@ -.. image:: pybind11-logo.png - -About this project -================== -**pybind11** is a lightweight header-only library that exposes C++ types in Python -and vice versa, mainly to create Python bindings of existing C++ code. Its -goals and syntax are similar to the excellent `Boost.Python`_ library by David -Abrahams: to minimize boilerplate code in traditional extension modules by -inferring type information using compile-time introspection. - -.. _Boost.Python: http://www.boost.org/doc/libs/release/libs/python/doc/index.html - -The main issue with Boost.Python—and the reason for creating such a similar -project—is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations. - -Core features -************* -The following core C++ features can be mapped to Python - -- Functions accepting and returning custom data structures per value, reference, or pointer -- Instance methods and static methods -- Overloaded functions -- Instance attributes and static attributes -- Arbitrary exception types -- Enumerations -- Callbacks -- Iterators and ranges -- Custom operators -- Single and multiple inheritance -- STL data structures -- Smart pointers with reference counting like ``std::shared_ptr`` -- Internal references with correct reference counting -- C++ classes with virtual (and pure virtual) methods can be extended in Python - -Goodies -******* -In addition to the core functionality, pybind11 provides some extra goodies: - -- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.7) are supported with an - implementation-agnostic interface. - -- It is possible to bind C++11 lambda functions with captured variables. The - lambda capture data is stored inside the resulting Python function object. - -- pybind11 uses C++11 move constructors and move assignment operators whenever - possible to efficiently transfer custom data types. - -- It's easy to expose the internal storage of custom data types through - Pythons' buffer protocols. This is handy e.g. for fast conversion between - C++ matrix classes like Eigen and NumPy without expensive copy operations. - -- pybind11 can automatically vectorize functions so that they are transparently - applied to all entries of one or more NumPy array arguments. - -- Python's slice-based access and assignment operations can be supported with - just a few lines of code. - -- Everything is contained in just a few header files; there is no need to link - against any additional libraries. - -- Binaries are generally smaller by a factor of at least 2 compared to - equivalent bindings generated by Boost.Python. A recent pybind11 conversion - of `PyRosetta`_, an enormous Boost.Python binding project, reported a binary - size reduction of **5.4x** and compile time reduction by **5.8x**. - -- Function signatures are precomputed at compile time (using ``constexpr``), - leading to smaller binaries. - -- With little extra effort, C++ types can be pickled and unpickled similar to - regular Python objects. - -.. _PyRosetta: http://graylab.jhu.edu/RosettaCon2016/PyRosetta-4.pdf - -Supported compilers -******************* - -1. Clang/LLVM (any non-ancient version with C++11 support) -2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 or newer -4. Intel C++ compiler v17 or newer (v16 with pybind11 v2.0 and v15 with pybind11 v2.0 and a `workaround `_ ) diff --git a/docs/limitations.rst b/docs/limitations.rst index 59474f82fd..be7300cdb6 100644 --- a/docs/limitations.rst +++ b/docs/limitations.rst @@ -1,6 +1,9 @@ Limitations ########### +Design choices +^^^^^^^^^^^^^^ + pybind11 strives to be a general solution to binding generation, but it also has certain limitations: @@ -11,9 +14,59 @@ certain limitations: - The NumPy interface ``pybind11::array`` greatly simplifies accessing numerical data from C++ (and vice versa), but it's not a full-blown array - class like ``Eigen::Array`` or ``boost.multi_array``. + class like ``Eigen::Array`` or ``boost.multi_array``. ``Eigen`` objects are + directly supported, however, with ``pybind11/eigen.h``. + +Large but useful features could be implemented in pybind11 but would lead to a +significant increase in complexity. Pybind11 strives to be simple and compact. +Users who require large new features are encouraged to write an extension to +pybind11; see `pybind11_json `_ for an +example. + + +Known bugs +^^^^^^^^^^ + +These are issues that hopefully will one day be fixed, but currently are +unsolved. If you know how to help with one of these issues, contributions +are welcome! + +- Intel 20.2 is currently having an issue with the test suite. + `#2573 `_ + +- Debug mode Python does not support 1-5 tests in the test suite currently. + `#2422 `_ + +- PyPy3 7.3.1 and 7.3.2 have issues with several tests on 32-bit Windows. + +Known limitations +^^^^^^^^^^^^^^^^^ + +These are issues that are probably solvable, but have not been fixed yet. A +clean, well written patch would likely be accepted to solve them. + +- Type casters are not kept alive recursively. + `#2527 `_ + One consequence is that containers of ``char *`` are currently not supported. + `#2245 `_ + +- The ``cpptest`` does not run on Windows with Python 3.8 or newer, due to DLL + loader changes. User code that is correctly installed should not be affected. + `#2560 `_ + +Python 3.9.0 warning +^^^^^^^^^^^^^^^^^^^^ + +Combining older versions of pybind11 (< 2.6.0) with Python on 3.9.0 will +trigger undefined behavior that typically manifests as crashes during +interpreter shutdown (but could also destroy your data. **You have been +warned**). -These features could be implemented but would lead to a significant increase in -complexity. I've decided to draw the line here to keep this project simple and -compact. Users who absolutely require these features are encouraged to fork -pybind11. +This issue has been +`fixed in Python `_. As a +mitigation until 3.9.1 is released and commonly used, pybind11 (2.6.0 or newer) +includes a temporary workaround specifically when Python 3.9.0 is detected at +runtime, leaking about 50 bytes of memory when a callback function is garbage +collected. For reference; the pybind11 test suite has about 2,000 such +callbacks, but only 49 are garbage collected before the end-of-process. Wheels +built with Python 3.9.0 will correctly avoid the leak when run in Python 3.9.1. diff --git a/docs/reference.rst b/docs/reference.rst index a9fbe60015..a678d41c88 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -46,12 +46,23 @@ With reference counting Convenience classes for specific Python types ============================================= -.. doxygenclass:: module +.. doxygenclass:: module_ :members: .. doxygengroup:: pytypes :members: +Convenience functions converting to Python types +================================================ + +.. doxygenfunction:: make_tuple(Args&&...) + +.. doxygenfunction:: make_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_iterator(Type &, Extra&&...) + +.. doxygenfunction:: make_key_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_key_iterator(Type &, Extra&&...) + .. _extras: Passing extra arguments to ``def`` or ``class_`` @@ -91,15 +102,15 @@ Inheritance See :doc:`/classes` and :doc:`/advanced/classes` for more detail. -.. doxygendefine:: PYBIND11_OVERLOAD +.. doxygendefine:: PYBIND11_OVERRIDE -.. doxygendefine:: PYBIND11_OVERLOAD_PURE +.. doxygendefine:: PYBIND11_OVERRIDE_PURE -.. doxygendefine:: PYBIND11_OVERLOAD_NAME +.. doxygendefine:: PYBIND11_OVERRIDE_NAME -.. doxygendefine:: PYBIND11_OVERLOAD_PURE_NAME +.. doxygendefine:: PYBIND11_OVERRIDE_PURE_NAME -.. doxygenfunction:: get_overload +.. doxygenfunction:: get_override Exceptions ========== @@ -110,7 +121,6 @@ Exceptions .. doxygenclass:: builtin_exception :members: - Literals ======== diff --git a/docs/release.rst b/docs/release.rst index 9846f971a6..9dbff03838 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,21 +1,92 @@ +On version numbers +^^^^^^^^^^^^^^^^^^ + +The two version numbers (C++ and Python) must match when combined (checked when +you build the PyPI package), and must be a valid `PEP 440 +`_ version when combined. + +For example: + +.. code-block:: C++ + + #define PYBIND11_VERSION_MAJOR X + #define PYBIND11_VERSION_MINOR Y + #define PYBIND11_VERSION_PATCH Z.dev1 + +For beta, ``PYBIND11_VERSION_PATCH`` should be ``Z.b1``. RC's can be ``Z.rc1``. +Always include the dot (even though PEP 440 allows it to be dropped). For a +final release, this must be a simple integer. + + To release a new version of pybind11: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Update the version number and push to pypi - - Update ``pybind11/_version.py`` (set release version, remove 'dev'). - - Update ``PYBIND11_VERSION_MAJOR`` etc. in ``include/pybind11/detail/common.h``. - - Ensure that all the information in ``setup.py`` is up-to-date. - - Update version in ``docs/conf.py``. - - Tag release date in ``docs/changelog.rst``. - - ``git add`` and ``git commit``. - - if new minor version: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` - - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``. +- Update the version number + - Update ``PYBIND11_VERSION_MAJOR`` etc. in + ``include/pybind11/detail/common.h``. PATCH should be a simple integer. + - Update ``pybind11/_version.py`` (match above) + - Ensure that all the information in ``setup.cfg`` is up-to-date, like + supported Python versions. + - Add release date in ``docs/changelog.rst``. + - Check to make sure + `needs-changelog `_ + issues are entered in the changelog (clear the label when done). + - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it + fails due to a known flake issue, either ignore or restart CI.) +- Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version + - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` + - Update branch: ``git checkout vX.Y``, ``git merge ``, ``git push`` +- Update tags (optional; if you skip this, the GitHub release makes a + non-annotated tag for you) + - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``. + - ``git push --tags``. +- Update stable + - ``git checkout stable`` + - ``git merge master`` - ``git push`` - - ``git push --tags``. - - ``python setup.py sdist upload``. - - ``python setup.py bdist_wheel upload``. +- Make a GitHub release (this shows up in the UI, sends new release + notifications to users watching releases, and also uploads PyPI packages). + (Note: if you do not use an existing tag, this creates a new lightweight tag + for you, so you could skip the above step). + - GUI method: click "Create a new release" on the far right, fill in the tag + name (if you didn't tag above, it will be made here), fill in a release + name like "Version X.Y.Z", and optionally copy-and-paste the changelog into + the description (processed as markdown by Pandoc). Check "pre-release" if + this is a beta/RC. You can get partway there with + ``cat docs/changelog.rst | pandsoc -f rst -t markdown``. + - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` + If this is a pre-release, add ``-p``. + - Get back to work - - Update ``_version.py`` (add 'dev' and increment minor). - - Update version in ``docs/conf.py`` - - Update version macros in ``include/pybind11/common.h`` - - ``git add`` and ``git commit``. - ``git push`` + - Make sure you are on master, not somewhere else: ``git checkout master`` + - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to + ``0.dev1`` and increment MINOR). + - Update ``_version.py`` to match + - Add a spot for in-development updates in ``docs/changelog.rst``. + - ``git add``, ``git commit``, ``git push`` + +If a version branch is updated, remember to set PATCH to ``1.dev1``. + +If you'd like to bump homebrew, run: + +.. code-block:: + + brew bump-formula-pr --url https://github.com/pybind/pybind11/archive/vX.Y.Z.tar.gz + +Conda-forge should automatically make a PR in a few hours, and automatically +merge it if there are no issues. + + +Manual packaging +^^^^^^^^^^^^^^^^ + +If you need to manually upload releases, you can download the releases from the job artifacts and upload them with twine. You can also make the files locally (not recommended in general, as your local directory is more likely to be "dirty" and SDists love picking up random unrelated/hidden files); this is the procedure: + +.. code-block:: bash + + python3 -m pip install build + python3 -m build + PYBIND11_SDIST_GLOBAL=1 python3 -m build + twine upload dist/* + +This makes SDists and wheels, and the final line uploads them. diff --git a/docs/requirements.txt b/docs/requirements.txt index 3818fe80ee..a1276a65d9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,7 @@ -breathe == 4.5.0 +breathe==4.26.1 +commonmark==0.9.1 +recommonmark==0.7.1 +sphinx==3.3.1 +sphinx_rtd_theme==0.5.0 +sphinxcontrib-moderncmakedomain==3.17 +sphinxcontrib-svg2pdfconverter==1.1.0 diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 3f5697391b..5691879c85 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,139 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.6: + +v2.7 +==== + +*Before* v2.7, ``py::str`` can hold ``PyUnicodeObject`` or ``PyBytesObject``, +and ``py::isinstance()`` is ``true`` for both ``py::str`` and +``py::bytes``. Starting with v2.7, ``py::str`` exclusively holds +``PyUnicodeObject`` (`#2409 `_), +and ``py::isinstance()`` is ``true`` only for ``py::str``. To help in +the transition of user code, the ``PYBIND11_STR_LEGACY_PERMISSIVE`` macro +is provided as an escape hatch to go back to the legacy behavior. This macro +will be removed in future releases. Two types of required fixes are expected +to be common: + +* Accidental use of ``py::str`` instead of ``py::bytes``, masked by the legacy + behavior. These are probably very easy to fix, by changing from + ``py::str`` to ``py::bytes``. + +* Reliance on py::isinstance(obj) being ``true`` for + ``py::bytes``. This is likely to be easy to fix in most cases by adding + ``|| py::isinstance(obj)``, but a fix may be more involved, e.g. if + ``py::isinstance`` appears in a template. Such situations will require + careful review and custom fixes. + + + +v2.6 +==== + +Usage of the ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function should +be replaced by ``PYBIND11_OVERRIDE*`` and ``get_override``. In the future, the +old macros may be deprecated and removed. + +``py::module`` has been renamed ``py::module_``, but a backward compatible +typedef has been included. This change was to avoid a language change in C++20 +that requires unqualified ``module`` not be placed at the start of a logical +line. Qualified usage is unaffected and the typedef will remain unless the +C++ language rules change again. + +The public constructors of ``py::module_`` have been deprecated. Use +``PYBIND11_MODULE`` or ``module_::create_extension_module`` instead. + +An error is now thrown when ``__init__`` is forgotten on subclasses. This was +incorrect before, but was not checked. Add a call to ``__init__`` if it is +missing. + +A ``py::type_error`` is now thrown when casting to a subclass (like +``py::bytes`` from ``py::object``) if the conversion is not valid. Make a valid +conversion instead. + +The undocumented ``h.get_type()`` method has been deprecated and replaced by +``py::type::of(h)``. + +Enums now have a ``__str__`` method pre-defined; if you want to override it, +the simplest fix is to add the new ``py::prepend()`` tag when defining +``"__str__"``. + +If ``__eq__`` defined but not ``__hash__``, ``__hash__`` is now set to +``None``, as in normal CPython. You should add ``__hash__`` if you intended the +class to be hashable, possibly using the new ``py::hash`` shortcut. + +The constructors for ``py::array`` now always take signed integers for size, +for consistency. This may lead to compiler warnings on some systems. Cast to +``py::ssize_t`` instead of ``std::size_t``. + +The ``tools/clang`` submodule and ``tools/mkdoc.py`` have been moved to a +standalone package, `pybind11-mkdoc`_. If you were using those tools, please +use them via a pip install from the new location. + +The ``pybind11`` package on PyPI no longer fills the wheel "headers" slot - if +you were using the headers from this slot, they are available by requesting the +``global`` extra, that is, ``pip install "pybind11[global]"``. (Most users will +be unaffected, as the ``pybind11/include`` location is reported by ``python -m +pybind11 --includes`` and ``pybind11.get_include()`` is still correct and has +not changed since 2.5). + +.. _pybind11-mkdoc: https://github.com/pybind/pybind11-mkdoc + +CMake support: +-------------- + +The minimum required version of CMake is now 3.4. Several details of the CMake +support have been deprecated; warnings will be shown if you need to change +something. The changes are: + +* ``PYBIND11_CPP_STANDARD=`` is deprecated, please use + ``CMAKE_CXX_STANDARD=`` instead, or any other valid CMake CXX or CUDA + standard selection method, like ``target_compile_features``. + +* If you do not request a standard, pybind11 targets will compile with the + compiler default, but not less than C++11, instead of forcing C++14 always. + If you depend on the old behavior, please use ``set(CMAKE_CXX_STANDARD 14 CACHE STRING "")`` + instead. + +* Direct ``pybind11::module`` usage should always be accompanied by at least + ``set(CMAKE_CXX_VISIBILITY_PRESET hidden)`` or similar - it used to try to + manually force this compiler flag (but not correctly on all compilers or with + CUDA). + +* ``pybind11_add_module``'s ``SYSTEM`` argument is deprecated and does nothing; + linking now behaves like other imported libraries consistently in both + config and submodule mode, and behaves like a ``SYSTEM`` library by + default. + +* If ``PYTHON_EXECUTABLE`` is not set, virtual environments (``venv``, + ``virtualenv``, and ``conda``) are prioritized over the standard search + (similar to the new FindPython mode). + +In addition, the following changes may be of interest: + +* ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` will be respected by + ``pybind11_add_module`` if set instead of linking to ``pybind11::lto`` or + ``pybind11::thin_lto``. + +* Using ``find_package(Python COMPONENTS Interpreter Development)`` before + pybind11 will cause pybind11 to use the new Python mechanisms instead of its + own custom search, based on a patched version of classic ``FindPythonInterp`` + / ``FindPythonLibs``. In the future, this may become the default. A recent + (3.15+ or 3.18.2+) version of CMake is recommended. + + + +v2.5 +==== + +The Python package now includes the headers as data in the package itself, as +well as in the "headers" wheel slot. ``pybind11 --includes`` and +``pybind11.get_include()`` report the new location, which is always correct +regardless of how pybind11 was installed, making the old ``user=`` argument +meaningless. If you are not using the function to get the location already, you +are encouraged to switch to the package location. + v2.2 ==== @@ -84,7 +217,7 @@ way to get and set object state. See :ref:`pickling` for details. ... .def(py::pickle( [](const Foo &self) { // __getstate__ - return py::make_tuple(f.value1(), f.value2(), ...); // unchanged + return py::make_tuple(self.value1(), self.value2(), ...); // unchanged }, [](py::tuple t) { // __setstate__, note: no `self` argument return new Foo(t[0].cast(), ...); diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 0809a0353a..de78e44470 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -40,8 +40,9 @@ struct sibling { handle value; sibling(const handle &value) : value(value.ptr()) /// Annotation indicating that a class derives from another given type template struct base { + PYBIND11_DEPRECATED("base() was deprecated in favor of specifying 'T' as a template argument to class_") - base() { } + base() { } // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute }; /// Keep patient alive while nurse lives @@ -61,7 +62,7 @@ struct metaclass { handle value; PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") - metaclass() {} + metaclass() { } // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute /// Override pybind11's default metaclass explicit metaclass(handle value) : value(value) { } @@ -73,6 +74,9 @@ struct module_local { const bool value; constexpr module_local(bool v = true) : /// Annotation to mark enums as an arithmetic type struct arithmetic { }; +/// Mark a function for addition at the beginning of the existing overload chain instead of the end +struct prepend { }; + /** \rst A call policy which places one or more guard variables (``Ts...``) around the function call. @@ -137,8 +141,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), - has_args(false), has_kwargs(false), has_kwonly_args(false) { } + is_operator(false), is_method(false), has_args(false), + has_kwargs(false), has_kw_only_args(false), prepend(false) { } /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -185,14 +189,20 @@ struct function_record { /// True if the function has a '**kwargs' argument bool has_kwargs : 1; - /// True once a 'py::kwonly' is encountered (any following args are keyword-only) - bool has_kwonly_args : 1; + /// True once a 'py::kw_only' is encountered (any following args are keyword-only) + bool has_kw_only_args : 1; + + /// True if this function is to be inserted at the beginning of the overload resolution chain + bool prepend : 1; /// Number of arguments (including py::args and/or py::kwargs, if present) std::uint16_t nargs; /// Number of trailing arguments (counted in `nargs`) that are keyword-only - std::uint16_t nargs_kwonly = 0; + std::uint16_t nargs_kw_only = 0; + + /// Number of leading arguments (counted in `nargs`) that are positional-only + std::uint16_t nargs_pos_only = 0; /// Python method object PyMethodDef *def = nullptr; @@ -369,10 +379,10 @@ template <> struct process_attribute : process_attribu static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; } }; -inline void process_kwonly_arg(const arg &a, function_record *r) { +inline void process_kw_only_arg(const arg &a, function_record *r) { if (!a.name || strlen(a.name) == 0) - pybind11_fail("arg(): cannot specify an unnamed argument after an kwonly() annotation"); - ++r->nargs_kwonly; + pybind11_fail("arg(): cannot specify an unnamed argument after an kw_only() annotation"); + ++r->nargs_kw_only; } /// Process a keyword argument attribute (*without* a default value) @@ -382,7 +392,7 @@ template <> struct process_attribute : process_attribute_default { r->args.emplace_back("self", nullptr, handle(), true /*convert*/, false /*none not allowed*/); r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); - if (r->has_kwonly_args) process_kwonly_arg(a, r); + if (r->has_kw_only_args) process_kw_only_arg(a, r); } }; @@ -415,14 +425,21 @@ template <> struct process_attribute : process_attribute_default { } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); - if (r->has_kwonly_args) process_kwonly_arg(a, r); + if (r->has_kw_only_args) process_kw_only_arg(a, r); } }; /// Process a keyword-only-arguments-follow pseudo argument -template <> struct process_attribute : process_attribute_default { - static void init(const kwonly &, function_record *r) { - r->has_kwonly_args = true; +template <> struct process_attribute : process_attribute_default { + static void init(const kw_only &, function_record *r) { + r->has_kw_only_args = true; + } +}; + +/// Process a positional-only-argument maker +template <> struct process_attribute : process_attribute_default { + static void init(const pos_only &, function_record *r) { + r->nargs_pos_only = static_cast(r->args.size()); } }; @@ -469,6 +486,12 @@ struct process_attribute : process_attribute_default static void init(const module_local &l, type_record *r) { r->module_local = l.value; } }; +/// Process a 'prepend' attribute, putting this at the beginning of the overload chain +template <> +struct process_attribute : process_attribute_default { + static void init(const prepend &, function_record *r) { r->prepend = true; } +}; + /// Process an 'arithmetic' attribute for enums (does nothing here) template <> struct process_attribute : process_attribute_default {}; @@ -524,7 +547,7 @@ template ::value...), size_t self = constexpr_sum(std::is_same::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { - return named == 0 || (self + named + has_args + has_kwargs) == nargs; + return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; } PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 8349a46b8b..d803004a10 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -13,6 +13,29 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Default, C-style strides +inline std::vector c_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + if (ndim > 0) + for (size_t i = ndim - 1; i > 0; --i) + strides[i - 1] = strides[i] * shape[i]; + return strides; +} + +// F-style strides; default when constructing an array_t with `ExtraFlags & f_style` +inline std::vector f_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + for (size_t i = 1; i < ndim; ++i) + strides[i] = strides[i - 1] * shape[i - 1]; + return strides; +} + +PYBIND11_NAMESPACE_END(detail) + /// Information record describing a Python buffer object struct buffer_info { void *ptr = nullptr; // Pointer to the underlying storage @@ -24,7 +47,7 @@ struct buffer_info { std::vector strides; // Number of bytes between adjacent entries (for each per dimension) bool readonly = false; // flag to indicate if the underlying storage may be written to - buffer_info() { } + buffer_info() = default; buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, detail::any_container shape_in, detail::any_container strides_in, bool readonly=false) @@ -53,7 +76,14 @@ struct buffer_info { explicit buffer_info(Py_buffer *view, bool ownview = true) : buffer_info(view->buf, view->itemsize, view->format, view->ndim, - {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}, view->readonly) { + {view->shape, view->shape + view->ndim}, + /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects + * ignore this flag and return a view with NULL strides. + * When strides are NULL, build them manually. */ + view->strides + ? std::vector(view->strides, view->strides + view->ndim) + : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), + view->readonly) { this->m_view = view; this->ownview = ownview; } diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 7a073d552f..64d590dbf8 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -11,13 +11,21 @@ #pragma once #include "pytypes.h" -#include "detail/typeid.h" +#include "detail/common.h" #include "detail/descr.h" -#include "detail/internals.h" +#include "detail/type_caster_base.h" +#include "detail/typeid.h" #include -#include +#include +#include +#include +#include +#include +#include #include #include +#include +#include #if defined(PYBIND11_CPP17) # if defined(__has_include) @@ -39,1026 +47,6 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -/// A life support system for temporary objects created by `type_caster::load()`. -/// Adding a patient will keep it alive up until the enclosing function returns. -class loader_life_support { -public: - /// A new patient frame is created when a function is entered - loader_life_support() { - get_internals().loader_patient_stack.push_back(nullptr); - } - - /// ... and destroyed after it returns - ~loader_life_support() { - auto &stack = get_internals().loader_patient_stack; - if (stack.empty()) - pybind11_fail("loader_life_support: internal error"); - - auto ptr = stack.back(); - stack.pop_back(); - Py_CLEAR(ptr); - - // A heuristic to reduce the stack's capacity (e.g. after long recursive calls) - if (stack.capacity() > 16 && stack.size() != 0 && stack.capacity() / stack.size() > 2) - stack.shrink_to_fit(); - } - - /// This can only be used inside a pybind11-bound function, either by `argument_loader` - /// at argument preparation time or by `py::cast()` at execution time. - PYBIND11_NOINLINE static void add_patient(handle h) { - auto &stack = get_internals().loader_patient_stack; - if (stack.empty()) - throw cast_error("When called outside a bound function, py::cast() cannot " - "do Python -> C++ conversions which require the creation " - "of temporary values"); - - auto &list_ptr = stack.back(); - if (list_ptr == nullptr) { - list_ptr = PyList_New(1); - if (!list_ptr) - pybind11_fail("loader_life_support: error allocating list"); - PyList_SET_ITEM(list_ptr, 0, h.inc_ref().ptr()); - } else { - auto result = PyList_Append(list_ptr, h.ptr()); - if (result == -1) - pybind11_fail("loader_life_support: error adding patient"); - } - } -}; - -// Gets the cache entry for the given type, creating it if necessary. The return value is the pair -// returned by emplace, i.e. an iterator for the entry and a bool set to `true` if the entry was -// just created. -inline std::pair all_type_info_get_cache(PyTypeObject *type); - -// Populates a just-created cache entry. -PYBIND11_NOINLINE inline void all_type_info_populate(PyTypeObject *t, std::vector &bases) { - std::vector check; - for (handle parent : reinterpret_borrow(t->tp_bases)) - check.push_back((PyTypeObject *) parent.ptr()); - - auto const &type_dict = get_internals().registered_types_py; - for (size_t i = 0; i < check.size(); i++) { - auto type = check[i]; - // Ignore Python2 old-style class super type: - if (!PyType_Check((PyObject *) type)) continue; - - // Check `type` in the current set of registered python types: - auto it = type_dict.find(type); - if (it != type_dict.end()) { - // We found a cache entry for it, so it's either pybind-registered or has pre-computed - // pybind bases, but we have to make sure we haven't already seen the type(s) before: we - // want to follow Python/virtual C++ rules that there should only be one instance of a - // common base. - for (auto *tinfo : it->second) { - // NB: Could use a second set here, rather than doing a linear search, but since - // having a large number of immediate pybind11-registered types seems fairly - // unlikely, that probably isn't worthwhile. - bool found = false; - for (auto *known : bases) { - if (known == tinfo) { found = true; break; } - } - if (!found) bases.push_back(tinfo); - } - } - else if (type->tp_bases) { - // It's some python type, so keep follow its bases classes to look for one or more - // registered types - if (i + 1 == check.size()) { - // When we're at the end, we can pop off the current element to avoid growing - // `check` when adding just one base (which is typical--i.e. when there is no - // multiple inheritance) - check.pop_back(); - i--; - } - for (handle parent : reinterpret_borrow(type->tp_bases)) - check.push_back((PyTypeObject *) parent.ptr()); - } - } -} - -/** - * Extracts vector of type_info pointers of pybind-registered roots of the given Python type. Will - * be just 1 pybind type for the Python type of a pybind-registered class, or for any Python-side - * derived class that uses single inheritance. Will contain as many types as required for a Python - * class that uses multiple inheritance to inherit (directly or indirectly) from multiple - * pybind-registered classes. Will be empty if neither the type nor any base classes are - * pybind-registered. - * - * The value is cached for the lifetime of the Python type. - */ -inline const std::vector &all_type_info(PyTypeObject *type) { - auto ins = all_type_info_get_cache(type); - if (ins.second) - // New cache entry: populate it - all_type_info_populate(type, ins.first->second); - - return ins.first->second; -} - -/** - * Gets a single pybind11 type info for a python type. Returns nullptr if neither the type nor any - * ancestors are pybind11-registered. Throws an exception if there are multiple bases--use - * `all_type_info` instead if you want to support multiple bases. - */ -PYBIND11_NOINLINE inline detail::type_info* get_type_info(PyTypeObject *type, bool do_throw = true) { - auto &bases = all_type_info(type); - if (bases.size() == 0) - return nullptr; - if (bases.size() > 1) { - if (do_throw) - pybind11_fail("pybind11::detail::get_type_info: type has multiple pybind11-registered bases"); - else - return nullptr; - } - return bases.front(); -} - -inline detail::type_info *get_local_type_info(const std::type_index &tp) { - auto &locals = registered_local_types_cpp(); - auto it = locals.find(tp); - if (it != locals.end()) - return it->second; - return nullptr; -} - -inline detail::type_info *get_global_type_info(const std::type_index &tp) { - auto &types = get_internals().registered_types_cpp; - auto it = types.find(tp); - if (it != types.end()) - return it->second; - return nullptr; -} - -/// Return the type info for a given C++ type; on lookup failure can either throw or return nullptr. -PYBIND11_NOINLINE inline detail::type_info *get_type_info(const std::type_index &tp, - bool throw_if_missing = false) { - if (auto ltype = get_local_type_info(tp)) - return ltype; - if (auto gtype = get_global_type_info(tp)) - return gtype; - - if (throw_if_missing) { - std::string tname = tp.name(); - detail::clean_type_id(tname); - pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + tname + "\""); - } - return nullptr; -} - -PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { - detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); -} - -struct value_and_holder { - instance *inst = nullptr; - size_t index = 0u; - const detail::type_info *type = nullptr; - void **vh = nullptr; - - // Main constructor for a found value/holder: - value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) : - inst{i}, index{index}, type{type}, - vh{inst->simple_layout ? inst->simple_value_holder : &inst->nonsimple.values_and_holders[vpos]} - {} - - // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) - value_and_holder() {} - - // Used for past-the-end iterator - value_and_holder(size_t index) : index{index} {} - - template V *&value_ptr() const { - return reinterpret_cast(vh[0]); - } - // True if this `value_and_holder` has a non-null value pointer - explicit operator bool() const { return value_ptr(); } - - template H &holder() const { - return reinterpret_cast(vh[1]); - } - void* holder_ptr() const { return &vh[1]; } - bool holder_constructed() const { - return inst->simple_layout - ? inst->simple_holder_constructed - : inst->nonsimple.status[index] & instance::status_holder_constructed; - } - void set_holder_constructed(bool v = true) { - if (inst->simple_layout) - inst->simple_holder_constructed = v; - else if (v) - inst->nonsimple.status[index] |= instance::status_holder_constructed; - else - inst->nonsimple.status[index] &= (uint8_t) ~instance::status_holder_constructed; - } - bool instance_registered() const { - return inst->simple_layout - ? inst->simple_instance_registered - : inst->nonsimple.status[index] & instance::status_instance_registered; - } - void set_instance_registered(bool v = true) { - if (inst->simple_layout) - inst->simple_instance_registered = v; - else if (v) - inst->nonsimple.status[index] |= instance::status_instance_registered; - else - inst->nonsimple.status[index] &= (uint8_t) ~instance::status_instance_registered; - } -}; - -// Container for accessing and iterating over an instance's values/holders -struct values_and_holders { -private: - instance *inst; - using type_vec = std::vector; - const type_vec &tinfo; - -public: - values_and_holders(instance *inst) : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} - - struct iterator { - private: - instance *inst = nullptr; - const type_vec *types = nullptr; - value_and_holder curr; - friend struct values_and_holders; - iterator(instance *inst, const type_vec *tinfo) - : inst{inst}, types{tinfo}, - curr(inst /* instance */, - types->empty() ? nullptr : (*types)[0] /* type info */, - 0, /* vpos: (non-simple types only): the first vptr comes first */ - 0 /* index */) - {} - // Past-the-end iterator: - iterator(size_t end) : curr(end) {} - public: - bool operator==(const iterator &other) const { return curr.index == other.curr.index; } - bool operator!=(const iterator &other) const { return curr.index != other.curr.index; } - iterator &operator++() { - if (!inst->simple_layout) - curr.vh += 1 + (*types)[curr.index]->holder_size_in_ptrs; - ++curr.index; - curr.type = curr.index < types->size() ? (*types)[curr.index] : nullptr; - return *this; - } - value_and_holder &operator*() { return curr; } - value_and_holder *operator->() { return &curr; } - }; - - iterator begin() { return iterator(inst, &tinfo); } - iterator end() { return iterator(tinfo.size()); } - - iterator find(const type_info *find_type) { - auto it = begin(), endit = end(); - while (it != endit && it->type != find_type) ++it; - return it; - } - - size_t size() { return tinfo.size(); } -}; - -/** - * Extracts C++ value and holder pointer references from an instance (which may contain multiple - * values/holders for python-side multiple inheritance) that match the given type. Throws an error - * if the given type (or ValueType, if omitted) is not a pybind11 base of the given instance. If - * `find_type` is omitted (or explicitly specified as nullptr) the first value/holder are returned, - * regardless of type (and the resulting .type will be nullptr). - * - * The returned object should be short-lived: in particular, it must not outlive the called-upon - * instance. - */ -PYBIND11_NOINLINE inline value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { - // Optimize common case: - if (!find_type || Py_TYPE(this) == find_type->type) - return value_and_holder(this, find_type, 0, 0); - - detail::values_and_holders vhs(this); - auto it = vhs.find(find_type); - if (it != vhs.end()) - return *it; - - if (!throw_if_missing) - return value_and_holder(); - -#if defined(NDEBUG) - pybind11_fail("pybind11::detail::instance::get_value_and_holder: " - "type is not a pybind11 base of the given instance " - "(compile in debug mode for type details)"); -#else - pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + - std::string(find_type->type->tp_name) + "' is not a pybind11 base of the given `" + - std::string(Py_TYPE(this)->tp_name) + "' instance"); -#endif -} - -PYBIND11_NOINLINE inline void instance::allocate_layout() { - auto &tinfo = all_type_info(Py_TYPE(this)); - - const size_t n_types = tinfo.size(); - - if (n_types == 0) - pybind11_fail("instance allocation failed: new instance has no pybind11-registered base types"); - - simple_layout = - n_types == 1 && tinfo.front()->holder_size_in_ptrs <= instance_simple_holder_in_ptrs(); - - // Simple path: no python-side multiple inheritance, and a small-enough holder - if (simple_layout) { - simple_value_holder[0] = nullptr; - simple_holder_constructed = false; - simple_instance_registered = false; - } - else { // multiple base types or a too-large holder - // Allocate space to hold: [v1*][h1][v2*][h2]...[bb...] where [vN*] is a value pointer, - // [hN] is the (uninitialized) holder instance for value N, and [bb...] is a set of bool - // values that tracks whether each associated holder has been initialized. Each [block] is - // padded, if necessary, to an integer multiple of sizeof(void *). - size_t space = 0; - for (auto t : tinfo) { - space += 1; // value pointer - space += t->holder_size_in_ptrs; // holder instance - } - size_t flags_at = space; - space += size_in_ptrs(n_types); // status bytes (holder_constructed and instance_registered) - - // Allocate space for flags, values, and holders, and initialize it to 0 (flags and values, - // in particular, need to be 0). Use Python's memory allocation functions: in Python 3.6 - // they default to using pymalloc, which is designed to be efficient for small allocations - // like the one we're doing here; in earlier versions (and for larger allocations) they are - // just wrappers around malloc. -#if PY_VERSION_HEX >= 0x03050000 - nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); - if (!nonsimple.values_and_holders) throw std::bad_alloc(); -#else - nonsimple.values_and_holders = (void **) PyMem_New(void *, space); - if (!nonsimple.values_and_holders) throw std::bad_alloc(); - std::memset(nonsimple.values_and_holders, 0, space * sizeof(void *)); -#endif - nonsimple.status = reinterpret_cast(&nonsimple.values_and_holders[flags_at]); - } - owned = true; -} - -PYBIND11_NOINLINE inline void instance::deallocate_layout() { - if (!simple_layout) - PyMem_Free(nonsimple.values_and_holders); -} - -PYBIND11_NOINLINE inline bool isinstance_generic(handle obj, const std::type_info &tp) { - handle type = detail::get_type_handle(tp, false); - if (!type) - return false; - return isinstance(obj, type); -} - -PYBIND11_NOINLINE inline std::string error_string() { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Unknown internal error occurred"); - return "Unknown internal error occurred"; - } - - error_scope scope; // Preserve error state - - std::string errorString; - if (scope.type) { - errorString += handle(scope.type).attr("__name__").cast(); - errorString += ": "; - } - if (scope.value) - errorString += (std::string) str(scope.value); - - PyErr_NormalizeException(&scope.type, &scope.value, &scope.trace); - -#if PY_MAJOR_VERSION >= 3 - if (scope.trace != nullptr) - PyException_SetTraceback(scope.value, scope.trace); -#endif - -#if !defined(PYPY_VERSION) - if (scope.trace) { - PyTracebackObject *trace = (PyTracebackObject *) scope.trace; - - /* Get the deepest trace possible */ - while (trace->tb_next) - trace = trace->tb_next; - - PyFrameObject *frame = trace->tb_frame; - errorString += "\n\nAt:\n"; - while (frame) { - int lineno = PyFrame_GetLineNumber(frame); - errorString += - " " + handle(frame->f_code->co_filename).cast() + - "(" + std::to_string(lineno) + "): " + - handle(frame->f_code->co_name).cast() + "\n"; - frame = frame->f_back; - } - } -#endif - - return errorString; -} - -PYBIND11_NOINLINE inline handle get_object_handle(const void *ptr, const detail::type_info *type ) { - auto &instances = get_internals().registered_instances; - auto range = instances.equal_range(ptr); - for (auto it = range.first; it != range.second; ++it) { - for (auto vh : values_and_holders(it->second)) { - if (vh.type == type) - return handle((PyObject *) it->second); - } - } - return handle(); -} - -inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) - return PyThreadState_GET(); -#elif PY_VERSION_HEX < 0x03000000 - return _PyThreadState_Current; -#elif PY_VERSION_HEX < 0x03050000 - return (PyThreadState*) _Py_atomic_load_relaxed(&_PyThreadState_Current); -#elif PY_VERSION_HEX < 0x03050200 - return (PyThreadState*) _PyThreadState_Current.value; -#else - return _PyThreadState_UncheckedGet(); -#endif -} - -// Forward declarations -inline void keep_alive_impl(handle nurse, handle patient); -inline PyObject *make_new_instance(PyTypeObject *type); - -enum class LoadType { - PureCpp, - DerivedCppSinglePySingle, - DerivedCppSinglePyMulti, - DerivedCppMulti, - /// Polymorphic casting or copy-based casting may be necessary. - ConversionNeeded, -}; - -typedef type_info* base_ptr_t; -typedef const std::vector bases_t; - -inline LoadType determine_load_type(handle src, const type_info* typeinfo, - const bases_t** out_bases = nullptr, - base_ptr_t* out_base = nullptr) { - // Null out inputs. - if (out_bases) - *out_bases = nullptr; - if (out_base) - *out_base = nullptr; - PyTypeObject *srctype = Py_TYPE(src.ptr()); - // See `type_caster_generic::load_impl` below for more detail on comments. - - // Case 1: If src is an exact type match for the target type then we can reinterpret_cast - // the instance's value pointer to the target type: - if (srctype == typeinfo->type) { - // TODO(eric.cousineau): Determine if the type is upcast from a type, which is - // still a pure C++ object? - return LoadType::PureCpp; - } - // Case 2: We have a derived class - else if (PyType_IsSubtype(srctype, typeinfo->type)) { - const bases_t& bases = all_type_info(srctype); - if (out_bases) - *out_bases = &bases; // Copy to output for caching. - const bool no_cpp_mi = typeinfo->simple_type; - // Case 2a: the python type is a Python-inherited derived class that inherits from just - // one simple (no MI) pybind11 class, or is an exact match, so the C++ instance is of - // the right type and we can use reinterpret_cast. - // (This is essentially the same as case 2b, but because not using multiple inheritance - // is extremely common, we handle it specially to avoid the loop iterator and type - // pointer lookup overhead) - // TODO(eric.cousineau): This seems to also capture C++-registered classes as well, not just Python-derived - // classes. - if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { - return LoadType::DerivedCppSinglePySingle; - } - // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see if - // we can find an exact match (or, for a simple C++ type, an inherited match); if so, we - // can safely reinterpret_cast to the relevant pointer. - else if (bases.size() > 1) { - for (auto base : bases) { - if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) : base->type == typeinfo->type) { - if (out_base) { - *out_base = base; - } - return LoadType::DerivedCppSinglePyMulti; - } - } - } - // Case 2c: C++ multiple inheritance is involved and we couldn't find an exact type match - // in the registered bases, above, so try implicit casting (needed for proper C++ casting - // when MI is involved). - return LoadType::DerivedCppMulti; - } else { - return LoadType::ConversionNeeded; - } -} - -class type_caster_generic { -public: - PYBIND11_NOINLINE type_caster_generic(const std::type_info &type_info) - : typeinfo(get_type_info(type_info)), cpptype(&type_info) { } - - type_caster_generic(const type_info *typeinfo) - : typeinfo(typeinfo), cpptype(typeinfo ? typeinfo->cpptype : nullptr) { } - - bool load(handle src, bool convert) { - return load_impl(src, convert); - } - - PYBIND11_NOINLINE static handle cast(const void *_src, return_value_policy policy, handle parent, - const detail::type_info *tinfo, - void *(*copy_constructor)(const void *), - void *(*move_constructor)(const void *), - holder_erased existing_holder = {}) { - if (!tinfo) // no type info: error will be set already - return handle(); - - void *src = const_cast(_src); - if (src == nullptr) - return none().release(); - - const bool take_ownership = policy == return_value_policy::automatic || policy == return_value_policy::take_ownership; - // We only come across `!existing_holder` if we are coming from `cast` and not `cast_holder`. - const bool is_bare_ptr = !existing_holder.ptr() && existing_holder.type_id() == HolderTypeId::Unknown; - - auto it_instances = get_internals().registered_instances.equal_range(src); - for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { - for (auto instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { - if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { - instance* const inst = it_i->second; - - bool try_to_reclaim = false; - if (!is_bare_ptr) { - switch (instance_type->release_info.holder_type_id) { - case detail::HolderTypeId::UniquePtr: { - try_to_reclaim = take_ownership; - break; - } - case detail::HolderTypeId::SharedPtr: { - if (take_ownership) { - // Only try to reclaim the object if (a) it is not owned and (b) has no holder. - if (!inst->simple_holder_constructed) { - if (inst->owned) - throw std::runtime_error("Internal error?"); - try_to_reclaim = true; - } - } - break; - } - default: { - // Otherwise, do not try any reclaiming. - break; - } - } - } - if (try_to_reclaim) { - // If this object has already been registered, but we wish to take ownership of it, - // then use the `has_cpp_release` mechanisms to reclaim ownership. - // @note This should be the sole occurrence of this registered object when releasing back. - // @note This code path should not be invoked for pure C++ - - // TODO(eric.cousineau): This may be still be desirable if this is a raw pointer... - // Need to think of a desirable workflow - and if there is possible interop. - if (!existing_holder) { - throw std::runtime_error("Internal error: Should have non-null holder."); - } - // TODO(eric.cousineau): This field may not be necessary if the lowest-level type is valid. - // See `move_only_holder_caster::load_value`. - if (!inst->reclaim_from_cpp) { - throw std::runtime_error("Instance is registered but does not have a registered reclaim method. Internal error?"); - } - return inst->reclaim_from_cpp(inst, existing_holder).release(); - } else { - // TODO(eric.cousineau): Should really check that ownership is consistent. - // e.g. if we say to take ownership of a pointer that is passed, does not have a holder... - // In the end, pybind11 would let ownership slip, and leak memory, possibly violating RAII (if someone is using that...) - return handle((PyObject *) it_i->second).inc_ref(); - } - } - } - } - - auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto wrapper = reinterpret_cast(inst.ptr()); - wrapper->owned = false; - void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); - - switch (policy) { - case return_value_policy::automatic: - case return_value_policy::take_ownership: - valueptr = src; - wrapper->owned = true; - break; - - case return_value_policy::automatic_reference: - case return_value_policy::reference: - valueptr = src; - wrapper->owned = false; - break; - - case return_value_policy::copy: - if (copy_constructor) - valueptr = copy_constructor(src); - else { -#if defined(NDEBUG) - throw cast_error("return_value_policy = copy, but type is " - "non-copyable! (compile in debug mode for details)"); -#else - std::string type_name(tinfo->cpptype->name()); - detail::clean_type_id(type_name); - throw cast_error("return_value_policy = copy, but type " + - type_name + " is non-copyable!"); -#endif - } - wrapper->owned = true; - break; - - case return_value_policy::move: - if (move_constructor) - valueptr = move_constructor(src); - else if (copy_constructor) - valueptr = copy_constructor(src); - else { -#if defined(NDEBUG) - throw cast_error("return_value_policy = move, but type is neither " - "movable nor copyable! " - "(compile in debug mode for details)"); -#else - std::string type_name(tinfo->cpptype->name()); - detail::clean_type_id(type_name); - throw cast_error("return_value_policy = move, but type " + - type_name + " is neither movable nor copyable!"); -#endif - } - wrapper->owned = true; - break; - - case return_value_policy::reference_internal: - valueptr = src; - wrapper->owned = false; - keep_alive_impl(inst, parent); - break; - - default: - throw cast_error("unhandled return_value_policy: should not happen!"); - } - - // TODO(eric.cousineau): Propagate `holder_erased` through this chain. - tinfo->init_instance(wrapper, existing_holder); - return inst.release(); - } - - // Base methods for generic caster; there are overridden in copyable_holder_caster - void load_value(value_and_holder &&v_h, LoadType) { - auto *&vptr = v_h.value_ptr(); - // Lazy allocation for unallocated values: - if (vptr == nullptr) { - auto *type = v_h.type ? v_h.type : typeinfo; - if (type->operator_new) { - vptr = type->operator_new(type->type_size); - } else { - #if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) - if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) - vptr = ::operator new(type->type_size, - std::align_val_t(type->type_align)); - else - #endif - vptr = ::operator new(type->type_size); - } - } - value = vptr; - } - bool try_implicit_casts(handle src, bool convert) { - for (auto &cast : typeinfo->implicit_casts) { - type_caster_generic sub_caster(*cast.first); - if (sub_caster.load(src, convert)) { - value = cast.second(sub_caster.value); - return true; - } - } - return false; - } - bool try_direct_conversions(handle src) { - for (auto &converter : *typeinfo->direct_conversions) { - if (converter(src.ptr(), value)) - return true; - } - return false; - } - void check_holder_compat() {} - - PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { - auto caster = type_caster_generic(ti); - if (caster.load(src, false)) - return caster.value; - return nullptr; - } - - /// Try to load with foreign typeinfo, if available. Used when there is no - /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { - constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; - const auto pytype = src.get_type(); - if (!hasattr(pytype, local_key)) - return false; - - type_info *foreign_typeinfo = reinterpret_borrow(getattr(pytype, local_key)); - // Only consider this foreign loader if actually foreign and is a loader of the correct cpp type - if (foreign_typeinfo->module_local_load == &local_load - || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) - return false; - - if (auto result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { - value = result; - return true; - } - return false; - } - - // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant - // bits of code between here and copyable_holder_caster where the two classes need different - // logic (without having to resort to virtual inheritance). - template - PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { - if (!src) return false; - if (!typeinfo) return try_load_foreign_module_local(src); - if (src.is_none()) { - // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; - value = nullptr; - return true; - } - - auto &this_ = static_cast(*this); - this_.check_holder_compat(); - - const bases_t* bases = nullptr; - base_ptr_t base_py_multi = nullptr; - LoadType load_type = determine_load_type(src, typeinfo, &bases, &base_py_multi); - switch (load_type) { - case LoadType::PureCpp: { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(), - load_type); - return true; - } - case LoadType::DerivedCppSinglePySingle: { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(), - load_type); - return true; - } - case LoadType::DerivedCppSinglePyMulti: { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(base_py_multi), - load_type); - return true; - } - case LoadType::DerivedCppMulti: { - if (this_.try_implicit_casts(src, convert)) - return true; - } - case LoadType::ConversionNeeded: { - break; - } - } - - // If nothing else succeeds, perform an implicit conversion - if (convert) { - for (auto &converter : typeinfo->implicit_conversions) { - auto temp = reinterpret_steal(converter(src.ptr(), typeinfo->type)); - if (load_impl(temp, false)) { - loader_life_support::add_patient(temp); - return true; - } - } - if (this_.try_direct_conversions(src)) - return true; - } - - // Failed to match local typeinfo. Try again with global. - if (typeinfo->module_local) { - if (auto gtype = get_global_type_info(*typeinfo->cpptype)) { - typeinfo = gtype; - return load(src, false); - } - } - - // Global typeinfo has precedence over foreign module_local - return try_load_foreign_module_local(src); - } - - - // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast - // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair - // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). - PYBIND11_NOINLINE static std::pair src_and_type( - const void *src, const std::type_info &cast_type, const std::type_info *rtti_type = nullptr) { - if (auto *tpi = get_type_info(cast_type)) - return {src, const_cast(tpi)}; - - // Not found, set error: - std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); - detail::clean_type_id(tname); - std::string msg = "Unregistered type : " + tname; - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return {nullptr, nullptr}; - } - - const type_info *typeinfo = nullptr; - const std::type_info *cpptype = nullptr; - void *value = nullptr; -}; - -/** - * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster - * needs to provide `operator T*()` and `operator T&()` operators. - * - * If the type supports moving the value away via an `operator T&&() &&` method, it should use - * `movable_cast_op_type` instead. - */ -template -using cast_op_type = - conditional_t>::value, - typename std::add_pointer>::type, - typename std::add_lvalue_reference>::type>; - -/** - * Determine suitable casting operator for a type caster with a movable value. Such a type caster - * needs to provide `operator T*()`, `operator T&()`, and `operator T&&() &&`. The latter will be - * called in appropriate contexts where the value can be moved rather than copied. - * - * These operator are automatically provided when using the PYBIND11_TYPE_CASTER macro. - */ -template -using movable_cast_op_type = - conditional_t::type>::value, - typename std::add_pointer>::type, - conditional_t::value, - typename std::add_rvalue_reference>::type, - typename std::add_lvalue_reference>::type>>; - -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template struct is_copy_constructible : std::is_copy_constructible {}; - -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. -template struct is_copy_constructible, - std::is_same, - // Avoid infinite recursion - negation> - >::value>> : is_copy_constructible {}; - -// Likewise for std::pair -// (after C++17 it is mandatory that the copy constructor not exist when the two types aren't themselves -// copy constructible, but this can not be relied upon when T1 or T2 are themselves containers). -template struct is_copy_constructible> - : all_of, is_copy_constructible> {}; - -// The same problems arise with std::is_copy_assignable, so we use the same workaround. -template struct is_copy_assignable : std::is_copy_assignable {}; -template struct is_copy_assignable, - std::is_same - >::value>> : is_copy_assignable {}; -template struct is_copy_assignable> - : all_of, is_copy_assignable> {}; - -PYBIND11_NAMESPACE_END(detail) - -// polymorphic_type_hook::get(src, tinfo) determines whether the object pointed -// to by `src` actually is an instance of some class derived from `itype`. -// If so, it sets `tinfo` to point to the std::type_info representing that derived -// type, and returns a pointer to the start of the most-derived object of that type -// (in which `src` is a subobject; this will be the same address as `src` in most -// single inheritance cases). If not, or if `src` is nullptr, it simply returns `src` -// and leaves `tinfo` at its default value of nullptr. -// -// The default polymorphic_type_hook just returns src. A specialization for polymorphic -// types determines the runtime type of the passed object and adjusts the this-pointer -// appropriately via dynamic_cast. This is what enables a C++ Animal* to appear -// to Python as a Dog (if Dog inherits from Animal, Animal is polymorphic, Dog is -// registered with pybind11, and this Animal is in fact a Dog). -// -// You may specialize polymorphic_type_hook yourself for types that want to appear -// polymorphic to Python but do not use C++ RTTI. (This is a not uncommon pattern -// in performance-sensitive applications, used most notably in LLVM.) -// -// polymorphic_type_hook_base allows users to specialize polymorphic_type_hook with -// std::enable_if. User provided specializations will always have higher priority than -// the default implementation and specialization provided in polymorphic_type_hook_base. -template -struct polymorphic_type_hook_base -{ - static const void *get(const itype *src, const std::type_info*&) { return src; } -}; -template -struct polymorphic_type_hook_base::value>> -{ - static const void *get(const itype *src, const std::type_info*& type) { - type = src ? &typeid(*src) : nullptr; - return dynamic_cast(src); - } -}; -template -struct polymorphic_type_hook : public polymorphic_type_hook_base {}; - -PYBIND11_NAMESPACE_BEGIN(detail) - -/// Generic type caster for objects stored on the heap -template class type_caster_base : public type_caster_generic { - using itype = intrinsic_t; - -public: - static constexpr auto name = _(); - - type_caster_base() : type_caster_base(typeid(type)) { } - explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) { } - - static handle cast(const itype &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) - policy = return_value_policy::copy; - return cast(&src, policy, parent); - } - - static handle cast(itype &&src, return_value_policy, handle parent) { - return cast(&src, return_value_policy::move, parent); - } - - // Returns a (pointer, type_info) pair taking care of necessary type lookup for a - // polymorphic type (using RTTI by default, but can be overridden by specializing - // polymorphic_type_hook). If the instance isn't derived, returns the base version. - static std::pair src_and_type(const itype *src) { - auto &cast_type = typeid(itype); - const std::type_info *instance_type = nullptr; - const void *vsrc = polymorphic_type_hook::get(src, instance_type); - if (instance_type && !same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type. If the derived type is registered - // with pybind11, we want to make the full derived object available. - // In the typical case where itype is polymorphic, we get the correct - // derived pointer (which may be != base pointer) by a dynamic_cast to - // most derived type. If itype is not polymorphic, we won't get here - // except via a user-provided specialization of polymorphic_type_hook, - // and the user has promised that no this-pointer adjustment is - // required in that case, so it's OK to use static_cast. - if (const auto *tpi = get_type_info(*instance_type)) - return {vsrc, tpi}; - } - // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, so - // don't do a cast - return type_caster_generic::src_and_type(src, cast_type, instance_type); - } - - static handle cast(const itype *src, return_value_policy policy, handle parent) { - auto st = src_and_type(src); - return type_caster_generic::cast( - st.first, policy, parent, st.second, - make_copy_constructor(src), make_move_constructor(src)); - } - - static handle cast_holder(const itype *src, holder_erased holder) { - auto st = src_and_type(src); - if (!holder) { - throw std::runtime_error("Internal error: Should not have null holder"); - } - return type_caster_generic::cast( - st.first, return_value_policy::take_ownership, {}, st.second, - nullptr, nullptr, holder); - } - - template using cast_op_type = detail::cast_op_type; - - operator itype*() { return (type *) value; } - operator itype&() { if (!value) throw reference_cast_error(); return *((itype *) value); } - -protected: - using Constructor = void *(*)(const void *); - - /* Only enabled when the types are {copy,move}-constructible *and* when the type - does not have a private operator new implementation. */ - template ::value>> - static auto make_copy_constructor(const T *x) -> decltype(new T(*x), Constructor{}) { - return [](const void *arg) -> void * { - return new T(*reinterpret_cast(arg)); - }; - } - - template ::value>> - static auto make_move_constructor(const T *x) -> decltype(new T(std::move(*const_cast(x))), Constructor{}) { - return [](const void *arg) -> void * { - return new T(std::move(*const_cast(reinterpret_cast(arg)))); - }; - } - - static Constructor make_copy_constructor(...) { return nullptr; } - static Constructor make_move_constructor(...) { return nullptr; } -}; - template class type_caster : public type_caster_base { }; template using make_caster = type_caster>; @@ -1076,9 +64,14 @@ template class type_caster> { private: using caster_t = make_caster; caster_t subcaster; - using subcaster_cast_op_type = typename caster_t::template cast_op_type; - static_assert(std::is_same::type &, subcaster_cast_op_type>::value, - "std::reference_wrapper caster requires T to have a caster with an `T &` operator"); + using reference_t = type&; + using subcaster_cast_op_type = + typename caster_t::template cast_op_type; + + static_assert(std::is_same::type &, subcaster_cast_op_type>::value || + std::is_same::value, + "std::reference_wrapper caster requires T to have a caster with an " + "`operator T &()` or `operator const T &()`"); public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } static constexpr auto name = caster_t::name; @@ -1089,7 +82,7 @@ template class type_caster> { return caster_t::cast(&src.get(), policy, parent); } template using cast_op_type = std::reference_wrapper; - operator std::reference_wrapper() { return subcaster.operator subcaster_cast_op_type&(); } + operator std::reference_wrapper() { return cast_op(subcaster); } }; #define PYBIND11_TYPE_CASTER(type, py_name) \ @@ -1122,6 +115,7 @@ template using is_std_char_type = any_of< std::is_same /* std::wstring */ >; + template struct type_caster::value && !is_std_char_type::value>> { using _py_type_0 = conditional_t; @@ -1135,6 +129,14 @@ struct type_caster::value && !is_std_char_t if (!src) return false; +#if !defined(PYPY_VERSION) + auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; +#else + // In PyPy 7.3.3, `PyIndex_Check` is implemented by calling `__index__`, + // while CPython only considers the existence of `nb_index`/`__index__`. + auto index_check = [](PyObject *o) { return hasattr(o, "__index__"); }; +#endif + if (std::is_floating_point::value) { if (convert || PyFloat_Check(src.ptr())) py_value = (py_type) PyFloat_AsDouble(src.ptr()); @@ -1142,29 +144,41 @@ struct type_caster::value && !is_std_char_t return false; } else if (PyFloat_Check(src.ptr())) { return false; - } else if (std::is_unsigned::value) { - py_value = as_unsigned(src.ptr()); - } else { // signed integer: - py_value = sizeof(T) <= sizeof(long) - ? (py_type) PyLong_AsLong(src.ptr()) - : (py_type) PYBIND11_LONG_AS_LONGLONG(src.ptr()); + } else if (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr())) { + return false; + } else { + handle src_or_index = src; +#if PY_VERSION_HEX < 0x03080000 + object index; + if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + index = reinterpret_steal(PyNumber_Index(src.ptr())); + if (!index) { + PyErr_Clear(); + if (!convert) + return false; + } + else { + src_or_index = index; + } + } +#endif + if (std::is_unsigned::value) { + py_value = as_unsigned(src_or_index.ptr()); + } else { // signed integer: + py_value = sizeof(T) <= sizeof(long) + ? (py_type) PyLong_AsLong(src_or_index.ptr()) + : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); + } } + // Python API reported an error bool py_err = py_value == (py_type) -1 && PyErr_Occurred(); - // Protect std::numeric_limits::min/max with parentheses - if (py_err || (std::is_integral::value && sizeof(py_type) != sizeof(T) && - (py_value < (py_type) (std::numeric_limits::min)() || - py_value > (py_type) (std::numeric_limits::max)()))) { - bool type_error = py_err && PyErr_ExceptionMatches( -#if PY_VERSION_HEX < 0x03000000 && !defined(PYPY_VERSION) - PyExc_SystemError -#else - PyExc_TypeError -#endif - ); + // Check to see if the conversion is valid (integers should match exactly) + // Signed/unsigned checks happen elsewhere + if (py_err || (std::is_integral::value && sizeof(py_type) != sizeof(T) && py_value != (py_type) (T) py_value)) { PyErr_Clear(); - if (type_error && convert && PyNumber_Check(src.ptr())) { + if (py_err && convert && PyNumber_Check(src.ptr())) { auto tmp = reinterpret_steal(std::is_floating_point::value ? PyNumber_Float(src.ptr()) : PyNumber_Long(src.ptr())); @@ -1245,7 +259,7 @@ template <> class type_caster : public type_caster { } /* Check if this is a C++ type */ - auto &bases = all_type_info((PyTypeObject *) h.get_type().ptr()); + auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); if (bases.size() == 1) { // Only allowing loading from a single-value type value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); return true; @@ -1277,7 +291,7 @@ template <> class type_caster { if (!src) return false; else if (src.ptr() == Py_True) { value = true; return true; } else if (src.ptr() == Py_False) { value = false; return true; } - else if (convert || !strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name)) { + else if (convert || !std::strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name)) { // (allow non-implicit conversion for numpy booleans) Py_ssize_t res = -1; @@ -1355,11 +369,11 @@ template struct string_caster { #endif } - object utfNbytes = reinterpret_steal(PyUnicode_AsEncodedString( + auto utfNbytes = reinterpret_steal(PyUnicode_AsEncodedString( load_src.ptr(), UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr)); if (!utfNbytes) { PyErr_Clear(); return false; } - const CharT *buffer = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); + const auto *buffer = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); if (UTF_N > 8) { buffer++; length--; } // Skip BOM for UTF-16/32 value = StringType(buffer, length); @@ -1373,7 +387,7 @@ template struct string_caster { static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { const char *buffer = reinterpret_cast(src.data()); - ssize_t nbytes = ssize_t(src.size() * sizeof(CharT)); + auto nbytes = ssize_t(src.size() * sizeof(CharT)); handle s = decode_utfN(buffer, nbytes); if (!s) throw error_already_set(); return s; @@ -1389,10 +403,8 @@ template struct string_caster { UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) : PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); #else - // PyPy seems to have multiple problems related to PyUnicode_UTF*: the UTF8 version - // sometimes segfaults for unknown reasons, while the UTF16 and 32 versions require a - // non-const char * arguments, which is also a nuisance, so bypass the whole thing by just - // passing the encoding as a string value, which works properly: + // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as well), + // so bypass the whole thing by just passing the encoding as a string value, which works properly: return PyUnicode_Decode(buffer, nbytes, UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr); #endif } @@ -1479,7 +491,7 @@ template struct type_caster 1 && str_len <= 4) { - unsigned char v0 = static_cast(value[0]); + auto v0 = static_cast(value[0]); size_t char0_bytes = !(v0 & 0x80) ? 1 : // low bits only: 0-127 (v0 & 0xE0) == 0xC0 ? 2 : // 0b110xxxxx - start of 2-byte sequence (v0 & 0xF0) == 0xE0 ? 3 : // 0b1110xxxx - start of 3-byte sequence @@ -1615,7 +627,11 @@ inline const detail::type_info* get_lowest_type(handle src, bool do_throw = true /// Type caster for holder types like std::shared_ptr, etc. -template +/// The SFINAE hook is provided to help work around the current lack of support +/// for smart-pointer interoperability. Please consider it an implementation +/// detail that may change in the future, as formal support for smart-pointer +/// interoperability is added into pybind11. +template struct copyable_holder_caster : public type_caster_base { public: using base = type_caster_base; @@ -1638,14 +654,7 @@ struct copyable_holder_caster : public type_caster_base { // see issue #2180 explicit operator type&() { return *(static_cast(this->value)); } explicit operator holder_type*() { return std::addressof(holder); } - - // Workaround for Intel compiler bug - // see pybind11 issue 94 - #if defined(__ICC) || defined(__INTEL_COMPILER) - operator holder_type&() { return holder; } - #else explicit operator holder_type&() { return holder; } - #endif // Risk increasing the `shared_ptr` ref count temporarily to maintain writeable // semantics without too much `const_cast<>` ooginess. @@ -1741,7 +750,10 @@ struct copyable_holder_caster : public type_caster_base { template class type_caster> : public copyable_holder_caster> { }; -template +/// Type caster for holder types like std::unique_ptr. +/// Please consider the SFINAE hook an implementation detail, as explained +/// in the comment for the copyable_holder_caster. +template struct move_only_holder_caster : type_caster_base { using base = type_caster_base; static_assert(std::is_base_of>::value, @@ -1904,6 +916,17 @@ struct pyobject_caster { template ::value, int> = 0> bool load(handle src, bool /* convert */) { +#if PY_MAJOR_VERSION < 3 && !defined(PYBIND11_STR_LEGACY_PERMISSIVE) + // For Python 2, without this implicit conversion, Python code would + // need to be cluttered with six.ensure_text() or similar, only to be + // un-cluttered later after Python 2 support is dropped. + if (std::is_same::value && isinstance(src)) { + PyObject *str_from_bytes = PyUnicode_FromEncodedObject(src.ptr(), "utf-8", nullptr); + if (!str_from_bytes) throw error_already_set(); + value = reinterpret_steal(str_from_bytes); + return true; + } +#endif if (!isinstance(src)) return false; value = reinterpret_borrow(src); @@ -1984,7 +1007,7 @@ template type_caster &load_type(type_ca throw cast_error("Unable to cast Python instance to C++ type (compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " + - (std::string) str(handle.get_type()) + " to C++ type '" + type_id() + "'"); + (std::string) str(type::handle_of(handle)) + " to C++ type '" + type_id() + "'"); #endif } return conv; @@ -2047,7 +1070,7 @@ detail::enable_if_t::value, T> move(object &&obj) { throw cast_error("Unable to cast Python instance to C++ rvalue: instance has multiple references" " (compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(obj.get_type()) + + throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) + " instance to C++ " + type_id() + " instance: instance has multiple references"); #endif @@ -2056,7 +1079,7 @@ detail::enable_if_t::value, T> move(object &&obj) { return ret; } -// Calling cast() on an rvalue calls pybind::cast with the object rvalue, which does: +// Calling cast() on an rvalue calls pybind11::cast with the object rvalue, which does: // - If we have to move (because T has no copy constructor), do it. This will fail if the moved // object has multiple references, but trying to copy will fail to compile. // - If both movable and copyable, check ref count: if 1, move; otherwise copy @@ -2085,16 +1108,16 @@ PYBIND11_NAMESPACE_BEGIN(detail) template ::value, int>> object object_or_cast(T &&o) { return pybind11::cast(std::forward(o)); } -struct overload_unused {}; // Placeholder type for the unneeded (and dead code) static variable in the OVERLOAD_INT macro -template using overload_caster_t = conditional_t< - cast_is_temporary_value_reference::value, make_caster, overload_unused>; +struct override_unused {}; // Placeholder type for the unneeded (and dead code) static variable in the PYBIND11_OVERRIDE_OVERRIDE macro +template using override_caster_t = conditional_t< + cast_is_temporary_value_reference::value, make_caster, override_unused>; // Trampoline use: for reference/pointer types to value-converted values, we do a value cast, then // store the result in the given variable. For other types, this is a no-op. template enable_if_t::value, T> cast_ref(object &&o, make_caster &caster) { return cast_op(load_type(caster, o)); } -template enable_if_t::value, T> cast_ref(object &&, overload_unused &) { +template enable_if_t::value, T> cast_ref(object &&, override_unused &) { pybind11_fail("Internal error: cast_ref fallback invoked"); } // Trampoline use: Having a pybind11::cast with an invalid reference type is going to static_assert, even @@ -2167,7 +1190,14 @@ struct arg_v : arg { #if !defined(NDEBUG) , type(type_id()) #endif - { } + { + // Workaround! See: + // https://github.com/pybind/pybind11/issues/2336 + // https://github.com/pybind/pybind11/pull/2685#issuecomment-731286700 + if (PyErr_Occurred()) { + PyErr_Clear(); + } + } public: /// Direct construction with name, default, and description @@ -2199,7 +1229,12 @@ struct arg_v : arg { /// \ingroup annotations /// Annotation indicating that all following arguments are keyword-only; the is the equivalent of an /// unnamed '*' argument (in Python 3) -struct kwonly {}; +struct kw_only {}; + +/// \ingroup annotations +/// Annotation indicating that all previous arguments are positional-only; the is the equivalent of an +/// unnamed '/' argument (in Python 3.8) +struct pos_only {}; template arg_v arg::operator=(T &&value) const { return {std::move(*this), std::forward(value)}; } @@ -2212,7 +1247,7 @@ inline namespace literals { String literal version of `arg` \endrst */ constexpr arg operator"" _a(const char *name, size_t) { return arg(name); } -} +} // namespace literals PYBIND11_NAMESPACE_BEGIN(detail) @@ -2375,7 +1410,7 @@ class unpacking_collector { } void process(list &args_list, detail::args_proxy ap) { - for (const auto &a : ap) + for (auto a : ap) args_list.append(a); } @@ -2407,7 +1442,7 @@ class unpacking_collector { void process(list &/*args_list*/, detail::kwargs_proxy kp) { if (!kp) return; - for (const auto &k : reinterpret_borrow(kp)) { + for (auto k : reinterpret_borrow(kp)) { if (m_kwargs.contains(k.first)) { #if defined(NDEBUG) multiple_values_error(); @@ -2452,16 +1487,26 @@ class unpacking_collector { dict m_kwargs; }; +// [workaround(intel)] Separate function required here +// We need to put this into a separate function because the Intel compiler +// fails to compile enable_if_t...>::value> +// (tested with ICC 2021.1 Beta 20200827). +template +constexpr bool args_are_all_positional() +{ + return all_of...>::value; +} + /// Collect only positional arguments for a Python function call template ...>::value>> + typename = enable_if_t()>> simple_collector collect_arguments(Args &&...args) { return simple_collector(std::forward(args)...); } /// Collect all arguments, including keywords and unpacking (only instantiated when needed) template ...>::value>> + typename = enable_if_t()>> unpacking_collector collect_arguments(Args &&...args) { // Following argument order rules for generalized unpacking according to PEP 448 static_assert( @@ -2487,13 +1532,25 @@ object object_api::call(Args &&...args) const { PYBIND11_NAMESPACE_END(detail) + +template +handle type::handle_of() { + static_assert( + std::is_base_of>::value, + "py::type::of only supports the case where T is a registered C++ types." + ); + + return detail::get_type_handle(typeid(T), true); +} + + #define PYBIND11_MAKE_OPAQUE(...) \ namespace pybind11 { namespace detail { \ template<> class type_caster<__VA_ARGS__> : public type_caster_base<__VA_ARGS__> { }; \ }} /// Lets you pass a type containing a `,` through a macro parameter without needing a separate -/// typedef, e.g.: `PYBIND11_OVERLOAD(PYBIND11_TYPE(ReturnType), PYBIND11_TYPE(Parent), f, arg)` +/// typedef, e.g.: `PYBIND11_OVERRIDE(PYBIND11_TYPE(ReturnType), PYBIND11_TYPE(Parent), f, arg)` #define PYBIND11_TYPE(...) __VA_ARGS__ PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 6b9ab9b82a..551a802ad3 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -32,10 +32,10 @@ PYBIND11_NAMESPACE_BEGIN(detail) template class duration_caster { public: - typedef typename type::rep rep; - typedef typename type::period period; + using rep = typename type::rep; + using period = typename type::period; - typedef std::chrono::duration> days; + using days = std::chrono::duration>; bool load(handle src, bool) { using namespace std::chrono; @@ -98,7 +98,7 @@ template class duration_caster { // This is for casting times on the system clock into datetime.datetime instances template class type_caster> { public: - typedef std::chrono::time_point type; + using type = std::chrono::time_point; bool load(handle src, bool) { using namespace std::chrono; @@ -140,7 +140,7 @@ template class type_caster(system_clock::from_time_t(std::mktime(&cal)) + msecs); return true; } @@ -150,13 +150,27 @@ template class type_caster(src)); + // Get out microseconds, and make sure they are positive, to avoid bug in eastern hemisphere time zones + // (cfr. https://github.com/pybind/pybind11/issues/2417) + using us_t = duration; + auto us = duration_cast(src.time_since_epoch() % seconds(1)); + if (us.count() < 0) + us += seconds(1); + + // Subtract microseconds BEFORE `system_clock::to_time_t`, because: + // > If std::time_t has lower precision, it is implementation-defined whether the value is rounded or truncated. + // (https://en.cppreference.com/w/cpp/chrono/system_clock/to_time_t) + std::time_t tt = system_clock::to_time_t(time_point_cast(src - us)); + + // std::localtime returns a pointer to a static internal std::tm object on success, + // or null pointer otherwise + std::tm *localtime_ptr = std::localtime(&tt); + if (!localtime_ptr) + throw cast_error("Unable to represent system_clock in local time"); + // this function uses static memory so it's best to copy it out asap just in case // otherwise other code that is using localtime may break this (not just python code) - std::tm localtime = *std::localtime(&tt); - - // Declare these special duration types so the conversions happen with the correct primitive types (int) - using us_t = duration; + std::tm localtime = *localtime_ptr; return PyDateTime_FromDateAndTime(localtime.tm_year + 1900, localtime.tm_mon + 1, @@ -164,7 +178,7 @@ template class type_caster(src.time_since_epoch() % seconds(1))).count()); + us.count()); } PYBIND11_TYPE_CASTER(type, _("datetime.datetime")); }; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index c03b32acab..2f414e5c7c 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -24,6 +24,18 @@ PYBIND11_NAMESPACE_BEGIN(detail) # define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) setattr((PyObject *) obj, "__qualname__", nameobj) #endif +inline std::string get_fully_qualified_tp_name(PyTypeObject *type) { +#if !defined(PYPY_VERSION) + return type->tp_name; +#else + auto module_name = handle((PyObject *) type).attr("__module__").cast(); + if (module_name == PYBIND11_BUILTINS_MODULE) + return type->tp_name; + else + return std::move(module_name) + "." + type->tp_name; +#endif +} + inline PyTypeObject *type_incref(PyTypeObject *type) { Py_INCREF(type); return type; @@ -117,7 +129,7 @@ extern "C" inline int pybind11_meta_setattro(PyObject* obj, PyObject* name, PyOb // 2. `Type.static_prop = other_static_prop` --> setattro: replace existing `static_prop` // 3. `Type.regular_attribute = value` --> setattro: regular attribute assignment const auto static_prop = (PyObject *) get_internals().static_property_type; - const auto call_descr_set = descr && PyObject_IsInstance(descr, static_prop) + const auto call_descr_set = descr && value && PyObject_IsInstance(descr, static_prop) && !PyObject_IsInstance(value, static_prop); if (call_descr_set) { // Call `static_property.__set__()` instead of replacing the `static_property`. @@ -169,10 +181,10 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P auto instance = reinterpret_cast(self); // Ensure that the base __init__ function(s) were called - for (auto vh : values_and_holders(instance)) { + for (const auto &vh : values_and_holders(instance)) { if (!vh.holder_constructed()) { PyErr_Format(PyExc_TypeError, "%.200s.__init__() must be called when overriding __init__", - vh.type->type->tp_name); + get_fully_qualified_tp_name(vh.type->type).c_str()); Py_DECREF(self); return nullptr; } @@ -181,6 +193,44 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P return self; } +/// Cleanup the type-info for a pybind11-registered type. +extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { + auto *type = (PyTypeObject *) obj; + auto &internals = get_internals(); + + // A pybind11-registered type will: + // 1) be found in internals.registered_types_py + // 2) have exactly one associated `detail::type_info` + auto found_type = internals.registered_types_py.find(type); + if (found_type != internals.registered_types_py.end() && + found_type->second.size() == 1 && + found_type->second[0]->type == type) { + + auto *tinfo = found_type->second[0]; + auto tindex = std::type_index(*tinfo->cpptype); + internals.direct_conversions.erase(tindex); + + if (tinfo->module_local) + registered_local_types_cpp().erase(tindex); + else + internals.registered_types_cpp.erase(tindex); + internals.registered_types_py.erase(tinfo->type); + + // Actually just `std::erase_if`, but that's only available in C++20 + auto &cache = internals.inactive_override_cache; + for (auto it = cache.begin(), last = cache.end(); it != last; ) { + if (it->first == (PyObject *) tinfo->type) + it = cache.erase(it); + else + ++it; + } + + delete tinfo; + } + + PyType_Type.tp_dealloc(obj); +} + /** This metaclass is assigned by default to all pybind11 types and is required in order for static properties to function correctly. Users may override this using `py::metaclass`. Return value: New reference. */ @@ -213,6 +263,8 @@ inline PyTypeObject* make_default_metaclass() { type->tp_getattro = pybind11_meta_getattro; #endif + type->tp_dealloc = pybind11_meta_dealloc; + if (PyType_Ready(type) < 0) pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!"); @@ -250,7 +302,7 @@ inline bool deregister_instance_impl(void *ptr, instance *self) { auto ®istered_instances = get_internals().registered_instances; auto range = registered_instances.equal_range(ptr); for (auto it = range.first; it != range.second; ++it) { - if (self == it->second && Py_TYPE(self) == Py_TYPE(it->second)) { + if (self == it->second) { registered_instances.erase(it); return true; } @@ -288,8 +340,6 @@ inline PyObject *make_new_instance(PyTypeObject *type) { // Allocate the value/holder internals: inst->allocate_layout(); - inst->owned = true; - return self; } @@ -304,12 +354,7 @@ extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, /// following default function will be used which simply throws an exception. extern "C" inline int pybind11_object_init(PyObject *self, PyObject *, PyObject *) { PyTypeObject *type = Py_TYPE(self); - std::string msg; -#if defined(PYPY_VERSION) - msg += handle((PyObject *) type).attr("__module__").cast() + "."; -#endif - msg += type->tp_name; - msg += ": No constructor defined!"; + std::string msg = get_fully_qualified_tp_name(type) + ": No constructor defined!"; PyErr_SetString(PyExc_TypeError, msg.c_str()); return -1; } @@ -448,7 +493,7 @@ extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) { extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) { if (!PyDict_Check(new_dict)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'", - Py_TYPE(new_dict)->tp_name); + get_fully_qualified_tp_name(Py_TYPE(new_dict)).c_str()); return -1; } PyObject *&dict = *_PyObject_GetDictPtr(self); @@ -475,11 +520,6 @@ extern "C" inline int pybind11_clear(PyObject *self) { /// Give instances of this type a `__dict__` and opt into garbage collection. inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto type = &heap_type->ht_type; -#if defined(PYPY_VERSION) && (PYPY_VERSION_NUM < 0x06000000) - pybind11_fail(std::string(type->tp_name) + ": dynamic attributes are " - "currently not supported in " - "conjunction with PyPy!"); -#endif type->tp_flags |= Py_TPFLAGS_HAVE_GC; type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t)sizeof(PyObject *); // and allocate enough space for it @@ -510,6 +550,12 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla } std::memset(view, 0, sizeof(Py_buffer)); buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { + delete info; + // view->obj = nullptr; // Was just memset to 0, so not necessary + PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); + return -1; + } view->obj = obj; view->ndim = 1; view->internal = info; @@ -519,12 +565,6 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla for (auto s : info->shape) view->len *= s; view->readonly = info->readonly; - if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { - if (view) - view->obj = nullptr; - PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); - return -1; - } if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) view->format = const_cast(info->format.c_str()); if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { @@ -567,17 +607,17 @@ inline PyObject* make_new_python_type(const type_record &rec) { #endif } - object module; + object module_; if (rec.scope) { if (hasattr(rec.scope, "__module__")) - module = rec.scope.attr("__module__"); + module_ = rec.scope.attr("__module__"); else if (hasattr(rec.scope, "__name__")) - module = rec.scope.attr("__name__"); + module_ = rec.scope.attr("__name__"); } auto full_name = c_str( #if !defined(PYPY_VERSION) - module ? str(module).cast() + "." + rec.name : + module_ ? str(module_).cast() + "." + rec.name : #endif rec.name); @@ -592,7 +632,7 @@ inline PyObject* make_new_python_type(const type_record &rec) { auto &internals = get_internals(); auto bases = tuple(rec.bases); - auto base = (bases.size() == 0) ? internals.instance_base + auto base = (bases.empty()) ? internals.instance_base : bases[0].ptr(); /* Danger zone: from now (and until PyType_Ready), make sure to @@ -616,7 +656,7 @@ inline PyObject* make_new_python_type(const type_record &rec) { type->tp_doc = tp_doc; type->tp_base = type_incref((PyTypeObject *)base); type->tp_basicsize = static_cast(sizeof(instance)); - if (bases.size() > 0) + if (!bases.empty()) type->tp_bases = bases.release().ptr(); /* Don't inherit base __init__ */ @@ -656,8 +696,8 @@ inline PyObject* make_new_python_type(const type_record &rec) { else Py_INCREF(type); // Keep it alive forever (reference leak) - if (module) // Needed by pydoc - setattr((PyObject *) type, "__module__", module); + if (module_) // Needed by pydoc + setattr((PyObject *) type, "__module__", module_); PYBIND11_SET_OLDPY_QUALNAME(type, qualname); diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9747611507..2bf2344638 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,7 +11,7 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 6 -#define PYBIND11_VERSION_PATCH dev0 +#define PYBIND11_VERSION_PATCH 3.dev1 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } @@ -27,7 +27,7 @@ # endif #endif -#if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER) +#if !(defined(_MSC_VER) && __cplusplus == 199711L) # if __cplusplus >= 201402L # define PYBIND11_CPP14 # if __cplusplus >= 201703L @@ -47,8 +47,10 @@ // Compiler version assertions #if defined(__INTEL_COMPILER) -# if __INTEL_COMPILER < 1700 -# error pybind11 requires Intel C++ compiler v17 or newer +# if __INTEL_COMPILER < 1800 +# error pybind11 requires Intel C++ compiler v18 or newer +# elif __INTEL_COMPILER < 1900 && defined(PYBIND11_CPP14) +# error pybind11 supports only C++11 with Intel C++ compiler v18. Use v19 or newer for C++14. # endif #elif defined(__clang__) && !defined(__apple_build_version__) # if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3) @@ -154,11 +156,30 @@ #include #include #include +#include #include #include #include #include #include +#if defined(__has_include) +# if __has_include() +# include +# endif +#endif + +// #define PYBIND11_STR_LEGACY_PERMISSIVE +// If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject +// (probably surprising and never documented, but this was the +// legacy behavior until and including v2.6.x). As a side-effect, +// pybind11::isinstance() is true for both pybind11::str and +// pybind11::bytes. +// If UNDEFINED, pybind11::str can only hold PyUnicodeObject, and +// pybind11::isinstance() is true only for pybind11::str. +// However, for Python 2 only (!), the pybind11::str caster +// implicitly decodes bytes to PyUnicodeObject. This is to ease +// the transition from the legacy behavior to the non-permissive +// behavior. #if PY_MAJOR_VERSION >= 3 /// Compatibility macros for various Python versions #define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyInstanceMethod_New(ptr) @@ -181,6 +202,7 @@ #define PYBIND11_STR_TYPE ::pybind11::str #define PYBIND11_BOOL_ATTR "__bool__" #define PYBIND11_NB_BOOL(ptr) ((ptr)->nb_bool) +#define PYBIND11_BUILTINS_MODULE "builtins" // Providing a separate declaration to make Clang's -Wmissing-prototypes happy. // See comment for PYBIND11_MODULE below for why this is marked "maybe unused". #define PYBIND11_PLUGIN_IMPL(name) \ @@ -208,6 +230,7 @@ #define PYBIND11_STR_TYPE ::pybind11::bytes #define PYBIND11_BOOL_ATTR "__nonzero__" #define PYBIND11_NB_BOOL(ptr) ((ptr)->nb_nonzero) +#define PYBIND11_BUILTINS_MODULE "__builtin__" // Providing a separate PyInit decl to make Clang's -Wmissing-prototypes happy. // See comment for PYBIND11_MODULE below for why this is marked "maybe unused". #define PYBIND11_PLUGIN_IMPL(name) \ @@ -262,13 +285,13 @@ extern "C" { ***Deprecated in favor of PYBIND11_MODULE*** This macro creates the entry point that will be invoked when the Python interpreter - imports a plugin library. Please create a `module` in the function body and return + imports a plugin library. Please create a `module_` in the function body and return the pointer to its underlying Python object at the end. .. code-block:: cpp PYBIND11_PLUGIN(example) { - pybind11::module m("example", "pybind11 example plugin"); + pybind11::module_ m("example", "pybind11 example plugin"); /// Set up bindings here return m.ptr(); } @@ -289,7 +312,7 @@ extern "C" { This macro creates the entry point that will be invoked when the Python interpreter imports an extension module. The module name is given as the fist argument and it should not be in quotes. The second macro argument defines a variable of type - `py::module` which can be used to initialize the module. + `py::module_` which can be used to initialize the module. The entry point is marked as "maybe unused" to aid dead-code detection analysis: since the entry point is typically only looked up at runtime and not referenced @@ -307,18 +330,22 @@ extern "C" { } \endrst */ #define PYBIND11_MODULE(name, variable) \ + static ::pybind11::module_::module_def \ + PYBIND11_CONCAT(pybind11_module_def_, name) PYBIND11_MAYBE_UNUSED; \ PYBIND11_MAYBE_UNUSED \ - static void PYBIND11_CONCAT(pybind11_init_, name)(pybind11::module &); \ + static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ PYBIND11_PLUGIN_IMPL(name) { \ PYBIND11_CHECK_PYTHON_VERSION \ PYBIND11_ENSURE_INTERNALS_READY \ - auto m = pybind11::module(PYBIND11_TOSTRING(name)); \ + auto m = ::pybind11::module_::create_extension_module( \ + PYBIND11_TOSTRING(name), nullptr, \ + &PYBIND11_CONCAT(pybind11_module_def_, name)); \ try { \ PYBIND11_CONCAT(pybind11_init_, name)(m); \ return m.ptr(); \ } PYBIND11_CATCH_INIT_EXCEPTIONS \ } \ - void PYBIND11_CONCAT(pybind11_init_, name)(pybind11::module &variable) + void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &variable) PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -531,7 +558,7 @@ struct instance { bool has_patients : 1; typedef void (*release_to_cpp_t)(instance* inst, holder_erased external_holder, object&& obj); - typedef object (*reclaim_from_cpp_t)(instance* inst, holder_erased external_holder); + using reclaim_from_cpp_t = object (*)(instance *, holder_erased); struct type_release_info_t { // Release an instance to C++ for pure C++ instances or Python-derived classes. @@ -588,7 +615,7 @@ using std::make_index_sequence; #else template struct index_sequence { }; template struct make_index_sequence_impl : make_index_sequence_impl { }; -template struct make_index_sequence_impl <0, S...> { typedef index_sequence type; }; +template struct make_index_sequence_impl <0, S...> { using type = index_sequence; }; template using make_index_sequence = typename make_index_sequence_impl::type; #endif @@ -602,8 +629,16 @@ template using select_indices = typename select_indices_impl using bool_constant = std::integral_constant; template struct negation : bool_constant { }; +// PGI/Intel cannot detect operator delete with the "compatible" void_t impl, so +// using the new one (C++14 defect, so generally works on newer compilers, even +// if not in C++17 mode) +#if defined(__PGIC__) || defined(__INTEL_COMPILER) +template using void_t = void; +#else template struct void_t_impl { using type = void; }; template using void_t = typename void_t_impl::type; +#endif + /// Compile-time all/any/none of that check the boolean value of all template types #if defined(__cpp_fold_expressions) && !(defined(_MSC_VER) && (_MSC_VER < 1916)) @@ -629,17 +664,17 @@ template class... Predicates> using satisfies_none_of /// Strip the class from a method type template struct remove_class { }; -template struct remove_class { typedef R type(A...); }; -template struct remove_class { typedef R type(A...); }; +template struct remove_class { using type = R (A...); }; +template struct remove_class { using type = R (A...); }; /// Helper template to strip away type modifiers -template struct intrinsic_type { typedef T type; }; -template struct intrinsic_type { typedef typename intrinsic_type::type type; }; -template struct intrinsic_type { typedef typename intrinsic_type::type type; }; -template struct intrinsic_type { typedef typename intrinsic_type::type type; }; -template struct intrinsic_type { typedef typename intrinsic_type::type type; }; -template struct intrinsic_type { typedef typename intrinsic_type::type type; }; -template struct intrinsic_type { typedef typename intrinsic_type::type type; }; +template struct intrinsic_type { using type = T; }; +template struct intrinsic_type { using type = typename intrinsic_type::type; }; +template struct intrinsic_type { using type = typename intrinsic_type::type; }; +template struct intrinsic_type { using type = typename intrinsic_type::type; }; +template struct intrinsic_type { using type = typename intrinsic_type::type; }; +template struct intrinsic_type { using type = typename intrinsic_type::type; }; +template struct intrinsic_type { using type = typename intrinsic_type::type; }; template using intrinsic_t = typename intrinsic_type::type; /// Helper type to replace 'void' in some expressions @@ -710,8 +745,9 @@ template using is_strict_base_of = bool_consta /// Like is_base_of, but also requires that the base type is accessible (i.e. that a Derived pointer /// can be converted to a Base pointer) +/// For unions, `is_base_of::value` is False, so we need to check `is_same` as well. template using is_accessible_base_of = bool_constant< - std::is_base_of::value && std::is_convertible::value>; + (std::is_same::value || std::is_base_of::value) && std::is_convertible::value>; template class Base> struct is_template_base_of_impl { @@ -748,6 +784,10 @@ template using is_function_pointer = bool_constant< std::is_pointer::value && std::is_function::type>::value>; template struct strip_function_object { + // If you are encountering an + // 'error: name followed by "::" must be a class or namespace name' + // with the Intel compiler and a noexcept function here, + // try to use noexcept(true) instead of plain noexcept. using type = typename remove_class::type; }; @@ -772,8 +812,10 @@ template using is_lambda = satisfies_none_of, /// Ignore that a variable is unused in compiler warnings inline void ignore_unused(const int *) { } +// [workaround(intel)] Internal error on fold expression /// Apply a function over each element of a parameter pack -#ifdef __cpp_fold_expressions +#if defined(__cpp_fold_expressions) && !defined(__INTEL_COMPILER) +// Intel compiler produces an internal error on this fold expression (tested with ICC 19.0.2) #define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) (((PATTERN), void()), ...) #else using expand_side_effects = bool[]; @@ -853,7 +895,7 @@ struct nodelete { template void operator()(T*) { } }; PYBIND11_NAMESPACE_BEGIN(detail) template struct overload_cast_impl { - constexpr overload_cast_impl() {} // MSVC 2015 needs this + constexpr overload_cast_impl() {}; // NOLINT(modernize-use-equals-default): MSVC 2015 needs this template constexpr auto operator()(Return (*pf)(Args...)) const noexcept @@ -931,8 +973,26 @@ class any_container { const std::vector *operator->() const { return &v; } }; -PYBIND11_NAMESPACE_END(detail) - +// Forward-declaration; see detail/class.h +std::string get_fully_qualified_tp_name(PyTypeObject*); +template +inline static std::shared_ptr try_get_shared_from_this(std::enable_shared_from_this *holder_value_ptr) { +// Pre C++17, this code path exploits undefined behavior, but is known to work on many platforms. +// Use at your own risk! +// See also https://en.cppreference.com/w/cpp/memory/enable_shared_from_this, and in particular +// the `std::shared_ptr gp1 = not_so_good.getptr();` and `try`-`catch` parts of the example. +#if defined(__cpp_lib_enable_shared_from_this) && (!defined(_MSC_VER) || _MSC_VER >= 1912) + return holder_value_ptr->weak_from_this().lock(); +#else + try { + return holder_value_ptr->shared_from_this(); + } + catch (const std::bad_weak_ptr &) { + return nullptr; + } +#endif +} +PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 35b95bcfae..3ef78c1179 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -132,6 +132,7 @@ void construct(value_and_holder &v_h, Alias *alias_ptr, bool) { template void construct(value_and_holder &v_h, Holder holder, bool need_alias) { auto *ptr = holder_helper>::get(holder); + no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance if (Class::has_alias && need_alias && !is_alias(ptr)) throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index e57cdad030..dd66ceb513 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -82,7 +82,7 @@ struct type_equal_to { template using type_map = std::unordered_map; -struct overload_hash { +struct override_hash { inline size_t operator()(const std::pair& v) const { size_t value = std::hash()(v.first); value ^= std::hash()(v.second); @@ -97,7 +97,7 @@ struct internals { type_map registered_types_cpp; // std::type_index -> pybind11's type information std::unordered_map> registered_types_py; // PyTypeObject* -> base type_info(s) std::unordered_multimap registered_instances; // void * -> instance* - std::unordered_set, overload_hash> inactive_overload_cache; + std::unordered_set, override_hash> inactive_override_cache; type_map> direct_conversions; std::unordered_map> patients; std::forward_list registered_exception_translators; @@ -112,7 +112,7 @@ struct internals { PyInterpreterState *istate = nullptr; ~internals() { // This destructor is called *after* Py_Finalize() in finalize_interpreter(). - // That *SHOULD BE* fine. The following details what happens whe PyThread_tss_free is called. + // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is called. // PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does nothing. // PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). Neither @@ -156,49 +156,60 @@ struct type_info { /// On MSVC, debug and release builds are not ABI-compatible! #if defined(_MSC_VER) && defined(_DEBUG) -# define PYBIND11_BUILD_TYPE "_debug" +# define PYBIND11_BUILD_TYPE "_debug" #else -# define PYBIND11_BUILD_TYPE "" +# define PYBIND11_BUILD_TYPE "" #endif /// Let's assume that different compilers are ABI-incompatible. -#if defined(_MSC_VER) -# define PYBIND11_COMPILER_TYPE "_msvc" -#elif defined(__INTEL_COMPILER) -# define PYBIND11_COMPILER_TYPE "_icc" -#elif defined(__clang__) -# define PYBIND11_COMPILER_TYPE "_clang" -#elif defined(__PGI) -# define PYBIND11_COMPILER_TYPE "_pgi" -#elif defined(__MINGW32__) -# define PYBIND11_COMPILER_TYPE "_mingw" -#elif defined(__CYGWIN__) -# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" -#elif defined(__GNUC__) -# define PYBIND11_COMPILER_TYPE "_gcc" -#else -# define PYBIND11_COMPILER_TYPE "_unknown" +/// A user can manually set this string if they know their +/// compiler is compatible. +#ifndef PYBIND11_COMPILER_TYPE +# if defined(_MSC_VER) +# define PYBIND11_COMPILER_TYPE "_msvc" +# elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_TYPE "_icc" +# elif defined(__clang__) +# define PYBIND11_COMPILER_TYPE "_clang" +# elif defined(__PGI) +# define PYBIND11_COMPILER_TYPE "_pgi" +# elif defined(__MINGW32__) +# define PYBIND11_COMPILER_TYPE "_mingw" +# elif defined(__CYGWIN__) +# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" +# elif defined(__GNUC__) +# define PYBIND11_COMPILER_TYPE "_gcc" +# else +# define PYBIND11_COMPILER_TYPE "_unknown" +# endif #endif -#if defined(_LIBCPP_VERSION) -# define PYBIND11_STDLIB "_libcpp" -#elif defined(__GLIBCXX__) || defined(__GLIBCPP__) -# define PYBIND11_STDLIB "_libstdcpp" -#else -# define PYBIND11_STDLIB "" +/// Also standard libs +#ifndef PYBIND11_STDLIB +# if defined(_LIBCPP_VERSION) +# define PYBIND11_STDLIB "_libcpp" +# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) +# define PYBIND11_STDLIB "_libstdcpp" +# else +# define PYBIND11_STDLIB "" +# endif #endif /// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. -#if defined(__GXX_ABI_VERSION) -# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) -#else -# define PYBIND11_BUILD_ABI "" +#ifndef PYBIND11_BUILD_ABI +# if defined(__GXX_ABI_VERSION) +# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) +# else +# define PYBIND11_BUILD_ABI "" +# endif #endif -#if defined(WITH_THREAD) -# define PYBIND11_INTERNALS_KIND "" -#else -# define PYBIND11_INTERNALS_KIND "_without_thread" +#ifndef PYBIND11_INTERNALS_KIND +# if defined(WITH_THREAD) +# define PYBIND11_INTERNALS_KIND "" +# else +# define PYBIND11_INTERNALS_KIND "_without_thread" +# endif #endif #define PYBIND11_INTERNALS_ID "__pybind11_internals_v" \ @@ -257,7 +268,7 @@ PYBIND11_NOINLINE inline internals &get_internals() { const PyGILState_STATE state; } gil; - constexpr auto *id = PYBIND11_INTERNALS_ID; + PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); auto builtins = handle(PyEval_GetBuiltins()); if (builtins.contains(id) && isinstance(builtins[id])) { internals_pp = static_cast(capsule(builtins[id])); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h new file mode 100644 index 0000000000..2423e24e9f --- /dev/null +++ b/include/pybind11/detail/type_caster_base.h @@ -0,0 +1,1063 @@ +/* + pybind11/detail/type_caster_base.h (originally first part of pybind11/cast.h) + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../pytypes.h" +#include "common.h" +#include "descr.h" +#include "internals.h" +#include "typeid.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +/// A life support system for temporary objects created by `type_caster::load()`. +/// Adding a patient will keep it alive up until the enclosing function returns. +class loader_life_support { +public: + /// A new patient frame is created when a function is entered + loader_life_support() { + get_internals().loader_patient_stack.push_back(nullptr); + } + + /// ... and destroyed after it returns + ~loader_life_support() { + auto &stack = get_internals().loader_patient_stack; + if (stack.empty()) + pybind11_fail("loader_life_support: internal error"); + + auto ptr = stack.back(); + stack.pop_back(); + Py_CLEAR(ptr); + + // A heuristic to reduce the stack's capacity (e.g. after long recursive calls) + if (stack.capacity() > 16 && !stack.empty() && stack.capacity() / stack.size() > 2) + stack.shrink_to_fit(); + } + + /// This can only be used inside a pybind11-bound function, either by `argument_loader` + /// at argument preparation time or by `py::cast()` at execution time. + PYBIND11_NOINLINE static void add_patient(handle h) { + auto &stack = get_internals().loader_patient_stack; + if (stack.empty()) + throw cast_error("When called outside a bound function, py::cast() cannot " + "do Python -> C++ conversions which require the creation " + "of temporary values"); + + auto &list_ptr = stack.back(); + if (list_ptr == nullptr) { + list_ptr = PyList_New(1); + if (!list_ptr) + pybind11_fail("loader_life_support: error allocating list"); + PyList_SET_ITEM(list_ptr, 0, h.inc_ref().ptr()); + } else { + auto result = PyList_Append(list_ptr, h.ptr()); + if (result == -1) + pybind11_fail("loader_life_support: error adding patient"); + } + } +}; + +// Gets the cache entry for the given type, creating it if necessary. The return value is the pair +// returned by emplace, i.e. an iterator for the entry and a bool set to `true` if the entry was +// just created. +inline std::pair all_type_info_get_cache(PyTypeObject *type); + +// Populates a just-created cache entry. +PYBIND11_NOINLINE inline void all_type_info_populate(PyTypeObject *t, std::vector &bases) { + std::vector check; + for (handle parent : reinterpret_borrow(t->tp_bases)) + check.push_back((PyTypeObject *) parent.ptr()); + + auto const &type_dict = get_internals().registered_types_py; + for (size_t i = 0; i < check.size(); i++) { + auto type = check[i]; + // Ignore Python2 old-style class super type: + if (!PyType_Check((PyObject *) type)) continue; + + // Check `type` in the current set of registered python types: + auto it = type_dict.find(type); + if (it != type_dict.end()) { + // We found a cache entry for it, so it's either pybind-registered or has pre-computed + // pybind bases, but we have to make sure we haven't already seen the type(s) before: we + // want to follow Python/virtual C++ rules that there should only be one instance of a + // common base. + for (auto *tinfo : it->second) { + // NB: Could use a second set here, rather than doing a linear search, but since + // having a large number of immediate pybind11-registered types seems fairly + // unlikely, that probably isn't worthwhile. + bool found = false; + for (auto *known : bases) { + if (known == tinfo) { found = true; break; } + } + if (!found) bases.push_back(tinfo); + } + } + else if (type->tp_bases) { + // It's some python type, so keep follow its bases classes to look for one or more + // registered types + if (i + 1 == check.size()) { + // When we're at the end, we can pop off the current element to avoid growing + // `check` when adding just one base (which is typical--i.e. when there is no + // multiple inheritance) + check.pop_back(); + i--; + } + for (handle parent : reinterpret_borrow(type->tp_bases)) + check.push_back((PyTypeObject *) parent.ptr()); + } + } +} + +/** + * Extracts vector of type_info pointers of pybind-registered roots of the given Python type. Will + * be just 1 pybind type for the Python type of a pybind-registered class, or for any Python-side + * derived class that uses single inheritance. Will contain as many types as required for a Python + * class that uses multiple inheritance to inherit (directly or indirectly) from multiple + * pybind-registered classes. Will be empty if neither the type nor any base classes are + * pybind-registered. + * + * The value is cached for the lifetime of the Python type. + */ +inline const std::vector &all_type_info(PyTypeObject *type) { + auto ins = all_type_info_get_cache(type); + if (ins.second) + // New cache entry: populate it + all_type_info_populate(type, ins.first->second); + + return ins.first->second; +} + +/** + * Gets a single pybind11 type info for a python type. Returns nullptr if neither the type nor any + * ancestors are pybind11-registered. Throws an exception if there are multiple bases--use + * `all_type_info` instead if you want to support multiple bases. + */ +PYBIND11_NOINLINE inline detail::type_info* get_type_info(PyTypeObject *type, bool do_throw = true) { + auto &bases = all_type_info(type); + if (bases.empty()) + return nullptr; + if (bases.size() > 1) { + if (do_throw) + pybind11_fail("pybind11::detail::get_type_info: type has multiple pybind11-registered bases"); + else + return nullptr; + } + return bases.front(); +} + +inline detail::type_info *get_local_type_info(const std::type_index &tp) { + auto &locals = registered_local_types_cpp(); + auto it = locals.find(tp); + if (it != locals.end()) + return it->second; + return nullptr; +} + +inline detail::type_info *get_global_type_info(const std::type_index &tp) { + auto &types = get_internals().registered_types_cpp; + auto it = types.find(tp); + if (it != types.end()) + return it->second; + return nullptr; +} + +/// Return the type info for a given C++ type; on lookup failure can either throw or return nullptr. +PYBIND11_NOINLINE inline detail::type_info *get_type_info(const std::type_index &tp, + bool throw_if_missing = false) { + if (auto ltype = get_local_type_info(tp)) + return ltype; + if (auto gtype = get_global_type_info(tp)) + return gtype; + + if (throw_if_missing) { + std::string tname = tp.name(); + detail::clean_type_id(tname); + pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + tname + "\""); + } + return nullptr; +} + +PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { + detail::type_info *type_info = get_type_info(tp, throw_if_missing); + return handle(type_info ? ((PyObject *) type_info->type) : nullptr); +} + +// Searches the inheritance graph for a registered Python instance, using all_type_info(). +PYBIND11_NOINLINE inline handle find_registered_python_instance( + void *src, const detail::type_info *tinfo, + bool should_take_ownership, holder_erased existing_holder) { + // We only come across `!existing_holder` if we are coming from `cast` and not `cast_holder`. + const bool is_bare_ptr = !existing_holder.ptr() && existing_holder.type_id() == HolderTypeId::Unknown; + + auto it_instances = get_internals().registered_instances.equal_range(src); + for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { + for (auto instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { + if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { + instance* const inst = it_i->second; + + bool try_to_reclaim = false; + if (!is_bare_ptr) { + switch (instance_type->release_info.holder_type_id) { + case detail::HolderTypeId::UniquePtr: { + try_to_reclaim = should_take_ownership; + break; + } + case detail::HolderTypeId::SharedPtr: { + if (should_take_ownership) { + // Only try to reclaim the object if (a) it is not owned and (b) has no holder. + if (!inst->simple_holder_constructed) { + if (inst->owned) + throw std::runtime_error("Internal error?"); + try_to_reclaim = true; + } + } + break; + } + default: { + // Otherwise, do not try any reclaiming. + break; + } + } + } + if (try_to_reclaim) { + // If this object has already been registered, but we wish to take ownership of it, + // then use the `has_cpp_release` mechanisms to reclaim ownership. + // @note This should be the sole occurrence of this registered object when releasing back. + // @note This code path should not be invoked for pure C++ + + // TODO(eric.cousineau): This may be still be desirable if this is a raw pointer... + // Need to think of a desirable workflow - and if there is possible interop. + if (!existing_holder) { + throw std::runtime_error("Internal error: Should have non-null holder."); + } + // TODO(eric.cousineau): This field may not be necessary if the lowest-level type is valid. + // See `move_only_holder_caster::load_value`. + if (!inst->reclaim_from_cpp) { + throw std::runtime_error("Instance is registered but does not have a registered reclaim method. Internal error?"); + } + return inst->reclaim_from_cpp(inst, existing_holder).release(); + } else { + // TODO(eric.cousineau): Should really check that ownership is consistent. + // e.g. if we say to take ownership of a pointer that is passed, does not have a holder... + // In the end, pybind11 would let ownership slip, and leak memory, possibly violating RAII (if someone is using that...) + return handle((PyObject *) it_i->second).inc_ref(); + } + } + } + } + return handle(); +} + +struct value_and_holder { + instance *inst = nullptr; + size_t index = 0u; + const detail::type_info *type = nullptr; + void **vh = nullptr; + + // Main constructor for a found value/holder: + value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) : + inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder : &inst->nonsimple.values_and_holders[vpos]} + {} + + // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) + value_and_holder() = default; + + // Used for past-the-end iterator + value_and_holder(size_t index) : index{index} {} + + template V *&value_ptr() const { + return reinterpret_cast(vh[0]); + } + // True if this `value_and_holder` has a non-null value pointer + explicit operator bool() const { return value_ptr(); } + + template H &holder() const { + return reinterpret_cast(vh[1]); + } + void* holder_ptr() const { return &vh[1]; } + bool holder_constructed() const { + return inst->simple_layout + ? inst->simple_holder_constructed + : inst->nonsimple.status[index] & instance::status_holder_constructed; + } + void set_holder_constructed(bool v = true) { + if (inst->simple_layout) + inst->simple_holder_constructed = v; + else if (v) + inst->nonsimple.status[index] |= instance::status_holder_constructed; + else + inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed; + } + bool instance_registered() const { + return inst->simple_layout + ? inst->simple_instance_registered + : inst->nonsimple.status[index] & instance::status_instance_registered; + } + void set_instance_registered(bool v = true) { + if (inst->simple_layout) + inst->simple_instance_registered = v; + else if (v) + inst->nonsimple.status[index] |= instance::status_instance_registered; + else + inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered; + } +}; + +// Container for accessing and iterating over an instance's values/holders +struct values_and_holders { +private: + instance *inst; + using type_vec = std::vector; + const type_vec &tinfo; + +public: + values_and_holders(instance *inst) : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} + + struct iterator { + private: + instance *inst = nullptr; + const type_vec *types = nullptr; + value_and_holder curr; + friend struct values_and_holders; + iterator(instance *inst, const type_vec *tinfo) + : inst{inst}, types{tinfo}, + curr(inst /* instance */, + types->empty() ? nullptr : (*types)[0] /* type info */, + 0, /* vpos: (non-simple types only): the first vptr comes first */ + 0 /* index */) + {} + // Past-the-end iterator: + iterator(size_t end) : curr(end) {} + public: + bool operator==(const iterator &other) const { return curr.index == other.curr.index; } + bool operator!=(const iterator &other) const { return curr.index != other.curr.index; } + iterator &operator++() { + if (!inst->simple_layout) + curr.vh += 1 + (*types)[curr.index]->holder_size_in_ptrs; + ++curr.index; + curr.type = curr.index < types->size() ? (*types)[curr.index] : nullptr; + return *this; + } + value_and_holder &operator*() { return curr; } + value_and_holder *operator->() { return &curr; } + }; + + iterator begin() { return iterator(inst, &tinfo); } + iterator end() { return iterator(tinfo.size()); } + + iterator find(const type_info *find_type) { + auto it = begin(), endit = end(); + while (it != endit && it->type != find_type) ++it; + return it; + } + + size_t size() { return tinfo.size(); } +}; + +/** + * Extracts C++ value and holder pointer references from an instance (which may contain multiple + * values/holders for python-side multiple inheritance) that match the given type. Throws an error + * if the given type (or ValueType, if omitted) is not a pybind11 base of the given instance. If + * `find_type` is omitted (or explicitly specified as nullptr) the first value/holder are returned, + * regardless of type (and the resulting .type will be nullptr). + * + * The returned object should be short-lived: in particular, it must not outlive the called-upon + * instance. + */ +PYBIND11_NOINLINE inline value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { + // Optimize common case: + if (!find_type || Py_TYPE(this) == find_type->type) + return value_and_holder(this, find_type, 0, 0); + + detail::values_and_holders vhs(this); + auto it = vhs.find(find_type); + if (it != vhs.end()) + return *it; + + if (!throw_if_missing) + return value_and_holder(); + +#if defined(NDEBUG) + pybind11_fail("pybind11::detail::instance::get_value_and_holder: " + "type is not a pybind11 base of the given instance " + "(compile in debug mode for type details)"); +#else + pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + + get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" + + get_fully_qualified_tp_name(Py_TYPE(this)) + "' instance"); +#endif +} + +PYBIND11_NOINLINE inline void instance::allocate_layout() { + auto &tinfo = all_type_info(Py_TYPE(this)); + + const size_t n_types = tinfo.size(); + + if (n_types == 0) + pybind11_fail("instance allocation failed: new instance has no pybind11-registered base types"); + + simple_layout = + n_types == 1 && tinfo.front()->holder_size_in_ptrs <= instance_simple_holder_in_ptrs(); + + // Simple path: no python-side multiple inheritance, and a small-enough holder + if (simple_layout) { + simple_value_holder[0] = nullptr; + simple_holder_constructed = false; + simple_instance_registered = false; + } + else { // multiple base types or a too-large holder + // Allocate space to hold: [v1*][h1][v2*][h2]...[bb...] where [vN*] is a value pointer, + // [hN] is the (uninitialized) holder instance for value N, and [bb...] is a set of bool + // values that tracks whether each associated holder has been initialized. Each [block] is + // padded, if necessary, to an integer multiple of sizeof(void *). + size_t space = 0; + for (auto t : tinfo) { + space += 1; // value pointer + space += t->holder_size_in_ptrs; // holder instance + } + size_t flags_at = space; + space += size_in_ptrs(n_types); // status bytes (holder_constructed and instance_registered) + + // Allocate space for flags, values, and holders, and initialize it to 0 (flags and values, + // in particular, need to be 0). Use Python's memory allocation functions: in Python 3.6 + // they default to using pymalloc, which is designed to be efficient for small allocations + // like the one we're doing here; in earlier versions (and for larger allocations) they are + // just wrappers around malloc. +#if PY_VERSION_HEX >= 0x03050000 + nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); + if (!nonsimple.values_and_holders) throw std::bad_alloc(); +#else + nonsimple.values_and_holders = (void **) PyMem_New(void *, space); + if (!nonsimple.values_and_holders) throw std::bad_alloc(); + std::memset(nonsimple.values_and_holders, 0, space * sizeof(void *)); +#endif + nonsimple.status = reinterpret_cast(&nonsimple.values_and_holders[flags_at]); + } + owned = true; +} + +PYBIND11_NOINLINE inline void instance::deallocate_layout() { + if (!simple_layout) + PyMem_Free(nonsimple.values_and_holders); +} + +PYBIND11_NOINLINE inline bool isinstance_generic(handle obj, const std::type_info &tp) { + handle type = detail::get_type_handle(tp, false); + if (!type) + return false; + return isinstance(obj, type); +} + +PYBIND11_NOINLINE inline std::string error_string() { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Unknown internal error occurred"); + return "Unknown internal error occurred"; + } + + error_scope scope; // Preserve error state + + std::string errorString; + if (scope.type) { + errorString += handle(scope.type).attr("__name__").cast(); + errorString += ": "; + } + if (scope.value) + errorString += (std::string) str(scope.value); + + PyErr_NormalizeException(&scope.type, &scope.value, &scope.trace); + +#if PY_MAJOR_VERSION >= 3 + if (scope.trace != nullptr) + PyException_SetTraceback(scope.value, scope.trace); +#endif + +#if !defined(PYPY_VERSION) + if (scope.trace) { + auto *trace = (PyTracebackObject *) scope.trace; + + /* Get the deepest trace possible */ + while (trace->tb_next) + trace = trace->tb_next; + + PyFrameObject *frame = trace->tb_frame; + errorString += "\n\nAt:\n"; + while (frame) { + int lineno = PyFrame_GetLineNumber(frame); + errorString += + " " + handle(frame->f_code->co_filename).cast() + + "(" + std::to_string(lineno) + "): " + + handle(frame->f_code->co_name).cast() + "\n"; + frame = frame->f_back; + } + } +#endif + + return errorString; +} + +PYBIND11_NOINLINE inline handle get_object_handle(const void *ptr, const detail::type_info *type ) { + auto &instances = get_internals().registered_instances; + auto range = instances.equal_range(ptr); + for (auto it = range.first; it != range.second; ++it) { + for (const auto &vh : values_and_holders(it->second)) { + if (vh.type == type) + return handle((PyObject *) it->second); + } + } + return handle(); +} + +inline PyThreadState *get_thread_state_unchecked() { +#if defined(PYPY_VERSION) + return PyThreadState_GET(); +#elif PY_VERSION_HEX < 0x03000000 + return _PyThreadState_Current; +#elif PY_VERSION_HEX < 0x03050000 + return (PyThreadState*) _Py_atomic_load_relaxed(&_PyThreadState_Current); +#elif PY_VERSION_HEX < 0x03050200 + return (PyThreadState*) _PyThreadState_Current.value; +#else + return _PyThreadState_UncheckedGet(); +#endif +} + +// Forward declarations +inline void keep_alive_impl(handle nurse, handle patient); +inline PyObject *make_new_instance(PyTypeObject *type); + +enum class LoadType { + PureCpp, + DerivedCppSinglePySingle, + DerivedCppSinglePyMulti, + DerivedCppMulti, + /// Polymorphic casting or copy-based casting may be necessary. + ConversionNeeded, +}; + +typedef type_info* base_ptr_t; +using bases_t = const std::vector; + +inline LoadType determine_load_type(handle src, const type_info* typeinfo, + const bases_t** out_bases = nullptr, + base_ptr_t* out_base = nullptr) { + // Null out inputs. + if (out_bases) + *out_bases = nullptr; + if (out_base) + *out_base = nullptr; + PyTypeObject *srctype = Py_TYPE(src.ptr()); + // See `type_caster_generic::load_impl` below for more detail on comments. + + // Case 1: If src is an exact type match for the target type then we can reinterpret_cast + // the instance's value pointer to the target type: + if (srctype == typeinfo->type) { + // TODO(eric.cousineau): Determine if the type is upcast from a type, which is + // still a pure C++ object? + return LoadType::PureCpp; + } + // Case 2: We have a derived class + else if (PyType_IsSubtype(srctype, typeinfo->type)) { + const bases_t& bases = all_type_info(srctype); + if (out_bases) + *out_bases = &bases; // Copy to output for caching. + const bool no_cpp_mi = typeinfo->simple_type; + // Case 2a: the python type is a Python-inherited derived class that inherits from just + // one simple (no MI) pybind11 class, or is an exact match, so the C++ instance is of + // the right type and we can use reinterpret_cast. + // (This is essentially the same as case 2b, but because not using multiple inheritance + // is extremely common, we handle it specially to avoid the loop iterator and type + // pointer lookup overhead) + // TODO(eric.cousineau): This seems to also capture C++-registered classes as well, not just Python-derived + // classes. + if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { + return LoadType::DerivedCppSinglePySingle; + } + // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see if + // we can find an exact match (or, for a simple C++ type, an inherited match); if so, we + // can safely reinterpret_cast to the relevant pointer. + else if (bases.size() > 1) { + for (auto base : bases) { + if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) : base->type == typeinfo->type) { + if (out_base) { + *out_base = base; + } + return LoadType::DerivedCppSinglePyMulti; + } + } + } + // Case 2c: C++ multiple inheritance is involved and we couldn't find an exact type match + // in the registered bases, above, so try implicit casting (needed for proper C++ casting + // when MI is involved). + return LoadType::DerivedCppMulti; + } else { + return LoadType::ConversionNeeded; + } +} + +class type_caster_generic { +public: + PYBIND11_NOINLINE type_caster_generic(const std::type_info &type_info) + : typeinfo(get_type_info(type_info)), cpptype(&type_info) { } + + type_caster_generic(const type_info *typeinfo) + : typeinfo(typeinfo), cpptype(typeinfo ? typeinfo->cpptype : nullptr) { } + + bool load(handle src, bool convert) { + return load_impl(src, convert); + } + + PYBIND11_NOINLINE static handle cast(const void *_src, return_value_policy policy, handle parent, + const detail::type_info *tinfo, + void *(*copy_constructor)(const void *), + void *(*move_constructor)(const void *), + holder_erased existing_holder = {}) { + if (!tinfo) // no type info: error will be set already + return handle(); + + void *src = const_cast(_src); + if (src == nullptr) + return none().release(); + + const bool should_take_ownership = policy == return_value_policy::automatic || policy == return_value_policy::take_ownership; + + if (handle registered_inst = find_registered_python_instance(src, tinfo, should_take_ownership, existing_holder)) + return registered_inst; + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto wrapper = reinterpret_cast(inst.ptr()); + wrapper->owned = false; + void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); + + switch (policy) { + case return_value_policy::automatic: + case return_value_policy::take_ownership: + valueptr = src; + wrapper->owned = true; + break; + + case return_value_policy::automatic_reference: + case return_value_policy::reference: + valueptr = src; + wrapper->owned = false; + break; + + case return_value_policy::copy: + if (copy_constructor) + valueptr = copy_constructor(src); + else { +#if defined(NDEBUG) + throw cast_error("return_value_policy = copy, but type is " + "non-copyable! (compile in debug mode for details)"); +#else + std::string type_name(tinfo->cpptype->name()); + detail::clean_type_id(type_name); + throw cast_error("return_value_policy = copy, but type " + + type_name + " is non-copyable!"); +#endif + } + wrapper->owned = true; + break; + + case return_value_policy::move: + if (move_constructor) + valueptr = move_constructor(src); + else if (copy_constructor) + valueptr = copy_constructor(src); + else { +#if defined(NDEBUG) + throw cast_error("return_value_policy = move, but type is neither " + "movable nor copyable! " + "(compile in debug mode for details)"); +#else + std::string type_name(tinfo->cpptype->name()); + detail::clean_type_id(type_name); + throw cast_error("return_value_policy = move, but type " + + type_name + " is neither movable nor copyable!"); +#endif + } + wrapper->owned = true; + break; + + case return_value_policy::reference_internal: + valueptr = src; + wrapper->owned = false; + keep_alive_impl(inst, parent); + break; + + default: + throw cast_error("unhandled return_value_policy: should not happen!"); + } + + // TODO(eric.cousineau): Propagate `holder_erased` through this chain. + tinfo->init_instance(wrapper, existing_holder); + + return inst.release(); + } + + // Base methods for generic caster; there are overridden in copyable_holder_caster + void load_value(value_and_holder &&v_h, LoadType) { + auto *&vptr = v_h.value_ptr(); + // Lazy allocation for unallocated values: + if (vptr == nullptr) { + auto *type = v_h.type ? v_h.type : typeinfo; + if (type->operator_new) { + vptr = type->operator_new(type->type_size); + } else { + #if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) + if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) + vptr = ::operator new(type->type_size, + std::align_val_t(type->type_align)); + else + #endif + vptr = ::operator new(type->type_size); + } + } + value = vptr; + } + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + type_caster_generic sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + return true; + } + } + return false; + } + bool try_direct_conversions(handle src) { + for (auto &converter : *typeinfo->direct_conversions) { + if (converter(src.ptr(), value)) + return true; + } + return false; + } + void check_holder_compat() {} + + PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { + auto caster = type_caster_generic(ti); + if (caster.load(src, false)) + return caster.value; + return nullptr; + } + + /// Try to load with foreign typeinfo, if available. Used when there is no + /// native typeinfo, or when the native one wasn't able to produce a value. + PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { + constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; + const auto pytype = type::handle_of(src); + if (!hasattr(pytype, local_key)) + return false; + + type_info *foreign_typeinfo = reinterpret_borrow(getattr(pytype, local_key)); + // Only consider this foreign loader if actually foreign and is a loader of the correct cpp type + if (foreign_typeinfo->module_local_load == &local_load + || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) + return false; + + if (auto result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { + value = result; + return true; + } + return false; + } + + // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant + // bits of code between here and copyable_holder_caster where the two classes need different + // logic (without having to resort to virtual inheritance). + template + PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { + if (!src) return false; + if (!typeinfo) return try_load_foreign_module_local(src); + if (src.is_none()) { + // Defer accepting None to other overloads (if we aren't in convert mode): + if (!convert) return false; + value = nullptr; + return true; + } + + auto &this_ = static_cast(*this); + this_.check_holder_compat(); + + const bases_t* bases = nullptr; + base_ptr_t base_py_multi = nullptr; + LoadType load_type = determine_load_type(src, typeinfo, &bases, &base_py_multi); + switch (load_type) { + case LoadType::PureCpp: { + this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(), + load_type); + return true; + } + case LoadType::DerivedCppSinglePySingle: { + this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(), + load_type); + return true; + } + case LoadType::DerivedCppSinglePyMulti: { + this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(base_py_multi), + load_type); + return true; + } + case LoadType::DerivedCppMulti: { + if (this_.try_implicit_casts(src, convert)) + return true; + } + case LoadType::ConversionNeeded: { + break; + } + } + + // If nothing else succeeds, perform an implicit conversion + if (convert) { + for (auto &converter : typeinfo->implicit_conversions) { + auto temp = reinterpret_steal(converter(src.ptr(), typeinfo->type)); + if (load_impl(temp, false)) { + loader_life_support::add_patient(temp); + return true; + } + } + if (this_.try_direct_conversions(src)) + return true; + } + + // Failed to match local typeinfo. Try again with global. + if (typeinfo->module_local) { + if (auto gtype = get_global_type_info(*typeinfo->cpptype)) { + typeinfo = gtype; + return load(src, false); + } + } + + // Global typeinfo has precedence over foreign module_local + return try_load_foreign_module_local(src); + } + + + // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast + // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair + // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). + PYBIND11_NOINLINE static std::pair src_and_type( + const void *src, const std::type_info &cast_type, const std::type_info *rtti_type = nullptr) { + if (auto *tpi = get_type_info(cast_type)) + return {src, const_cast(tpi)}; + + // Not found, set error: + std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); + detail::clean_type_id(tname); + std::string msg = "Unregistered type : " + tname; + PyErr_SetString(PyExc_TypeError, msg.c_str()); + return {nullptr, nullptr}; + } + + const type_info *typeinfo = nullptr; + const std::type_info *cpptype = nullptr; + void *value = nullptr; +}; + +/** + * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster + * needs to provide `operator T*()` and `operator T&()` operators. + * + * If the type supports moving the value away via an `operator T&&() &&` method, it should use + * `movable_cast_op_type` instead. + */ +template +using cast_op_type = + conditional_t>::value, + typename std::add_pointer>::type, + typename std::add_lvalue_reference>::type>; + +/** + * Determine suitable casting operator for a type caster with a movable value. Such a type caster + * needs to provide `operator T*()`, `operator T&()`, and `operator T&&() &&`. The latter will be + * called in appropriate contexts where the value can be moved rather than copied. + * + * These operator are automatically provided when using the PYBIND11_TYPE_CASTER macro. + */ +template +using movable_cast_op_type = + conditional_t::type>::value, + typename std::add_pointer>::type, + conditional_t::value, + typename std::add_rvalue_reference>::type, + typename std::add_lvalue_reference>::type>>; + +// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template struct is_copy_constructible : std::is_copy_constructible {}; + +// Specialization for types that appear to be copy constructible but also look like stl containers +// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if +// so, copy constructability depends on whether the value_type is copy constructible. +template struct is_copy_constructible, + std::is_same, + // Avoid infinite recursion + negation> + >::value>> : is_copy_constructible {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the copy constructor not exist when the two types aren't themselves +// copy constructible, but this can not be relied upon when T1 or T2 are themselves containers). +template struct is_copy_constructible> + : all_of, is_copy_constructible> {}; + +// The same problems arise with std::is_copy_assignable, so we use the same workaround. +template struct is_copy_assignable : std::is_copy_assignable {}; +template struct is_copy_assignable, + std::is_same + >::value>> : is_copy_assignable {}; +template struct is_copy_assignable> + : all_of, is_copy_assignable> {}; + +PYBIND11_NAMESPACE_END(detail) + +// polymorphic_type_hook::get(src, tinfo) determines whether the object pointed +// to by `src` actually is an instance of some class derived from `itype`. +// If so, it sets `tinfo` to point to the std::type_info representing that derived +// type, and returns a pointer to the start of the most-derived object of that type +// (in which `src` is a subobject; this will be the same address as `src` in most +// single inheritance cases). If not, or if `src` is nullptr, it simply returns `src` +// and leaves `tinfo` at its default value of nullptr. +// +// The default polymorphic_type_hook just returns src. A specialization for polymorphic +// types determines the runtime type of the passed object and adjusts the this-pointer +// appropriately via dynamic_cast. This is what enables a C++ Animal* to appear +// to Python as a Dog (if Dog inherits from Animal, Animal is polymorphic, Dog is +// registered with pybind11, and this Animal is in fact a Dog). +// +// You may specialize polymorphic_type_hook yourself for types that want to appear +// polymorphic to Python but do not use C++ RTTI. (This is a not uncommon pattern +// in performance-sensitive applications, used most notably in LLVM.) +// +// polymorphic_type_hook_base allows users to specialize polymorphic_type_hook with +// std::enable_if. User provided specializations will always have higher priority than +// the default implementation and specialization provided in polymorphic_type_hook_base. +template +struct polymorphic_type_hook_base +{ + static const void *get(const itype *src, const std::type_info*&) { return src; } +}; +template +struct polymorphic_type_hook_base::value>> +{ + static const void *get(const itype *src, const std::type_info*& type) { + type = src ? &typeid(*src) : nullptr; + return dynamic_cast(src); + } +}; +template +struct polymorphic_type_hook : public polymorphic_type_hook_base {}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +/// Generic type caster for objects stored on the heap +template class type_caster_base : public type_caster_generic { + using itype = intrinsic_t; + +public: + static constexpr auto name = _(); + + type_caster_base() : type_caster_base(typeid(type)) { } + explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) { } + + static handle cast(const itype &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) + policy = return_value_policy::copy; + return cast(&src, policy, parent); + } + + static handle cast(itype &&src, return_value_policy, handle parent) { + return cast(&src, return_value_policy::move, parent); + } + + // Returns a (pointer, type_info) pair taking care of necessary type lookup for a + // polymorphic type (using RTTI by default, but can be overridden by specializing + // polymorphic_type_hook). If the instance isn't derived, returns the base version. + static std::pair src_and_type(const itype *src) { + auto &cast_type = typeid(itype); + const std::type_info *instance_type = nullptr; + const void *vsrc = polymorphic_type_hook::get(src, instance_type); + if (instance_type && !same_type(cast_type, *instance_type)) { + // This is a base pointer to a derived type. If the derived type is registered + // with pybind11, we want to make the full derived object available. + // In the typical case where itype is polymorphic, we get the correct + // derived pointer (which may be != base pointer) by a dynamic_cast to + // most derived type. If itype is not polymorphic, we won't get here + // except via a user-provided specialization of polymorphic_type_hook, + // and the user has promised that no this-pointer adjustment is + // required in that case, so it's OK to use static_cast. + if (const auto *tpi = get_type_info(*instance_type)) + return {vsrc, tpi}; + } + // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, so + // don't do a cast + return type_caster_generic::src_and_type(src, cast_type, instance_type); + } + + static handle cast(const itype *src, return_value_policy policy, handle parent) { + auto st = src_and_type(src); + return type_caster_generic::cast( + st.first, policy, parent, st.second, + make_copy_constructor(src), make_move_constructor(src)); + } + + static handle cast_holder(const itype *src, holder_erased holder) { + auto st = src_and_type(src); + if (!holder) { + throw std::runtime_error("Internal error: Should not have null holder"); + } + return type_caster_generic::cast( + st.first, return_value_policy::take_ownership, {}, st.second, + nullptr, nullptr, holder); + } + + template using cast_op_type = detail::cast_op_type; + + operator itype*() { return (type *) value; } + operator itype&() { if (!value) throw reference_cast_error(); return *((itype *) value); } + +protected: + using Constructor = void *(*)(const void *); + + /* Only enabled when the types are {copy,move}-constructible *and* when the type + does not have a private operator new implementation. */ + template ::value>> + static auto make_copy_constructor(const T *x) -> decltype(new T(*x), Constructor{}) { + return [](const void *arg) -> void * { + return new T(*reinterpret_cast(arg)); + }; + } + + template ::value>> + static auto make_move_constructor(const T *x) -> decltype(new T(std::move(*const_cast(x))), Constructor{}) { + return [](const void *arg) -> void * { + return new T(std::move(*const_cast(reinterpret_cast(arg)))); + }; + } + + static Constructor make_copy_constructor(...) { return nullptr; } + static Constructor make_move_constructor(...) { return nullptr; } +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 4c8f8b4667..3983953071 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -565,7 +565,7 @@ struct type_caster< if (!need_copy) { // We don't need a converting copy, but we also need to check whether the strides are // compatible with the Ref's stride requirements - Array aref = reinterpret_borrow(src); + auto aref = reinterpret_borrow(src); if (aref && (!need_writeable || aref.writeable())) { fits = props::conformable(aref); @@ -710,9 +710,9 @@ struct type_caster::value>> { template struct type_caster::value>> { - typedef typename Type::Scalar Scalar; - typedef remove_reference_t().outerIndexPtr())> StorageIndex; - typedef typename Type::Index Index; + using Scalar = typename Type::Scalar; + using StorageIndex = remove_reference_t().outerIndexPtr())>; + using Index = typename Type::Index; static constexpr bool rowMajor = Type::IsRowMajor; bool load(handle src, bool) { @@ -720,11 +720,11 @@ struct type_caster::value>> { return false; auto obj = reinterpret_borrow(src); - object sparse_module = module::import("scipy.sparse"); + object sparse_module = module_::import("scipy.sparse"); object matrix_type = sparse_module.attr( rowMajor ? "csr_matrix" : "csc_matrix"); - if (!obj.get_type().is(matrix_type)) { + if (!type::handle_of(obj).is(matrix_type)) { try { obj = matrix_type(obj); } catch (const error_already_set &) { @@ -751,7 +751,7 @@ struct type_caster::value>> { static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { const_cast(src).makeCompressed(); - object matrix_type = module::import("scipy.sparse").attr( + object matrix_type = module_::import("scipy.sparse").attr( rowMajor ? "csr_matrix" : "csc_matrix"); array data(src.nonZeros(), src.valuePtr()); diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index eae86c714c..204aaf989f 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -45,26 +45,24 @@ }); } \endrst */ -#define PYBIND11_EMBEDDED_MODULE(name, variable) \ - static void PYBIND11_CONCAT(pybind11_init_, name)(pybind11::module &); \ - static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \ - auto m = pybind11::module(PYBIND11_TOSTRING(name)); \ - try { \ - PYBIND11_CONCAT(pybind11_init_, name)(m); \ - return m.ptr(); \ - } catch (pybind11::error_already_set &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } \ - } \ - PYBIND11_EMBEDDED_MODULE_IMPL(name) \ - pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name) \ - (PYBIND11_TOSTRING(name), \ - PYBIND11_CONCAT(pybind11_init_impl_, name)); \ - void PYBIND11_CONCAT(pybind11_init_, name)(pybind11::module &variable) +#define PYBIND11_EMBEDDED_MODULE(name, variable) \ + static ::pybind11::module_::module_def \ + PYBIND11_CONCAT(pybind11_module_def_, name); \ + static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ + static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \ + auto m = ::pybind11::module_::create_extension_module( \ + PYBIND11_TOSTRING(name), nullptr, \ + &PYBIND11_CONCAT(pybind11_module_def_, name)); \ + try { \ + PYBIND11_CONCAT(pybind11_init_, name)(m); \ + return m.ptr(); \ + } PYBIND11_CATCH_INIT_EXCEPTIONS \ + } \ + PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name) \ + (PYBIND11_TOSTRING(name), \ + PYBIND11_CONCAT(pybind11_init_impl_, name)); \ + void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &variable) PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -109,7 +107,7 @@ inline void initialize_interpreter(bool init_signal_handlers = true) { Py_InitializeEx(init_signal_handlers ? 1 : 0); // Make .py files in the working directory available by default - module::import("sys").attr("path").cast().append("."); + module_::import("sys").attr("path").cast().append("."); } /** \rst diff --git a/include/pybind11/eval.h b/include/pybind11/eval.h index ba82cf42ae..fa6b8af479 100644 --- a/include/pybind11/eval.h +++ b/include/pybind11/eval.h @@ -14,6 +14,22 @@ #include "pybind11.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +inline void ensure_builtins_in_globals(object &global) { + #if PY_VERSION_HEX < 0x03080000 + // Running exec and eval on Python 2 and 3 adds `builtins` module under + // `__builtins__` key to globals if not yet present. + // Python 3.8 made PyRun_String behave similarly. Let's also do that for + // older versions, for consistency. + if (!global.contains("__builtins__")) + global["__builtins__"] = module_::import(PYBIND11_BUILTINS_MODULE); + #else + (void) global; + #endif +} + +PYBIND11_NAMESPACE_END(detail) enum eval_mode { /// Evaluate a string containing an isolated expression @@ -31,6 +47,8 @@ object eval(str expr, object global = globals(), object local = object()) { if (!local) local = global; + detail::ensure_builtins_in_globals(global); + /* PyRun_String does not accept a PyObject / encoding specifier, this seems to be the only alternative */ std::string buffer = "# -*- coding: utf-8 -*-\n" + (std::string) expr; @@ -52,7 +70,7 @@ object eval(str expr, object global = globals(), object local = object()) { template object eval(const char (&s)[N], object global = globals(), object local = object()) { /* Support raw string literals by removing common leading whitespace */ - auto expr = (s[0] == '\n') ? str(module::import("textwrap").attr("dedent")(s)) + auto expr = (s[0] == '\n') ? str(module_::import("textwrap").attr("dedent")(s)) : str(s); return eval(expr, global, local); } @@ -66,7 +84,7 @@ void exec(const char (&s)[N], object global = globals(), object local = object() eval(s, global, local); } -#if defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x3000000 +#if defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x03000000 template object eval_file(str, object, object) { pybind11_fail("eval_file not supported in PyPy3. Use eval"); @@ -85,6 +103,8 @@ object eval_file(str fname, object global = globals(), object local = object()) if (!local) local = global; + detail::ensure_builtins_in_globals(global); + int start; switch (mode) { case eval_expr: start = Py_eval_input; break; diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 57b6cd210f..92c17dc225 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -58,7 +58,10 @@ struct type_caster> { struct func_handle { function f; func_handle(function&& f_) : f(std::move(f_)) {} - func_handle(const func_handle&) = default; + func_handle(const func_handle& f_) { + gil_scoped_acquire acq; + f = f_.f; + } ~func_handle() { gil_scoped_acquire acq; function kill_f(std::move(f)); diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h new file mode 100644 index 0000000000..32f1a8963b --- /dev/null +++ b/include/pybind11/gil.h @@ -0,0 +1,193 @@ +/* + pybind11/gil.h: RAII helpers for managing the GIL + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "detail/internals.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + + +PYBIND11_NAMESPACE_BEGIN(detail) + +// forward declarations +PyThreadState *get_thread_state_unchecked(); + +PYBIND11_NAMESPACE_END(detail) + + +#if defined(WITH_THREAD) && !defined(PYPY_VERSION) + +/* The functions below essentially reproduce the PyGILState_* API using a RAII + * pattern, but there are a few important differences: + * + * 1. When acquiring the GIL from an non-main thread during the finalization + * phase, the GILState API blindly terminates the calling thread, which + * is often not what is wanted. This API does not do this. + * + * 2. The gil_scoped_release function can optionally cut the relationship + * of a PyThreadState and its associated thread, which allows moving it to + * another thread (this is a fairly rare/advanced use case). + * + * 3. The reference count of an acquired thread state can be controlled. This + * can be handy to prevent cases where callbacks issued from an external + * thread would otherwise constantly construct and destroy thread state data + * structures. + * + * See the Python bindings of NanoGUI (http://github.com/wjakob/nanogui) for an + * example which uses features 2 and 3 to migrate the Python thread of + * execution to another thread (to run the event loop on the original thread, + * in this case). + */ + +class gil_scoped_acquire { +public: + PYBIND11_NOINLINE gil_scoped_acquire() { + auto const &internals = detail::get_internals(); + tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate); + + if (!tstate) { + /* Check if the GIL was acquired using the PyGILState_* API instead (e.g. if + calling from a Python thread). Since we use a different key, this ensures + we don't create a new thread state and deadlock in PyEval_AcquireThread + below. Note we don't save this state with internals.tstate, since we don't + create it we would fail to clear it (its reference count should be > 0). */ + tstate = PyGILState_GetThisThreadState(); + } + + if (!tstate) { + tstate = PyThreadState_New(internals.istate); + #if !defined(NDEBUG) + if (!tstate) + pybind11_fail("scoped_acquire: could not create thread state!"); + #endif + tstate->gilstate_counter = 0; + PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); + } else { + release = detail::get_thread_state_unchecked() != tstate; + } + + if (release) { + PyEval_AcquireThread(tstate); + } + + inc_ref(); + } + + void inc_ref() { + ++tstate->gilstate_counter; + } + + PYBIND11_NOINLINE void dec_ref() { + --tstate->gilstate_counter; + #if !defined(NDEBUG) + if (detail::get_thread_state_unchecked() != tstate) + pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); + if (tstate->gilstate_counter < 0) + pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); + #endif + if (tstate->gilstate_counter == 0) { + #if !defined(NDEBUG) + if (!release) + pybind11_fail("scoped_acquire::dec_ref(): internal error!"); + #endif + PyThreadState_Clear(tstate); + if (active) + PyThreadState_DeleteCurrent(); + PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); + release = false; + } + } + + /// This method will disable the PyThreadState_DeleteCurrent call and the + /// GIL won't be acquired. This method should be used if the interpreter + /// could be shutting down when this is called, as thread deletion is not + /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and + /// protect subsequent code. + PYBIND11_NOINLINE void disarm() { + active = false; + } + + PYBIND11_NOINLINE ~gil_scoped_acquire() { + dec_ref(); + if (release) + PyEval_SaveThread(); + } +private: + PyThreadState *tstate = nullptr; + bool release = true; + bool active = true; +}; + +class gil_scoped_release { +public: + explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + // `get_internals()` must be called here unconditionally in order to initialize + // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an + // initialization race could occur as multiple threads try `gil_scoped_acquire`. + const auto &internals = detail::get_internals(); + tstate = PyEval_SaveThread(); + if (disassoc) { + auto key = internals.tstate; + PYBIND11_TLS_DELETE_VALUE(key); + } + } + + /// This method will disable the PyThreadState_DeleteCurrent call and the + /// GIL won't be acquired. This method should be used if the interpreter + /// could be shutting down when this is called, as thread deletion is not + /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and + /// protect subsequent code. + PYBIND11_NOINLINE void disarm() { + active = false; + } + + ~gil_scoped_release() { + if (!tstate) + return; + // `PyEval_RestoreThread()` should not be called if runtime is finalizing + if (active) + PyEval_RestoreThread(tstate); + if (disassoc) { + auto key = detail::get_internals().tstate; + PYBIND11_TLS_REPLACE_VALUE(key, tstate); + } + } +private: + PyThreadState *tstate; + bool disassoc; + bool active = true; +}; +#elif defined(PYPY_VERSION) +class gil_scoped_acquire { + PyGILState_STATE state; +public: + gil_scoped_acquire() { state = PyGILState_Ensure(); } + ~gil_scoped_acquire() { PyGILState_Release(state); } + void disarm() {} +}; + +class gil_scoped_release { + PyThreadState *state; +public: + gil_scoped_release() { state = PyEval_SaveThread(); } + ~gil_scoped_release() { PyEval_RestoreThread(state); } + void disarm() {} +}; +#else +class gil_scoped_acquire { + void disarm() {} +}; +class gil_scoped_release { + void disarm() {} +}; +#endif + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/iostream.h b/include/pybind11/iostream.h index eaf92dfa49..9dee755431 100644 --- a/include/pybind11/iostream.h +++ b/include/pybind11/iostream.h @@ -30,7 +30,7 @@ class pythonbuf : public std::streambuf { object pywrite; object pyflush; - int overflow(int c) { + int overflow(int c) override { if (!traits_type::eq_int_type(c, traits_type::eof())) { *pptr() = traits_type::to_char_type(c); pbump(1); @@ -38,22 +38,33 @@ class pythonbuf : public std::streambuf { return sync() == 0 ? traits_type::not_eof(c) : traits_type::eof(); } - int sync() { + // This function must be non-virtual to be called in a destructor. If the + // rare MSVC test failure shows up with this version, then this should be + // simplified to a fully qualified call. + int _sync() { if (pbase() != pptr()) { - // This subtraction cannot be negative, so dropping the sign - str line(pbase(), static_cast(pptr() - pbase())); { gil_scoped_acquire tmp; + + // This subtraction cannot be negative, so dropping the sign. + str line(pbase(), static_cast(pptr() - pbase())); + pywrite(line); pyflush(); + + // Placed inside gil_scoped_aquire as a mutex to avoid a race + setp(pbase(), epptr()); } - setp(pbase(), epptr()); } return 0; } + int sync() override { + return _sync(); + } + public: pythonbuf(object pyostream, size_t buffer_size = 1024) @@ -67,8 +78,8 @@ class pythonbuf : public std::streambuf { pythonbuf(pythonbuf&&) = default; /// Sync before destroy - ~pythonbuf() { - sync(); + ~pythonbuf() override { + _sync(); } }; @@ -96,7 +107,7 @@ PYBIND11_NAMESPACE_END(detail) { py::scoped_ostream_redirect output{std::cerr, py::module::import("sys").attr("stderr")}; - std::cerr << "Hello, World!"; + std::cout << "Hello, World!"; } \endrst */ class scoped_ostream_redirect { @@ -108,7 +119,7 @@ class scoped_ostream_redirect { public: scoped_ostream_redirect( std::ostream &costream = std::cout, - object pyostream = module::import("sys").attr("stdout")) + object pyostream = module_::import("sys").attr("stdout")) : costream(costream), buffer(pyostream) { old = costream.rdbuf(&buffer); } @@ -139,7 +150,7 @@ class scoped_estream_redirect : public scoped_ostream_redirect { public: scoped_estream_redirect( std::ostream &costream = std::cerr, - object pyostream = module::import("sys").attr("stderr")) + object pyostream = module_::import("sys").attr("stderr")) : scoped_ostream_redirect(costream,pyostream) {} }; @@ -199,7 +210,7 @@ PYBIND11_NAMESPACE_END(detail) m.noisy_function_with_error_printing() \endrst */ -inline class_ add_ostream_redirect(module m, std::string name = "ostream_redirect") { +inline class_ add_ostream_redirect(module_ m, std::string name = "ostream_redirect") { return class_(m, name.c_str(), module_local()) .def(init(), arg("stdout")=true, arg("stderr")=true) .def("__enter__", &detail::OstreamRedirect::enter) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1ddd59dc44..6e2bbad7af 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -33,7 +34,9 @@ whole npy_intp / ssize_t / Py_intptr_t business down to just ssize_t for all size and dimension types (e.g. shape, strides, indexing), instead of inflicting this upon the library user. */ -static_assert(sizeof(ssize_t) == sizeof(Py_intptr_t), "ssize_t != Py_intptr_t"); +static_assert(sizeof(::pybind11::ssize_t) == sizeof(Py_intptr_t), "ssize_t != Py_intptr_t"); +static_assert(std::is_signed::value, "Py_intptr_t must be signed"); +// We now can reinterpret_cast between py::ssize_t and Py_intptr_t (MSVC + PyPy cares) PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -222,7 +225,7 @@ struct npy_api { }; static npy_api lookup() { - module m = module::import("numpy.core.multiarray"); + module_ m = module_::import("numpy.core.multiarray"); auto c = m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), NULL); @@ -281,7 +284,7 @@ template struct is_complex : std::false_type { }; template struct is_complex> : std::true_type { }; template struct array_info_scalar { - typedef T type; + using type = T; static constexpr bool is_array = false; static constexpr bool is_empty = false; static constexpr auto extents = _(""); @@ -328,6 +331,12 @@ template using is_pod_struct = all_of< satisfies_none_of >; +// Replacement for std::is_pod (deprecated in C++20) +template using is_pod = all_of< + std::is_standard_layout, + std::is_trivial +>; + template ssize_t byte_offset_unsafe(const Strides &) { return 0; } template ssize_t byte_offset_unsafe(const Strides &strides, ssize_t i, Ix... index) { @@ -419,6 +428,10 @@ class unchecked_mutable_reference : public unchecked_reference { using ConstBase::ConstBase; using ConstBase::Dynamic; public: + // Bring in const-qualified versions from base class + using ConstBase::operator(); + using ConstBase::operator[]; + /// Mutable, unchecked access to data at the given indices. template T& operator()(Ix... index) { static_assert(ssize_t{sizeof...(Ix)} == Dims || Dynamic, @@ -494,14 +507,24 @@ class dtype : public object { return detail::array_descriptor_proxy(m_ptr)->names != nullptr; } - /// Single-character type code. + /// Single-character code for dtype's kind. + /// For example, floating point types are 'f' and integral types are 'i'. char kind() const { return detail::array_descriptor_proxy(m_ptr)->kind; } + /// Single-character for dtype's type. + /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'd'. + char char_() const { + // Note: The signature, `dtype::char_` follows the naming of NumPy's + // public Python API (i.e., ``dtype.char``), rather than its internal + // C API (``PyArray_Descr::type``). + return detail::array_descriptor_proxy(m_ptr)->type; + } + private: static object _dtype_from_pep3118() { - static PyObject *obj = module::import("numpy.core._internal") + static PyObject *obj = module_::import("numpy.core._internal") .attr("_dtype_from_pep3118").cast().release().ptr(); return reinterpret_borrow(obj); } @@ -560,7 +583,7 @@ class array : public buffer { const void *ptr = nullptr, handle base = handle()) { if (strides->empty()) - *strides = c_strides(*shape, dt.itemsize()); + *strides = detail::c_strides(*shape, dt.itemsize()); auto ndim = shape->size(); if (ndim != strides->size()) @@ -579,7 +602,10 @@ class array : public buffer { auto &api = detail::npy_api::get(); auto tmp = reinterpret_steal(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr.release().ptr(), (int) ndim, shape->data(), strides->data(), + api.PyArray_Type_, descr.release().ptr(), (int) ndim, + // Use reinterpret_cast for PyPy on Windows (remove if fixed, checked on 7.3.1) + reinterpret_cast(shape->data()), + reinterpret_cast(strides->data()), const_cast(ptr), flags, nullptr)); if (!tmp) throw error_already_set(); @@ -611,8 +637,8 @@ class array : public buffer { template explicit array(ssize_t count, const T *ptr, handle base = handle()) : array({count}, {}, ptr, base) { } - explicit array(const buffer_info &info) - : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } + explicit array(const buffer_info &info, handle base = handle()) + : array(pybind11::dtype(info), info.shape, info.strides, info.ptr, base) { } /// Array descriptor (dtype) pybind11::dtype dtype() const { @@ -751,10 +777,12 @@ class array : public buffer { /// then resize will succeed only if it makes a reshape, i.e. original size doesn't change void resize(ShapeContainer new_shape, bool refcheck = true) { detail::npy_api::PyArray_Dims d = { - new_shape->data(), int(new_shape->size()) + // Use reinterpret_cast for PyPy on Windows (remove if fixed, checked on 7.3.1) + reinterpret_cast(new_shape->data()), + int(new_shape->size()) }; // try to resize, set ordering param to -1 cause it's not used anyway - object new_array = reinterpret_steal( + auto new_array = reinterpret_steal( detail::npy_api::get().PyArray_Resize_(m_ptr, &d, int(refcheck), -1) ); if (!new_array) throw error_already_set(); @@ -788,25 +816,6 @@ class array : public buffer { throw std::domain_error("array is not writeable"); } - // Default, C-style strides - static std::vector c_strides(const std::vector &shape, ssize_t itemsize) { - auto ndim = shape.size(); - std::vector strides(ndim, itemsize); - if (ndim > 0) - for (size_t i = ndim - 1; i > 0; --i) - strides[i - 1] = strides[i] * shape[i]; - return strides; - } - - // F-style strides; default when constructing an array_t with `ExtraFlags & f_style` - static std::vector f_strides(const std::vector &shape, ssize_t itemsize) { - auto ndim = shape.size(); - std::vector strides(ndim, itemsize); - for (size_t i = 1; i < ndim; ++i) - strides[i] = strides[i - 1] * shape[i - 1]; - return strides; - } - template void check_dimensions(Ix... index) const { check_dimensions_impl(ssize_t(0), shape(), ssize_t(index)...); } @@ -858,14 +867,16 @@ template class array_t : public if (!m_ptr) throw error_already_set(); } - explicit array_t(const buffer_info& info) : array(info) { } + explicit array_t(const buffer_info& info, handle base = handle()) : array(info, base) { } array_t(ShapeContainer shape, StridesContainer strides, const T *ptr = nullptr, handle base = handle()) : array(std::move(shape), std::move(strides), ptr, base) { } explicit array_t(ShapeContainer shape, const T *ptr = nullptr, handle base = handle()) : array_t(private_ctor{}, std::move(shape), - ExtraFlags & f_style ? f_strides(*shape, itemsize()) : c_strides(*shape, itemsize()), + ExtraFlags & f_style + ? detail::f_strides(*shape, itemsize()) + : detail::c_strides(*shape, itemsize()), ptr, base) { } explicit array_t(ssize_t count, const T *ptr = nullptr, handle base = handle()) @@ -934,7 +945,8 @@ template class array_t : public static bool check_(handle h) { const auto &api = detail::npy_api::get(); return api.PyArray_Check_(h.ptr()) - && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, dtype::of().ptr()); + && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, dtype::of().ptr()) + && detail::check_flags(h.ptr(), ExtraFlags & (array::c_style | array::f_style)); } protected: @@ -1287,19 +1299,6 @@ struct npy_format_descriptor_object { #endif // __CLION_IDE__ -template -using array_iterator = typename std::add_pointer::type; - -template -array_iterator array_begin(const buffer_info& buffer) { - return array_iterator(reinterpret_cast(buffer.ptr)); -} - -template -array_iterator array_end(const buffer_info& buffer) { - return array_iterator(reinterpret_cast(buffer.ptr) + buffer.size); -} - class common_iterator { public: using container_type = std::vector; @@ -1313,7 +1312,7 @@ class common_iterator { m_strides.back() = static_cast(strides.back()); for (size_type i = m_strides.size() - 1; i != 0; --i) { size_type j = i - 1; - value_type s = static_cast(shape[i]); + auto s = static_cast(shape[i]); m_strides[j] = strides[j] + m_strides[i] - strides[i] * s; } } @@ -1491,7 +1490,7 @@ struct vectorize_arg { using call_type = remove_reference_t; // Is this a vectorized argument? static constexpr bool vectorize = - satisfies_any_of::value && + satisfies_any_of::value && satisfies_none_of::value && (!std::is_reference::value || (std::is_lvalue_reference::value && std::is_const::value)); @@ -1499,9 +1498,66 @@ struct vectorize_arg { using type = conditional_t, array::forcecast>, T>; }; + +// py::vectorize when a return type is present +template +struct vectorize_returned_array { + using Type = array_t; + + static Type create(broadcast_trivial trivial, const std::vector &shape) { + if (trivial == broadcast_trivial::f_trivial) + return array_t(shape); + else + return array_t(shape); + } + + static Return *mutable_data(Type &array) { + return array.mutable_data(); + } + + static Return call(Func &f, Args &... args) { + return f(args...); + } + + static void call(Return *out, size_t i, Func &f, Args &... args) { + out[i] = f(args...); + } +}; + +// py::vectorize when a return type is not present +template +struct vectorize_returned_array { + using Type = none; + + static Type create(broadcast_trivial, const std::vector &) { + return none(); + } + + static void *mutable_data(Type &) { + return nullptr; + } + + static detail::void_type call(Func &f, Args &... args) { + f(args...); + return {}; + } + + static void call(void *, size_t, Func &f, Args &... args) { + f(args...); + } +}; + + template struct vectorize_helper { + +// NVCC for some reason breaks if NVectorized is private +#ifdef __CUDACC__ +public: +#else private: +#endif + static constexpr size_t N = sizeof...(Args); static constexpr size_t NVectorized = constexpr_sum(vectorize_arg::vectorize...); static_assert(NVectorized >= 1, @@ -1526,6 +1582,8 @@ struct vectorize_helper { using arg_call_types = std::tuple::call_type...>; template using param_n_t = typename std::tuple_element::type; + using returned_array = vectorize_returned_array; + // Runs a vectorized function given arguments tuple and three index sequences: // - Index is the full set of 0 ... (N-1) argument indices; // - VIndex is the subset of argument indices with vectorized parameters, letting us access @@ -1549,7 +1607,7 @@ struct vectorize_helper { ssize_t nd = 0; std::vector shape(0); auto trivial = broadcast(buffers, nd, shape); - size_t ndim = (size_t) nd; + auto ndim = (size_t) nd; size_t size = std::accumulate(shape.begin(), shape.end(), (size_t) 1, std::multiplies()); @@ -1557,20 +1615,19 @@ struct vectorize_helper { // not wrapped in an array). if (size == 1 && ndim == 0) { PYBIND11_EXPAND_SIDE_EFFECTS(params[VIndex] = buffers[BIndex].ptr); - return cast(f(*reinterpret_cast *>(params[Index])...)); + return cast(returned_array::call(f, *reinterpret_cast *>(params[Index])...)); } - array_t result; - if (trivial == broadcast_trivial::f_trivial) result = array_t(shape); - else result = array_t(shape); + auto result = returned_array::create(trivial, shape); if (size == 0) return std::move(result); /* Call the function */ + auto mutable_data = returned_array::mutable_data(result); if (trivial == broadcast_trivial::non_trivial) - apply_broadcast(buffers, params, result, i_seq, vi_seq, bi_seq); + apply_broadcast(buffers, params, mutable_data, size, shape, i_seq, vi_seq, bi_seq); else - apply_trivial(buffers, params, result.mutable_data(), size, i_seq, vi_seq, bi_seq); + apply_trivial(buffers, params, mutable_data, size, i_seq, vi_seq, bi_seq); return std::move(result); } @@ -1593,7 +1650,7 @@ struct vectorize_helper { }}; for (size_t i = 0; i < size; ++i) { - out[i] = f(*reinterpret_cast *>(params[Index])...); + returned_array::call(out, i, f, *reinterpret_cast *>(params[Index])...); for (auto &x : vecparams) x.first += x.second; } } @@ -1601,19 +1658,18 @@ struct vectorize_helper { template void apply_broadcast(std::array &buffers, std::array ¶ms, - array_t &output_array, + Return *out, + size_t size, + const std::vector &output_shape, index_sequence, index_sequence, index_sequence) { - buffer_info output = output_array.request(); - multi_array_iterator input_iter(buffers, output.shape); + multi_array_iterator input_iter(buffers, output_shape); - for (array_iterator iter = array_begin(output), end = array_end(output); - iter != end; - ++iter, ++input_iter) { + for (size_t i = 0; i < size; ++i, ++input_iter) { PYBIND11_EXPAND_SIDE_EFFECTS(( params[VIndex] = input_iter.template data() )); - *iter = f(*reinterpret_cast *>(std::get(params))...); + returned_array::call(out, i, f, *reinterpret_cast *>(std::get(params))...); } } }; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index b4382fc006..bb66359d73 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -29,6 +29,7 @@ # pragma warning(disable: 4996) // warning C4996: The POSIX name for this item is deprecated. Instead, use the ISO C and C++ conformant name # pragma warning(disable: 4702) // warning C4702: unreachable code # pragma warning(disable: 4522) // warning C4522: multiple assignment operators specified +# pragma warning(disable: 4505) // warning C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) #elif defined(__GNUG__) && !defined(__clang__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-but-set-parameter" @@ -42,10 +43,16 @@ #endif #include "attr.h" +#include "gil.h" #include "options.h" #include "detail/class.h" #include "detail/init.h" +#include +#include +#include +#include + #if defined(__GNUG__) && !defined(__clang__) # include #endif @@ -58,7 +65,7 @@ void unused(Args&&...) {} /// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object class cpp_function : public function { public: - cpp_function() { } + cpp_function() = default; cpp_function(std::nullptr_t) { } /// Construct a cpp_function from a vanilla function pointer @@ -111,9 +118,16 @@ class cpp_function : public function { object name() const { return attr("__name__"); } protected: + struct InitializingFunctionRecordDeleter { + // `destruct(function_record, false)`: `initialize_generic` copies strings and + // takes care of cleaning up in case of exceptions. So pass `false` to `free_strings`. + void operator()(detail::function_record * rec) { destruct(rec, false); } + }; + using unique_function_record = std::unique_ptr; + /// Space optimization: don't inline this frequently instantiated fragment - PYBIND11_NOINLINE detail::function_record *make_function_record() { - return new detail::function_record(); + PYBIND11_NOINLINE unique_function_record make_function_record() { + return unique_function_record(new detail::function_record()); } /// Special internal constructor for functors, lambda functions, etc. @@ -123,7 +137,9 @@ class cpp_function : public function { struct capture { remove_reference_t f; }; /* Store the function including any extra state it might have (e.g. a lambda capture object) */ - auto rec = make_function_record(); + // The unique_ptr makes sure nothing is leaked in case of an exception. + auto unique_rec = make_function_record(); + auto rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ if (sizeof(capture) <= sizeof(rec->data)) { @@ -168,7 +184,7 @@ class cpp_function : public function { /* Get a pointer to the capture object */ auto data = (sizeof(capture) <= sizeof(call.func.data) ? &call.func.data : call.func.data[0]); - capture *cap = const_cast(reinterpret_cast(data)); + auto *cap = const_cast(reinterpret_cast(data)); /* Override policy for rvalues -- usually to enforce rvp::move on an rvalue */ return_value_policy policy = return_value_policy_override::policy(call.func.policy); @@ -190,11 +206,13 @@ class cpp_function : public function { process_attributes::init(extra..., rec); { - constexpr bool has_kwonly_args = any_of...>::value, + constexpr bool has_kw_only_args = any_of...>::value, + has_pos_only_args = any_of...>::value, has_args = any_of...>::value, has_arg_annotations = any_of...>::value; - static_assert(has_arg_annotations || !has_kwonly_args, "py::kwonly requires the use of argument annotations"); - static_assert(!(has_args && has_kwonly_args), "py::kwonly cannot be combined with a py::args argument"); + static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations"); + static_assert(has_arg_annotations || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings and aligning the annotations to the argument)"); + static_assert(!(has_args && has_kw_only_args), "py::kw_only cannot be combined with a py::args argument"); } /* Generate a readable signature describing the function's arguments and return value types */ @@ -202,7 +220,8 @@ class cpp_function : public function { PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); /* Register the function with Python from generic (non-templated) code */ - initialize_generic(rec, signature.text, types.data(), sizeof...(Args)); + // Pass on the ownership over the `unique_rec` to `initialize_generic`. `rec` stays valid. + initialize_generic(std::move(unique_rec), signature.text, types.data(), sizeof...(Args)); if (cast_in::has_args) rec->has_args = true; if (cast_in::has_kwargs) rec->has_kwargs = true; @@ -218,27 +237,58 @@ class cpp_function : public function { } } + // Utility class that keeps track of all duplicated strings, and cleans them up in its destructor, + // unless they are released. Basically a RAII-solution to deal with exceptions along the way. + class strdup_guard { + public: + ~strdup_guard() { + for (auto s : strings) + std::free(s); + } + char *operator()(const char *s) { + auto t = strdup(s); + strings.push_back(t); + return t; + } + void release() { + strings.clear(); + } + private: + std::vector strings; + }; + /// Register a function call with Python (generic non-templated code goes here) - void initialize_generic(detail::function_record *rec, const char *text, + void initialize_generic(unique_function_record &&unique_rec, const char *text, const std::type_info *const *types, size_t args) { + // Do NOT receive `unique_rec` by value. If this function fails to move out the unique_ptr, + // we do not want this to destuct the pointer. `initialize` (the caller) still relies on the + // pointee being alive after this call. Only move out if a `capsule` is going to keep it alive. + auto rec = unique_rec.get(); + + // Keep track of strdup'ed strings, and clean them up as long as the function's capsule + // has not taken ownership yet (when `unique_rec.release()` is called). + // Note: This cannot easily be fixed by a `unique_ptr` with custom deleter, because the strings + // are only referenced before strdup'ing. So only *after* the following block could `destruct` + // safely be called, but even then, `repr` could still throw in the middle of copying all strings. + strdup_guard guarded_strdup; /* Create copies of all referenced C-style strings */ - rec->name = strdup(rec->name ? rec->name : ""); - if (rec->doc) rec->doc = strdup(rec->doc); + rec->name = guarded_strdup(rec->name ? rec->name : ""); + if (rec->doc) rec->doc = guarded_strdup(rec->doc); for (auto &a: rec->args) { if (a.name) - a.name = strdup(a.name); + a.name = guarded_strdup(a.name); if (a.descr) - a.descr = strdup(a.descr); + a.descr = guarded_strdup(a.descr); else if (a.value) - a.descr = strdup(repr(a.value).cast().c_str()); + a.descr = guarded_strdup(repr(a.value).cast().c_str()); } rec->is_constructor = !strcmp(rec->name, "__init__") || !strcmp(rec->name, "__setstate__"); #if !defined(NDEBUG) && !defined(PYBIND11_DISABLE_NEW_STYLE_INIT_WARNING) if (rec->is_constructor && !rec->is_new_style_constructor) { - const auto class_name = std::string(((PyTypeObject *) rec->scope.ptr())->tp_name); + const auto class_name = detail::get_fully_qualified_tp_name((PyTypeObject *) rec->scope.ptr()); const auto func_name = std::string(rec->name); PyErr_WarnEx( PyExc_FutureWarning, @@ -260,7 +310,10 @@ class cpp_function : public function { // Write arg name for everything except *args and **kwargs. if (*(pc + 1) == '*') continue; - + // Separator for keyword-only arguments, placed before the kw + // arguments start + if (rec->nargs_kw_only > 0 && arg_index + rec->nargs_kw_only == args) + signature += "*, "; if (arg_index < rec->args.size() && rec->args[arg_index].name) { signature += rec->args[arg_index].name; } else if (arg_index == 0 && rec->is_method) { @@ -275,6 +328,10 @@ class cpp_function : public function { signature += " = "; signature += rec->args[arg_index].descr; } + // Separator for positional-only arguments (placed after the + // argument, rather than before like * + if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only) + signature += ", /"; arg_index++; } else if (c == '%') { const std::type_info *t = types[type_index++]; @@ -300,19 +357,20 @@ class cpp_function : public function { signature += c; } } + if (arg_index != args || types[type_index] != nullptr) pybind11_fail("Internal error while parsing type signature (2)"); #if PY_MAJOR_VERSION < 3 if (strcmp(rec->name, "__next__") == 0) { std::free(rec->name); - rec->name = strdup("next"); + rec->name = guarded_strdup("next"); } else if (strcmp(rec->name, "__bool__") == 0) { std::free(rec->name); - rec->name = strdup("__nonzero__"); + rec->name = guarded_strdup("__nonzero__"); } #endif - rec->signature = strdup(signature.c_str()); + rec->signature = guarded_strdup(signature.c_str()); rec->args.shrink_to_fit(); rec->nargs = (std::uint16_t) args; @@ -343,9 +401,10 @@ class cpp_function : public function { rec->def->ml_meth = reinterpret_cast(reinterpret_cast(*dispatcher)); rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; - capsule rec_capsule(rec, [](void *ptr) { + capsule rec_capsule(unique_rec.release(), [](void *ptr) { destruct((detail::function_record *) ptr); }); + guarded_strdup.release(); object scope_module; if (rec->scope) { @@ -360,10 +419,9 @@ class cpp_function : public function { if (!m_ptr) pybind11_fail("cpp_function::cpp_function(): Could not allocate function object"); } else { - /* Append at the end of the overload chain */ + /* Append at the beginning or end of the overload chain */ m_ptr = rec->sibling.ptr(); inc_ref(); - chain_start = chain; if (chain->is_method != rec->is_method) pybind11_fail("overloading a method with both static and instance methods is not supported; " #if defined(NDEBUG) @@ -373,9 +431,24 @@ class cpp_function : public function { std::string(pybind11::str(rec->scope.attr("__name__"))) + "." + std::string(rec->name) + signature #endif ); - while (chain->next) - chain = chain->next; - chain->next = rec; + + if (rec->prepend) { + // Beginning of chain; we need to replace the capsule's current head-of-the-chain + // pointer with this one, then make this one point to the previous head of the + // chain. + chain_start = rec; + rec->next = chain; + auto rec_capsule = reinterpret_borrow(((PyCFunctionObject *) m_ptr)->m_self); + rec_capsule.set_pointer(unique_rec.release()); + guarded_strdup.release(); + } else { + // Or end of chain (normal behavior) + chain_start = chain; + while (chain->next) + chain = chain->next; + chain->next = unique_rec.release(); + guarded_strdup.release(); + } } std::string signatures; @@ -413,10 +486,10 @@ class cpp_function : public function { } /* Install docstring */ - PyCFunctionObject *func = (PyCFunctionObject *) m_ptr; - if (func->m_ml->ml_doc) - std::free(const_cast(func->m_ml->ml_doc)); - func->m_ml->ml_doc = strdup(signatures.c_str()); + auto *func = (PyCFunctionObject *) m_ptr; + std::free(const_cast(func->m_ml->ml_doc)); + // Install docstring if it's non-empty (when at least one option is enabled) + func->m_ml->ml_doc = signatures.empty() ? nullptr : strdup(signatures.c_str()); if (rec->is_method) { m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); @@ -427,22 +500,42 @@ class cpp_function : public function { } /// When a cpp_function is GCed, release any memory allocated by pybind11 - static void destruct(detail::function_record *rec) { + static void destruct(detail::function_record *rec, bool free_strings = true) { + // If on Python 3.9, check the interpreter "MICRO" (patch) version. + // If this is running on 3.9.0, we have to work around a bug. + #if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 + static bool is_zero = Py_GetVersion()[4] == '0'; + #endif + while (rec) { detail::function_record *next = rec->next; if (rec->free_data) rec->free_data(rec); - std::free((char *) rec->name); - std::free((char *) rec->doc); - std::free((char *) rec->signature); - for (auto &arg: rec->args) { - std::free(const_cast(arg.name)); - std::free(const_cast(arg.descr)); - arg.value.dec_ref(); + // During initialization, these strings might not have been copied yet, + // so they cannot be freed. Once the function has been created, they can. + // Check `make_function_record` for more details. + if (free_strings) { + std::free((char *) rec->name); + std::free((char *) rec->doc); + std::free((char *) rec->signature); + for (auto &arg: rec->args) { + std::free(const_cast(arg.name)); + std::free(const_cast(arg.descr)); + } } + for (auto &arg: rec->args) + arg.value.dec_ref(); if (rec->def) { std::free(const_cast(rec->def->ml_doc)); - delete rec->def; + // Python 3.9.0 decref's these in the wrong order; rec->def + // If loaded on 3.9.0, let these leak (use Python 3.9.1 at runtime to fix) + // See https://github.com/python/cpython/pull/22670 + #if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 + if (!is_zero) + delete rec->def; + #else + delete rec->def; + #endif } delete rec; rec = next; @@ -458,22 +551,22 @@ class cpp_function : public function { *it = overloads; /* Need to know how many arguments + keyword arguments there are to pick the right overload */ - const size_t n_args_in = (size_t) PyTuple_GET_SIZE(args_in); + const auto n_args_in = (size_t) PyTuple_GET_SIZE(args_in); handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr, result = PYBIND11_TRY_NEXT_OVERLOAD; auto self_value_and_holder = value_and_holder(); if (overloads->is_constructor) { - const auto tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); - const auto pi = reinterpret_cast(parent.ptr()); - self_value_and_holder = pi->get_value_and_holder(tinfo, false); - - if (!self_value_and_holder.type || !self_value_and_holder.inst) { + if (!PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { PyErr_SetString(PyExc_TypeError, "__init__(self, ...) called with invalid `self` argument"); return nullptr; } + const auto tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); + const auto pi = reinterpret_cast(parent.ptr()); + self_value_and_holder = pi->get_value_and_holder(tinfo, true); + // If this value is already registered it must mean __init__ is invoked multiple times; // we really can't support that in C++, so just ignore the second __init__. if (self_value_and_holder.instance_registered()) @@ -515,7 +608,7 @@ class cpp_function : public function { size_t num_args = func.nargs; // Number of positional arguments that we need if (func.has_args) --num_args; // (but don't count py::args if (func.has_kwargs) --num_args; // or py::kwargs) - size_t pos_args = num_args - func.nargs_kwonly; + size_t pos_args = num_args - func.nargs_kw_only; if (!func.has_args && n_args_in > pos_args) continue; // Too many positional arguments for this overload @@ -536,7 +629,7 @@ class cpp_function : public function { self_value_and_holder.type->dealloc(self_value_and_holder); call.init_self = PyTuple_GET_ITEM(args_in, 0); - call.args.push_back(reinterpret_cast(&self_value_and_holder)); + call.args.emplace_back(reinterpret_cast(&self_value_and_holder)); call.args_convert.push_back(false); ++args_copied; } @@ -564,16 +657,36 @@ class cpp_function : public function { // We'll need to copy this if we steal some kwargs for defaults dict kwargs = reinterpret_borrow(kwargs_in); + // 1.5. Fill in any missing pos_only args from defaults if they exist + if (args_copied < func.nargs_pos_only) { + for (; args_copied < func.nargs_pos_only; ++args_copied) { + const auto &arg_rec = func.args[args_copied]; + handle value; + + if (arg_rec.value) { + value = arg_rec.value; + } + if (value) { + call.args.push_back(value); + call.args_convert.push_back(arg_rec.convert); + } else + break; + } + + if (args_copied < func.nargs_pos_only) + continue; // Not enough defaults to fill the positional arguments + } + // 2. Check kwargs and, failing that, defaults that may help complete the list if (args_copied < num_args) { bool copied_kwargs = false; for (; args_copied < num_args; ++args_copied) { - const auto &arg = func.args[args_copied]; + const auto &arg_rec = func.args[args_copied]; handle value; - if (kwargs_in && arg.name) - value = PyDict_GetItemString(kwargs.ptr(), arg.name); + if (kwargs_in && arg_rec.name) + value = PyDict_GetItemString(kwargs.ptr(), arg_rec.name); if (value) { // Consume a kwargs value @@ -581,14 +694,18 @@ class cpp_function : public function { kwargs = reinterpret_steal(PyDict_Copy(kwargs.ptr())); copied_kwargs = true; } - PyDict_DelItemString(kwargs.ptr(), arg.name); - } else if (arg.value) { - value = arg.value; + PyDict_DelItemString(kwargs.ptr(), arg_rec.name); + } else if (arg_rec.value) { + value = arg_rec.value; + } + + if (!arg_rec.none && value.is_none()) { + break; } if (value) { call.args.push_back(value); - call.args_convert.push_back(arg.convert); + call.args_convert.push_back(arg_rec.convert); } else break; @@ -599,7 +716,7 @@ class cpp_function : public function { } // 3. Check everything was consumed (unless we have a kwargs arg) - if (kwargs && kwargs.size() > 0 && !func.has_kwargs) + if (kwargs && !kwargs.empty() && !func.has_kwargs) continue; // Unconsumed kwargs, but no py::kwargs argument to accept them // 4a. If we have a py::args argument, create a new tuple with leftovers @@ -697,7 +814,7 @@ class cpp_function : public function { } catch (error_already_set &e) { e.restore(); return nullptr; -#if defined(__GNUG__) && !defined(__clang__) +#ifdef __GLIBCXX__ } catch ( abi::__forced_unwind& ) { throw; #endif @@ -779,18 +896,27 @@ class cpp_function : public function { for (size_t ti = overloads->is_constructor ? 1 : 0; ti < args_.size(); ++ti) { if (!some_args) some_args = true; else msg += ", "; - msg += pybind11::repr(args_[ti]); + try { + msg += pybind11::repr(args_[ti]); + } catch (const error_already_set&) { + msg += ""; + } } if (kwargs_in) { auto kwargs = reinterpret_borrow(kwargs_in); - if (kwargs.size() > 0) { + if (!kwargs.empty()) { if (some_args) msg += "; "; msg += "kwargs: "; bool first = true; for (auto kwarg : kwargs) { if (first) first = false; else msg += ", "; - msg += pybind11::str("{}={!r}").format(kwarg.first, kwarg.second); + msg += pybind11::str("{}=").format(kwarg.first); + try { + msg += pybind11::repr(kwarg.second); + } catch (const error_already_set&) { + msg += ""; + } } } } @@ -816,27 +942,18 @@ class cpp_function : public function { }; /// Wrapper for Python extension modules -class module : public object { +class module_ : public object { public: - PYBIND11_OBJECT_DEFAULT(module, object, PyModule_Check) + PYBIND11_OBJECT_DEFAULT(module_, object, PyModule_Check) /// Create a new top-level Python module with the given name and docstring - explicit module(const char *name, const char *doc = nullptr) { - if (!options::show_user_defined_docstrings()) doc = nullptr; + PYBIND11_DEPRECATED("Use PYBIND11_MODULE or module_::create_extension_module instead") + explicit module_(const char *name, const char *doc = nullptr) { #if PY_MAJOR_VERSION >= 3 - PyModuleDef *def = new PyModuleDef(); - std::memset(def, 0, sizeof(PyModuleDef)); - def->m_name = name; - def->m_doc = doc; - def->m_size = -1; - Py_INCREF(def); - m_ptr = PyModule_Create(def); + *this = create_extension_module(name, doc, new PyModuleDef()); #else - m_ptr = Py_InitModule3(name, nullptr, doc); + *this = create_extension_module(name, doc, nullptr); #endif - if (m_ptr == nullptr) - pybind11_fail("Internal error in module::module()"); - inc_ref(); } /** \rst @@ -845,7 +962,7 @@ class module : public object { details on the ``Extra&& ... extra`` argument, see section :ref:`extras`. \endrst */ template - module &def(const char *name_, Func &&f, const Extra& ... extra) { + module_ &def(const char *name_, Func &&f, const Extra& ... extra) { cpp_function func(std::forward(f), name(name_), scope(*this), sibling(getattr(*this, name_, none())), extra...); // NB: allow overwriting here because cpp_function sets up a chain with the intention of @@ -860,14 +977,14 @@ class module : public object { .. code-block:: cpp - py::module m("example", "pybind11 example plugin"); - py::module m2 = m.def_submodule("sub", "A submodule of 'example'"); - py::module m3 = m2.def_submodule("subsub", "A submodule of 'example.sub'"); + py::module_ m("example", "pybind11 example plugin"); + py::module_ m2 = m.def_submodule("sub", "A submodule of 'example'"); + py::module_ m3 = m2.def_submodule("subsub", "A submodule of 'example.sub'"); \endrst */ - module def_submodule(const char *name, const char *doc = nullptr) { + module_ def_submodule(const char *name, const char *doc = nullptr) { std::string full_name = std::string(PyModule_GetName(m_ptr)) + std::string(".") + std::string(name); - auto result = reinterpret_borrow(PyImport_AddModule(full_name.c_str())); + auto result = reinterpret_borrow(PyImport_AddModule(full_name.c_str())); if (doc && options::show_user_defined_docstrings()) result.attr("__doc__") = pybind11::str(doc); attr(name) = result; @@ -875,11 +992,11 @@ class module : public object { } /// Import and return a module or throws `error_already_set`. - static module import(const char *name) { + static module_ import(const char *name) { PyObject *obj = PyImport_ImportModule(name); if (!obj) throw error_already_set(); - return reinterpret_steal(obj); + return reinterpret_steal(obj); } /// Reload the module or throws `error_already_set`. @@ -887,14 +1004,16 @@ class module : public object { PyObject *obj = PyImport_ReloadModule(ptr()); if (!obj) throw error_already_set(); - *this = reinterpret_steal(obj); + *this = reinterpret_steal(obj); } - // Adds an object to the module using the given name. Throws if an object with the given name - // already exists. - // - // overwrite should almost always be false: attempting to overwrite objects that pybind11 has - // established will, in most cases, break things. + /** \rst + Adds an object to the module using the given name. Throws if an object with the given name + already exists. + + ``overwrite`` should almost always be false: attempting to overwrite objects that pybind11 has + established will, in most cases, break things. + \endrst */ PYBIND11_NOINLINE void add_object(const char *name, handle obj, bool overwrite = false) { if (!overwrite && hasattr(*this, name)) pybind11_fail("Error during initialization: multiple incompatible definitions with name \"" + @@ -902,25 +1021,71 @@ class module : public object { PyModule_AddObject(ptr(), name, obj.inc_ref().ptr() /* steals a reference */); } + +#if PY_MAJOR_VERSION >= 3 + using module_def = PyModuleDef; +#else + struct module_def {}; +#endif + + /** \rst + Create a new top-level module that can be used as the main module of a C extension. + + For Python 3, ``def`` should point to a statically allocated module_def. + For Python 2, ``def`` can be a nullptr and is completely ignored. + \endrst */ + static module_ create_extension_module(const char *name, const char *doc, module_def *def) { +#if PY_MAJOR_VERSION >= 3 + // module_def is PyModuleDef + def = new (def) PyModuleDef { // Placement new (not an allocation). + /* m_base */ PyModuleDef_HEAD_INIT, + /* m_name */ name, + /* m_doc */ options::show_user_defined_docstrings() ? doc : nullptr, + /* m_size */ -1, + /* m_methods */ nullptr, + /* m_slots */ nullptr, + /* m_traverse */ nullptr, + /* m_clear */ nullptr, + /* m_free */ nullptr + }; + auto m = PyModule_Create(def); +#else + // Ignore module_def *def; only necessary for Python 3 + (void) def; + auto m = Py_InitModule3(name, nullptr, options::show_user_defined_docstrings() ? doc : nullptr); +#endif + if (m == nullptr) { + if (PyErr_Occurred()) + throw error_already_set(); + pybind11_fail("Internal error in module_::create_extension_module()"); + } + // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when returned from PyInit_... + // For Python 2, reinterpret_borrow is correct. + return reinterpret_borrow(m); + } }; +// When inside a namespace (or anywhere as long as it's not the first item on a line), +// C++20 allows "module" to be used. This is provided for backward compatibility, and for +// simplicity, if someone wants to use py::module for example, that is perfectly safe. +using module = module_; + /// \ingroup python_builtins /// Return a dictionary representing the global variables in the current execution frame, /// or ``__main__.__dict__`` if there is no frame (usually when the interpreter is embedded). inline dict globals() { PyObject *p = PyEval_GetGlobals(); - return reinterpret_borrow(p ? p : module::import("__main__").attr("__dict__").ptr()); + return reinterpret_borrow(p ? p : module_::import("__main__").attr("__dict__").ptr()); } PYBIND11_NAMESPACE_BEGIN(detail) /// Generic support for creating new Python heap types class generic_type : public object { - template friend class class_; public: PYBIND11_OBJECT_DEFAULT(generic_type, object, PyType_Check) protected: void initialize(const type_record &rec) { - if (rec.scope && hasattr(rec.scope, rec.name)) + if (rec.scope && hasattr(rec.scope, "__dict__") && rec.scope.attr("__dict__").contains(rec.name)) pybind11_fail("generic_type: cannot initialize type \"" + std::string(rec.name) + "\": an object with that name is already defined"); @@ -986,13 +1151,13 @@ class generic_type : public object { void install_buffer_funcs( buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { - PyHeapTypeObject *type = (PyHeapTypeObject*) m_ptr; + auto *type = (PyHeapTypeObject*) m_ptr; auto tinfo = detail::get_type_info(&type->ht_type); if (!type->ht_type.tp_as_buffer) pybind11_fail( "To be able to register buffer protocol support for the type '" + - std::string(tinfo->type->tp_name) + + get_fully_qualified_tp_name(tinfo->type) + "' the associated class<>(..) invocation must " "include the pybind11::buffer_protocol() annotation!"); @@ -1164,7 +1329,7 @@ template <> struct holder_check_impl : public holder_check_impl { template static bool check_destruct(detail::instance* inst, detail::holder_erased holder_raw) { - const holder_type& h = holder_raw.cast(); + const auto& h = holder_raw.cast(); handle src((PyObject*)inst); const detail::type_info *lowest_type = get_lowest_type(src, false); if (!lowest_type) @@ -1323,8 +1488,13 @@ class class_ : public detail::generic_type { static void del_wrapped(handle self, object del_orig) { // This should be called when the item is *actually* being deleted // TODO(eric.cousineau): Do we care about use cases where the user manually calls this? - detail::instance* inst = (detail::instance*)self.ptr(); - const detail::type_info *lowest_type = detail::get_lowest_type(self); + auto *inst = (detail::instance *) self.ptr(); + const detail::type_info *lowest_type = detail::get_lowest_type(self, false); + if (!lowest_type) { + // This should only happen in a multi-inheritance case (which is + // not yet supported for this fork's ownership transfer setup). + return; + } auto& release_info = lowest_type->release_info; // The references are as follows: // 1. When Python calls __del__ via tp_del (default slot) @@ -1397,7 +1567,7 @@ class class_ : public detail::generic_type { } } bool transfer_holder = true; - holder_type& holder = v_h.holder(); + auto &holder = v_h.holder(); if (!external_holder_raw.ptr()) { if (holder_check::allow_null_external_holder(holder)) transfer_holder = false; @@ -1406,7 +1576,7 @@ class class_ : public detail::generic_type { } if (transfer_holder) { if (external_holder_raw.type_id() == holder_type_id) { - holder_type& external_holder = external_holder_raw.mutable_cast(); + auto &external_holder = external_holder_raw.mutable_cast(); external_holder = std::move(holder); } else { // Only allow unique_ptr<> -> shared_ptr<> @@ -1444,8 +1614,8 @@ class class_ : public detail::generic_type { { // TODO(eric.cousineau): Consider releasing a raw pointer, to make it easier for // interop with purely raw pointers? Nah, just rely on release. - holder_type& holder = v_h.holder(); - holder_type& external_holder = external_holder_raw.mutable_cast(); + auto &holder = v_h.holder(); + auto &external_holder = external_holder_raw.mutable_cast(); new (&holder) holder_type(std::move(external_holder)); v_h.set_holder_constructed(true); @@ -1536,15 +1706,20 @@ class class_ : public detail::generic_type { return *this; } - template class_& def_buffer(Func &&func) { + template + class_& def_buffer(Func &&func) { struct capture { Func func; }; - capture *ptr = new capture { std::forward(func) }; + auto *ptr = new capture { std::forward(func) }; install_buffer_funcs([](PyObject *obj, void *ptr) -> buffer_info* { detail::make_caster caster; if (!caster.load(obj, false)) return nullptr; return new buffer_info(((capture *) ptr)->func(caster)); }, ptr); + weakref(m_ptr, cpp_function([ptr](handle wr) { + delete ptr; + wr.dec_ref(); + })).release(); return *this; } @@ -1671,14 +1846,13 @@ class class_ : public detail::generic_type { template static void init_holder(detail::instance *inst, detail::value_and_holder &v_h, const holder_type * /* unused */, const std::enable_shared_from_this * /* dummy */) { - try { - auto sh = std::dynamic_pointer_cast( - v_h.value_ptr()->shared_from_this()); - if (sh) { - new (std::addressof(v_h.holder())) holder_type(std::move(sh)); - v_h.set_holder_constructed(); - } - } catch (const std::bad_weak_ptr &) {} + + auto sh = std::dynamic_pointer_cast( + detail::try_get_shared_from_this(v_h.value_ptr())); + if (sh) { + new (std::addressof(v_h.holder())) holder_type(std::move(sh)); + v_h.set_holder_constructed(); + } if (!v_h.holder_constructed() && inst->owned) { new (std::addressof(v_h.holder())) holder_type(v_h.value_ptr()); @@ -1737,7 +1911,7 @@ class class_ : public detail::generic_type { if (!v_h.holder_constructed()) { throw std::runtime_error("Bad edge case"); } - holder_type& holder = v_h.holder(); + auto& holder = v_h.holder(); holder_check::accept_holder(holder_ptr, holder); } @@ -1746,7 +1920,7 @@ class class_ : public detail::generic_type { handle h_type = self.get_type(); // Use hacky Python-style inheritance check. - PyTypeObject *py_type = (PyTypeObject*)h_type.ptr(); + auto *py_type = (PyTypeObject*)h_type.ptr(); bool is_py_derived = py_type->tp_dealloc != detail::pybind11_object_dealloc; bool can_add_del = true; @@ -1842,6 +2016,16 @@ detail::initimpl::pickle_factory pickle(GetState &&g, SetSta } PYBIND11_NAMESPACE_BEGIN(detail) + +inline str enum_name(handle arg) { + dict entries = arg.get_type().attr("__entries"); + for (auto kv : entries) { + if (handle(kv.second[int_(0)]).equal(arg)) + return pybind11::str(kv.first); + } + return "???"; +} + struct enum_base { enum_base(handle base, handle parent) : m_base(base), m_parent(parent) { } @@ -1851,29 +2035,21 @@ struct enum_base { auto static_property = handle((PyObject *) get_internals().static_property_type); m_base.attr("__repr__") = cpp_function( - [](handle arg) -> str { - handle type = arg.get_type(); + [](object arg) -> str { + handle type = type::handle_of(arg); object type_name = type.attr("__name__"); - dict entries = type.attr("__entries"); - for (const auto &kv : entries) { - object other = kv.second[int_(0)]; - if (other.equal(arg)) - return pybind11::str("{}.{}").format(type_name, kv.first); - } - return pybind11::str("{}.???").format(type_name); + return pybind11::str("<{}.{}: {}>").format(type_name, enum_name(arg), int_(arg)); }, name("__repr__"), is_method(m_base) ); - m_base.attr("name") = property(cpp_function( + m_base.attr("name") = property(cpp_function(&enum_name, name("name"), is_method(m_base))); + + m_base.attr("__str__") = cpp_function( [](handle arg) -> str { - dict entries = arg.get_type().attr("__entries"); - for (const auto &kv : entries) { - if (handle(kv.second[int_(0)]).equal(arg)) - return pybind11::str(kv.first); - } - return "???"; + object type_name = type::handle_of(arg).attr("__name__"); + return pybind11::str("{}.{}").format(type_name, enum_name(arg)); }, name("name"), is_method(m_base) - )); + ); m_base.attr("__doc__") = static_property(cpp_function( [](handle arg) -> std::string { @@ -1882,7 +2058,7 @@ struct enum_base { if (((PyTypeObject *) arg.ptr())->tp_doc) docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; docstring += "Members:"; - for (const auto &kv : entries) { + for (auto kv : entries) { auto key = std::string(pybind11::str(kv.first)); auto comment = kv.second[int_(1)]; docstring += "\n\n " + key; @@ -1896,7 +2072,7 @@ struct enum_base { m_base.attr("__members__") = static_property(cpp_function( [](handle arg) -> dict { dict entries = arg.attr("__entries"), m; - for (const auto &kv : entries) + for (auto kv : entries) m[kv.first] = kv.second[int_(0)]; return m; }, name("__members__")), none(), none(), "" @@ -1905,11 +2081,11 @@ struct enum_base { #define PYBIND11_ENUM_OP_STRICT(op, expr, strict_behavior) \ m_base.attr(op) = cpp_function( \ [](object a, object b) { \ - if (!a.get_type().is(b.get_type())) \ + if (!type::handle_of(a).is(type::handle_of(b))) \ strict_behavior; \ return expr; \ }, \ - name(op), is_method(m_base)) + name(op), is_method(m_base), arg("other")) #define PYBIND11_ENUM_OP_CONV(op, expr) \ m_base.attr(op) = cpp_function( \ @@ -1917,7 +2093,7 @@ struct enum_base { int_ a(a_), b(b_); \ return expr; \ }, \ - name(op), is_method(m_base)) + name(op), is_method(m_base), arg("other")) #define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \ m_base.attr(op) = cpp_function( \ @@ -1925,7 +2101,7 @@ struct enum_base { int_ a(a_); \ return expr; \ }, \ - name(op), is_method(m_base)) + name(op), is_method(m_base), arg("other")) if (is_convertible) { PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b)); @@ -1984,7 +2160,7 @@ struct enum_base { PYBIND11_NOINLINE void export_values() { dict entries = m_base.attr("__entries"); - for (const auto &kv : entries) + for (auto kv : entries) m_parent.attr(kv.first) = kv.second[int_(0)]; } @@ -2011,7 +2187,8 @@ template class enum_ : public class_ { constexpr bool is_convertible = std::is_convertible::value; m_base.init(is_arithmetic, is_convertible); - def(init([](Scalar i) { return static_cast(i); })); + def(init([](Scalar i) { return static_cast(i); }), arg("value")); + def_property_readonly("value", [](Type value) { return (Scalar) value; }); def("__int__", [](Type value) { return (Scalar) value; }); #if PY_MAJOR_VERSION < 3 def("__long__", [](Type value) { return (Scalar) value; }); @@ -2025,7 +2202,7 @@ template class enum_ : public class_ { detail::initimpl::setstate(v_h, static_cast(arg), Py_TYPE(v_h.inst) != v_h.type->type); }, detail::is_new_style_constructor(), - pybind11::name("__setstate__"), is_method(*this)); + pybind11::name("__setstate__"), is_method(*this), arg("state")); } /// Export enumeration entries into the parent scope @@ -2120,10 +2297,12 @@ PYBIND11_NAMESPACE_END(detail) template ()), +#endif typename... Extra> iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) { - typedef detail::iterator_state state; + using state = detail::iterator_state; if (!detail::get_type_info(typeid(state), false)) { class_(handle(), "iterator", pybind11::module_local()) @@ -2149,10 +2328,12 @@ iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) { template ()).first), +#endif typename... Extra> iterator make_key_iterator(Iterator first, Sentinel last, Extra &&... extra) { - typedef detail::iterator_state state; + using state = detail::iterator_state; if (!detail::get_type_info(typeid(state), false)) { class_(handle(), "iterator", pybind11::module_local()) @@ -2190,7 +2371,7 @@ template void implicitly_convertible() { struct set_flag { bool &flag; - set_flag(bool &flag) : flag(flag) { flag = true; } + set_flag(bool &flag_) : flag(flag_) { flag_ = true; } ~set_flag() { flag = false; } }; auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * { @@ -2231,11 +2412,11 @@ template class exception : public object { public: exception() = default; - exception(handle scope, const char *name, PyObject *base = PyExc_Exception) { + exception(handle scope, const char *name, handle base = PyExc_Exception) { std::string full_name = scope.attr("__name__").cast() + std::string(".") + name; - m_ptr = PyErr_NewException(const_cast(full_name.c_str()), base, NULL); - if (hasattr(scope, name)) + m_ptr = PyErr_NewException(const_cast(full_name.c_str()), base.ptr(), NULL); + if (hasattr(scope, "__dict__") && scope.attr("__dict__").contains(name)) pybind11_fail("Error during initialization: multiple incompatible " "definitions with name \"" + std::string(name) + "\""); scope.attr(name) = *this; @@ -2264,7 +2445,7 @@ PYBIND11_NAMESPACE_END(detail) template exception ®ister_exception(handle scope, const char *name, - PyObject *base = PyExc_Exception) { + handle base = PyExc_Exception) { auto &ex = detail::get_exception_object(); if (!ex) ex = exception(scope, name, base); @@ -2293,7 +2474,7 @@ PYBIND11_NOINLINE inline void print(tuple args, dict kwargs) { file = kwargs["file"].cast(); } else { try { - file = module::import("sys").attr("stdout"); + file = module_::import("sys").attr("stdout"); } catch (const error_already_set &) { /* If print() is called from code that is executed as part of garbage collection during interpreter shutdown, @@ -2318,151 +2499,6 @@ void print(Args &&...args) { detail::print(c.args(), c.kwargs()); } -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) - -/* The functions below essentially reproduce the PyGILState_* API using a RAII - * pattern, but there are a few important differences: - * - * 1. When acquiring the GIL from an non-main thread during the finalization - * phase, the GILState API blindly terminates the calling thread, which - * is often not what is wanted. This API does not do this. - * - * 2. The gil_scoped_release function can optionally cut the relationship - * of a PyThreadState and its associated thread, which allows moving it to - * another thread (this is a fairly rare/advanced use case). - * - * 3. The reference count of an acquired thread state can be controlled. This - * can be handy to prevent cases where callbacks issued from an external - * thread would otherwise constantly construct and destroy thread state data - * structures. - * - * See the Python bindings of NanoGUI (http://github.com/wjakob/nanogui) for an - * example which uses features 2 and 3 to migrate the Python thread of - * execution to another thread (to run the event loop on the original thread, - * in this case). - */ - -class gil_scoped_acquire { -public: - PYBIND11_NOINLINE gil_scoped_acquire() { - auto const &internals = detail::get_internals(); - tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate); - - if (!tstate) { - /* Check if the GIL was acquired using the PyGILState_* API instead (e.g. if - calling from a Python thread). Since we use a different key, this ensures - we don't create a new thread state and deadlock in PyEval_AcquireThread - below. Note we don't save this state with internals.tstate, since we don't - create it we would fail to clear it (its reference count should be > 0). */ - tstate = PyGILState_GetThisThreadState(); - } - - if (!tstate) { - tstate = PyThreadState_New(internals.istate); - #if !defined(NDEBUG) - if (!tstate) - pybind11_fail("scoped_acquire: could not create thread state!"); - #endif - tstate->gilstate_counter = 0; - PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); - } else { - release = detail::get_thread_state_unchecked() != tstate; - } - - if (release) { - /* Work around an annoying assertion in PyThreadState_Swap */ - #if defined(Py_DEBUG) - PyInterpreterState *interp = tstate->interp; - tstate->interp = nullptr; - #endif - PyEval_AcquireThread(tstate); - #if defined(Py_DEBUG) - tstate->interp = interp; - #endif - } - - inc_ref(); - } - - void inc_ref() { - ++tstate->gilstate_counter; - } - - PYBIND11_NOINLINE void dec_ref() { - --tstate->gilstate_counter; - #if !defined(NDEBUG) - if (detail::get_thread_state_unchecked() != tstate) - pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); - if (tstate->gilstate_counter < 0) - pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); - #endif - if (tstate->gilstate_counter == 0) { - #if !defined(NDEBUG) - if (!release) - pybind11_fail("scoped_acquire::dec_ref(): internal error!"); - #endif - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); - PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); - release = false; - } - } - - PYBIND11_NOINLINE ~gil_scoped_acquire() { - dec_ref(); - if (release) - PyEval_SaveThread(); - } -private: - PyThreadState *tstate = nullptr; - bool release = true; -}; - -class gil_scoped_release { -public: - explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { - // `get_internals()` must be called here unconditionally in order to initialize - // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an - // initialization race could occur as multiple threads try `gil_scoped_acquire`. - const auto &internals = detail::get_internals(); - tstate = PyEval_SaveThread(); - if (disassoc) { - auto key = internals.tstate; - PYBIND11_TLS_DELETE_VALUE(key); - } - } - ~gil_scoped_release() { - if (!tstate) - return; - PyEval_RestoreThread(tstate); - if (disassoc) { - auto key = detail::get_internals().tstate; - PYBIND11_TLS_REPLACE_VALUE(key, tstate); - } - } -private: - PyThreadState *tstate; - bool disassoc; -}; -#elif defined(PYPY_VERSION) -class gil_scoped_acquire { - PyGILState_STATE state; -public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } - ~gil_scoped_acquire() { PyGILState_Release(state); } -}; - -class gil_scoped_release { - PyThreadState *state; -public: - gil_scoped_release() { state = PyEval_SaveThread(); } - ~gil_scoped_release() { PyEval_RestoreThread(state); } -}; -#else -class gil_scoped_acquire { }; -class gil_scoped_release { }; -#endif - error_already_set::~error_already_set() { if (m_type) { gil_scoped_acquire gil; @@ -2473,8 +2509,9 @@ error_already_set::~error_already_set() { } } -inline function get_type_overload(const void *this_ptr, const detail::type_info *this_type, const char *name) { - handle self = detail::get_object_handle(this_ptr, this_type); +PYBIND11_NAMESPACE_BEGIN(detail) +inline function get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { + handle self = get_object_handle(this_ptr, this_type); if (!self) return function(); handle type = self.get_type(); @@ -2483,14 +2520,14 @@ inline function get_type_overload(const void *this_ptr, const detail::type_info std::string full_name = type.attr("__qualname__").cast() + "." + name; auto key = std::make_pair(type.ptr(), full_name); - /* Cache functions that aren't overloaded in Python to avoid + /* Cache functions that aren't overridden in Python to avoid many costly Python dictionary lookups below */ - auto &cache = detail::get_internals().inactive_overload_cache; + auto &cache = get_internals().inactive_override_cache; if (cache.find(key) != cache.end()) return function(); - function overload = getattr(self, name, function()); - if (overload.is_cpp_function()) { + function override = getattr(self, name, function()); + if (override.is_cpp_function()) { cache.insert(key); return function(); } @@ -2530,34 +2567,36 @@ inline function get_type_overload(const void *this_ptr, const detail::type_info Py_DECREF(result); #endif - return overload; + return override; } +PYBIND11_NAMESPACE_END(detail) /** \rst Try to retrieve a python method by the provided name from the instance pointed to by the this_ptr. - :this_ptr: The pointer to the object the overload should be retrieved for. This should be the first - non-trampoline class encountered in the inheritance chain. - :name: The name of the overloaded Python method to retrieve. + :this_ptr: The pointer to the object the overridden method should be retrieved for. This should be + the first non-trampoline class encountered in the inheritance chain. + :name: The name of the overridden Python method to retrieve. :return: The Python method by this name from the object or an empty function wrapper. \endrst */ -template function get_overload(const T *this_ptr, const char *name) { +template function get_override(const T *this_ptr, const char *name) { auto tinfo = detail::get_type_info(typeid(T)); - return tinfo ? get_type_overload(this_ptr, tinfo, name) : function(); + return tinfo ? detail::get_type_override(this_ptr, tinfo, name) : function(); } -#define PYBIND11_OVERLOAD_INT(ret_type, cname, name, ...) { \ +#define PYBIND11_OVERRIDE_IMPL(ret_type, cname, name, ...) \ + do { \ pybind11::gil_scoped_acquire gil; \ - pybind11::function overload = pybind11::get_overload(static_cast(this), name); \ - if (overload) { \ - auto o = overload(__VA_ARGS__); \ + pybind11::function override = pybind11::get_override(static_cast(this), name); \ + if (override) { \ + auto o = override(__VA_ARGS__); \ if (pybind11::detail::cast_is_temporary_value_reference::value) { \ - static pybind11::detail::overload_caster_t caster; \ + static pybind11::detail::override_caster_t caster; \ return pybind11::detail::cast_ref(std::move(o), caster); \ } \ else return pybind11::detail::cast_safe(std::move(o)); \ } \ - } + } while (false) /** \rst Macro to populate the virtual method in the trampoline class. This macro tries to look up a method named 'fn' @@ -2568,7 +2607,7 @@ template function get_overload(const T *this_ptr, const char *name) { .. code-block:: cpp std::string toString() override { - PYBIND11_OVERLOAD_NAME( + PYBIND11_OVERRIDE_NAME( std::string, // Return type (ret_type) Animal, // Parent class (cname) "__str__", // Name of method in Python (name) @@ -2576,17 +2615,21 @@ template function get_overload(const T *this_ptr, const char *name) { ); } \endrst */ -#define PYBIND11_OVERLOAD_NAME(ret_type, cname, name, fn, ...) \ - PYBIND11_OVERLOAD_INT(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__) \ - return cname::fn(__VA_ARGS__) +#define PYBIND11_OVERRIDE_NAME(ret_type, cname, name, fn, ...) \ + do { \ + PYBIND11_OVERRIDE_IMPL(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__); \ + return cname::fn(__VA_ARGS__); \ + } while (false) /** \rst - Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERLOAD_NAME`, except that it - throws if no overload can be found. + Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERRIDE_NAME`, except that it + throws if no override can be found. \endrst */ -#define PYBIND11_OVERLOAD_PURE_NAME(ret_type, cname, name, fn, ...) \ - PYBIND11_OVERLOAD_INT(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__) \ - pybind11::pybind11_fail("Tried to call pure virtual function \"" PYBIND11_STRINGIFY(cname) "::" name "\""); +#define PYBIND11_OVERRIDE_PURE_NAME(ret_type, cname, name, fn, ...) \ + do { \ + PYBIND11_OVERRIDE_IMPL(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__); \ + pybind11::pybind11_fail("Tried to call pure virtual function \"" PYBIND11_STRINGIFY(cname) "::" name "\""); \ + } while (false) /** \rst Macro to populate the virtual method in the trampoline class. This macro tries to look up the method @@ -2603,7 +2646,7 @@ template function get_overload(const T *this_ptr, const char *name) { // Trampoline (need one for each virtual function) std::string go(int n_times) override { - PYBIND11_OVERLOAD_PURE( + PYBIND11_OVERRIDE_PURE( std::string, // Return type (ret_type) Animal, // Parent class (cname) go, // Name of function in C++ (must match Python name) (fn) @@ -2612,15 +2655,39 @@ template function get_overload(const T *this_ptr, const char *name) { } }; \endrst */ -#define PYBIND11_OVERLOAD(ret_type, cname, fn, ...) \ - PYBIND11_OVERLOAD_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) +#define PYBIND11_OVERRIDE(ret_type, cname, fn, ...) \ + PYBIND11_OVERRIDE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) /** \rst - Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERLOAD`, except that it throws - if no overload can be found. + Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERRIDE`, except that it throws + if no override can be found. \endrst */ +#define PYBIND11_OVERRIDE_PURE(ret_type, cname, fn, ...) \ + PYBIND11_OVERRIDE_PURE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) + + +// Deprecated versions + +PYBIND11_DEPRECATED("get_type_overload has been deprecated") +inline function get_type_overload(const void *this_ptr, const detail::type_info *this_type, const char *name) { + return detail::get_type_override(this_ptr, this_type, name); +} + +template +inline function get_overload(const T *this_ptr, const char *name) { + return get_override(this_ptr, name); +} + +#define PYBIND11_OVERLOAD_INT(ret_type, cname, name, ...) \ + PYBIND11_OVERRIDE_IMPL(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__) +#define PYBIND11_OVERLOAD_NAME(ret_type, cname, name, fn, ...) \ + PYBIND11_OVERRIDE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, fn, __VA_ARGS__) +#define PYBIND11_OVERLOAD_PURE_NAME(ret_type, cname, name, fn, ...) \ + PYBIND11_OVERRIDE_PURE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, fn, __VA_ARGS__); +#define PYBIND11_OVERLOAD(ret_type, cname, fn, ...) \ + PYBIND11_OVERRIDE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__) #define PYBIND11_OVERLOAD_PURE(ret_type, cname, fn, ...) \ - PYBIND11_OVERLOAD_PURE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) + PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f1d8e96cdf..2ce750da97 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -20,6 +20,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /* A few forward declarations */ class handle; class object; class str; class iterator; +class type; struct arg; struct arg_v; PYBIND11_NAMESPACE_BEGIN(detail) @@ -35,7 +36,7 @@ namespace accessor_policies { struct sequence_item; struct list_item; struct tuple_item; -} +} // namespace accessor_policies using obj_attr_accessor = accessor; using str_attr_accessor = accessor; using item_accessor = accessor; @@ -152,7 +153,8 @@ class object_api : public pyobject_tag { /// Return the object's current reference count int ref_count() const { return static_cast(Py_REFCNT(derived().ptr())); } - /// Return a handle to the Python type object underlying the instance + + // TODO PYBIND11_DEPRECATED("Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()") handle get_type() const; private: @@ -241,7 +243,7 @@ class object : public handle { ~object() { dec_ref(); } /** \rst - Resets the internal pointer to ``nullptr`` without without decreasing the + Resets the internal pointer to ``nullptr`` without decreasing the object's reference count. The function returns a raw handle to the original Python object. \endrst */ @@ -278,8 +280,10 @@ class object : public handle { struct borrowed_t { }; struct stolen_t { }; +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Issue in breathe 4.26.1 template friend T reinterpret_borrow(handle); template friend T reinterpret_steal(handle); +#endif public: // Only accessible from derived classes and the reinterpret_* functions @@ -331,13 +335,27 @@ class error_already_set : public std::runtime_error { error_already_set(const error_already_set &) = default; error_already_set(error_already_set &&) = default; - inline ~error_already_set(); + inline ~error_already_set() override; /// Give the currently-held error back to Python, if any. If there is currently a Python error /// already set it is cleared first. After this call, the current object no longer stores the /// error variables (but the `.what()` string is still available). void restore() { PyErr_Restore(m_type.release().ptr(), m_value.release().ptr(), m_trace.release().ptr()); } + /// If it is impossible to raise the currently-held error, such as in destructor, we can write + /// it out using Python's unraisable hook (sys.unraisablehook). The error context should be + /// some object whose repr() helps identify the location of the error. Python already knows the + /// type and value of the error, so there is no need to repeat that. For example, __func__ could + /// be helpful. After this call, the current object no longer stores the error variables, + /// and neither does Python. + void discard_as_unraisable(object err_context) { + restore(); + PyErr_WriteUnraisable(err_context.ptr()); + } + void discard_as_unraisable(const char *err_context) { + discard_as_unraisable(reinterpret_steal(PYBIND11_FROM_STRING(err_context))); + } + // Does nothing; provided for backwards compatibility. PYBIND11_DEPRECATED("Use of error_already_set.clear() is deprecated") void clear() {} @@ -737,11 +755,14 @@ inline bool PyIterable_Check(PyObject *obj) { } inline bool PyNone_Check(PyObject *o) { return o == Py_None; } -#if PY_MAJOR_VERSION >= 3 inline bool PyEllipsis_Check(PyObject *o) { return o == Py_Ellipsis; } -#endif +#ifdef PYBIND11_STR_LEGACY_PERMISSIVE inline bool PyUnicode_Check_Permissive(PyObject *o) { return PyUnicode_Check(o) || PYBIND11_BYTES_CHECK(o); } +#define PYBIND11_STR_CHECK_FUN detail::PyUnicode_Check_Permissive +#else +#define PYBIND11_STR_CHECK_FUN PyUnicode_Check +#endif inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; } @@ -785,7 +806,9 @@ PYBIND11_NAMESPACE_END(detail) Name(handle h, stolen_t) : Parent(h, stolen_t{}) { } \ PYBIND11_DEPRECATED("Use py::isinstance(obj) instead") \ bool check() const { return m_ptr != nullptr && (bool) CheckFun(m_ptr); } \ - static bool check_(handle h) { return h.ptr() != nullptr && CheckFun(h.ptr()); } + static bool check_(handle h) { return h.ptr() != nullptr && CheckFun(h.ptr()); } \ + template \ + Name(const ::pybind11::detail::accessor &a) : Name(object(a)) { } #define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ @@ -795,15 +818,24 @@ PYBIND11_NAMESPACE_END(detail) { if (!m_ptr) throw error_already_set(); } \ Name(object &&o) \ : Parent(check_(o) ? o.release().ptr() : ConvertFun(o.ptr()), stolen_t{}) \ - { if (!m_ptr) throw error_already_set(); } \ - template \ - Name(const ::pybind11::detail::accessor &a) : Name(object(a)) { } + { if (!m_ptr) throw error_already_set(); } + +#define PYBIND11_OBJECT_CVT_DEFAULT(Name, Parent, CheckFun, ConvertFun) \ + PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ + Name() : Parent() { } + +#define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ + ::pybind11::type_error("Object of type '" + \ + ::pybind11::detail::get_fully_qualified_tp_name(Py_TYPE(o_ptr)) + \ + "' is not an instance of '" #Name "'") #define PYBIND11_OBJECT(Name, Parent, CheckFun) \ PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ /* This is deliberately not 'explicit' to allow implicit conversion from object: */ \ - Name(const object &o) : Parent(o) { } \ - Name(object &&o) : Parent(std::move(o)) { } + Name(const object &o) : Parent(o) \ + { if (m_ptr && !check_(m_ptr)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); } \ + Name(object &&o) : Parent(std::move(o)) \ + { if (m_ptr && !check_(m_ptr)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); } #define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ PYBIND11_OBJECT(Name, Parent, CheckFun) \ @@ -879,6 +911,32 @@ class iterator : public object { object value = {}; }; + + +class type : public object { +public: + PYBIND11_OBJECT(type, object, PyType_Check) + + /// Return a type handle from a handle or an object + static handle handle_of(handle h) { return handle((PyObject*) Py_TYPE(h.ptr())); } + + /// Return a type object from a handle or an object + static type of(handle h) { return type(type::handle_of(h), borrowed_t{}); } + + // Defined in pybind11/cast.h + /// Convert C++ type to handle if previously registered. Does not convert + /// standard types, like int, float. etc. yet. + /// See https://github.com/pybind/pybind11/issues/2486 + template + static handle handle_of(); + + /// Convert C++ type to type if previously registered. Does not convert + /// standard types, like int, float. etc. yet. + /// See https://github.com/pybind/pybind11/issues/2486 + template + static type of() {return type(type::handle_of(), borrowed_t{}); } +}; + class iterable : public object { public: PYBIND11_OBJECT_DEFAULT(iterable, object, detail::PyIterable_Check) @@ -888,7 +946,7 @@ class bytes; class str : public object { public: - PYBIND11_OBJECT_CVT(str, object, detail::PyUnicode_Check_Permissive, raw_str) + PYBIND11_OBJECT_CVT(str, object, PYBIND11_STR_CHECK_FUN, raw_str) str(const char *c, size_t n) : object(PyUnicode_FromStringAndSize(c, (ssize_t) n), stolen_t{}) { @@ -909,7 +967,7 @@ class str : public object { Return a string representation of the object. This is analogous to the ``str()`` function in Python. \endrst */ - explicit str(handle h) : object(raw_str(h.ptr()), stolen_t{}) { } + explicit str(handle h) : object(raw_str(h.ptr()), stolen_t{}) { if (!m_ptr) throw error_already_set(); } operator std::string() const { object temp = *this; @@ -934,8 +992,8 @@ class str : public object { /// Return string representation -- always returns a new reference, even if already a str static PyObject *raw_str(PyObject *op) { PyObject *str_value = PyObject_Str(op); - if (!str_value) throw error_already_set(); #if PY_MAJOR_VERSION < 3 + if (!str_value) throw error_already_set(); PyObject *unicode = PyUnicode_FromEncodedObject(str_value, "utf-8", nullptr); Py_XDECREF(str_value); str_value = unicode; #endif @@ -949,7 +1007,7 @@ inline namespace literals { String literal version of `str` \endrst */ inline str operator"" _s(const char *s, size_t size) { return {s, size}; } -} +} // namespace literals /// \addtogroup pytypes /// @{ @@ -1013,6 +1071,34 @@ inline str::str(const bytes& b) { m_ptr = obj.release().ptr(); } +/// \addtogroup pytypes +/// @{ +class bytearray : public object { +public: + PYBIND11_OBJECT_CVT(bytearray, object, PyByteArray_Check, PyByteArray_FromObject) + + bytearray(const char *c, size_t n) + : object(PyByteArray_FromStringAndSize(c, (ssize_t) n), stolen_t{}) { + if (!m_ptr) pybind11_fail("Could not allocate bytearray object!"); + } + + bytearray() + : bytearray("", 0) {} + + explicit bytearray(const std::string &s) : bytearray(s.data(), s.size()) { } + + size_t size() const { return static_cast(PyByteArray_Size(m_ptr)); } + + explicit operator std::string() const { + char *buffer = PyByteArray_AS_STRING(m_ptr); + ssize_t size = PyByteArray_GET_SIZE(m_ptr); + return std::string(buffer, static_cast(size)); + } +}; +// Note: breathe >= 4.17.0 will fail to build docs if the below two constructors +// are included in the doxygen group; close here and reopen after as a workaround +/// @} pytypes + /// \addtogroup pytypes /// @{ class none : public object { @@ -1021,13 +1107,11 @@ class none : public object { none() : object(Py_None, borrowed_t{}) { } }; -#if PY_MAJOR_VERSION >= 3 class ellipsis : public object { public: PYBIND11_OBJECT(ellipsis, object, detail::PyEllipsis_Check) ellipsis() : object(Py_Ellipsis, borrowed_t{}) { } }; -#endif class bool_ : public object { public: @@ -1117,11 +1201,16 @@ class float_ : public object { class weakref : public object { public: - PYBIND11_OBJECT_DEFAULT(weakref, object, PyWeakref_Check) + PYBIND11_OBJECT_CVT_DEFAULT(weakref, object, PyWeakref_Check, raw_weakref) explicit weakref(handle obj, handle callback = {}) : object(PyWeakref_NewRef(obj.ptr(), callback.ptr()), stolen_t{}) { if (!m_ptr) pybind11_fail("Could not allocate weak reference!"); } + +private: + static PyObject *raw_weakref(PyObject *o) { + return PyWeakref_NewRef(o, nullptr); + } }; class slice : public object { @@ -1192,12 +1281,24 @@ class capsule : public object { } template operator T *() const { + return get_pointer(); + } + + /// Get the pointer the capsule holds. + template + T* get_pointer() const { auto name = this->name(); - T * result = static_cast(PyCapsule_GetPointer(m_ptr, name)); + T *result = static_cast(PyCapsule_GetPointer(m_ptr, name)); if (!result) pybind11_fail("Unable to extract capsule contents!"); return result; } + /// Replaces a capsule's pointer *without* calling the destructor on the existing one. + void set_pointer(const void *value) { + if (PyCapsule_SetPointer(m_ptr, const_cast(value)) != 0) + pybind11_fail("Could not set capsule pointer"); + } + const char *name() const { return PyCapsule_GetName(m_ptr); } }; @@ -1215,6 +1316,15 @@ class tuple : public object { detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } }; +// We need to put this into a separate function because the Intel compiler +// fails to compile enable_if_t...>::value> part below +// (tested with ICC 2021.1 Beta 20200827). +template +constexpr bool args_are_all_keyword_or_ds() +{ + return detail::all_of...>::value; +} + class dict : public object { public: PYBIND11_OBJECT_CVT(dict, object, PyDict_Check, raw_dict) @@ -1222,7 +1332,7 @@ class dict : public object { if (!m_ptr) pybind11_fail("Could not allocate dict object!"); } template ...>::value>, + typename = detail::enable_if_t()>, // MSVC workaround: it can't compile an out-of-line definition, so defer the collector typename collector = detail::deferred_t, Args...>> explicit dict(Args &&...args) : dict(collector(std::forward(args)...).kwargs()) { } @@ -1326,7 +1436,7 @@ class buffer : public object { buffer_info request(bool writable = false) const { int flags = PyBUF_STRIDES | PyBUF_FORMAT; if (writable) flags |= PyBUF_WRITABLE; - Py_buffer *view = new Py_buffer(); + auto *view = new Py_buffer(); if (PyObject_GetBuffer(m_ptr, view, flags) != 0) { delete view; throw error_already_set(); @@ -1474,13 +1584,17 @@ inline memoryview memoryview::from_buffer( /// \addtogroup python_builtins /// @{ + +/// Get the length of a Python object. inline size_t len(handle h) { ssize_t result = PyObject_Length(h.ptr()); if (result < 0) - pybind11_fail("Unable to compute length of object"); + throw error_already_set(); return (size_t) result; } +/// Get the length hint of a Python object. +/// Returns 0 when this cannot be determined. inline size_t len_hint(handle h) { #if PY_VERSION_HEX >= 0x03040000 ssize_t result = PyObject_LengthHint(h.ptr(), 0); @@ -1524,7 +1638,7 @@ class wrapper : public Base { public: using Base::Base; - virtual ~wrapper() { + virtual ~wrapper() { // NOLINT(modernize-use-override) delete_py_if_in_cpp(); } @@ -1642,7 +1756,7 @@ template str_attr_accessor object_api::doc() const { return attr("__doc__"); } template -handle object_api::get_type() const { return (PyObject *) Py_TYPE(derived().ptr()); } +handle object_api::get_type() const { return type::handle_of(derived()); } template bool object_api::rich_compare(object_api const &other, int value) const { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 6c2bebda87..18fbafb1e2 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -144,7 +144,7 @@ template struct list_caster { using value_conv = make_caster; bool load(handle src, bool convert) { - if (!isinstance(src) || isinstance(src)) + if (!isinstance(src) || isinstance(src) || isinstance(src)) return false; auto s = reinterpret_borrow(src); value.clear(); @@ -289,7 +289,7 @@ template struct optional_caster { PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name + _("]")); }; -#if PYBIND11_HAS_OPTIONAL +#if defined(PYBIND11_HAS_OPTIONAL) template struct type_caster> : public optional_caster> {}; @@ -297,7 +297,7 @@ template<> struct type_caster : public void_caster {}; #endif -#if PYBIND11_HAS_EXP_OPTIONAL +#if defined(PYBIND11_HAS_EXP_OPTIONAL) template struct type_caster> : public optional_caster> {}; @@ -369,7 +369,7 @@ struct variant_caster> { PYBIND11_TYPE_CASTER(Type, _("Union[") + detail::concat(make_caster::name...) + _("]")); }; -#if PYBIND11_HAS_VARIANT +#if defined(PYBIND11_HAS_VARIANT) template struct type_caster> : variant_caster> { }; #endif diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 61b94b6220..83195ee496 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -223,7 +223,7 @@ void vector_modifiers(enable_if_treserve((size_t) slicelength); for (size_t i=0; i struct vector_has_data_and_format::format(), std::declval().data()), typename Vector::value_type*>::value>> : std::true_type {}; +// [workaround(intel)] Separate function required here +// Workaround as the Intel compiler does not compile the enable_if_t part below +// (tested with icc (ICC) 2021.1 Beta 20200827) +template +constexpr bool args_any_are_buffer() { + return detail::any_of...>::value; +} + +// [workaround(intel)] Separate function required here +// [workaround(msvc)] Can't use constexpr bool in return type + // Add the buffer interface to a vector template -enable_if_t...>::value> -vector_buffer(Class_& cl) { +void vector_buffer_impl(Class_& cl, std::true_type) { using T = typename Vector::value_type; static_assert(vector_has_data_and_format::value, "There is not an appropriate format descriptor for this vector"); @@ -397,21 +407,31 @@ vector_buffer(Class_& cl) { if (!detail::compare_buffer_info::compare(info) || (ssize_t) sizeof(T) != info.itemsize) throw type_error("Format mismatch (Python: " + info.format + " C++: " + format_descriptor::format() + ")"); - auto vec = std::unique_ptr(new Vector()); - vec->reserve((size_t) info.shape[0]); T *p = static_cast(info.ptr); ssize_t step = info.strides[0] / static_cast(sizeof(T)); T *end = p + info.shape[0] * step; - for (; p != end; p += step) - vec->push_back(*p); - return vec.release(); + if (step == 1) { + return Vector(p, end); + } + else { + Vector vec; + vec.reserve((size_t) info.shape[0]); + for (; p != end; p += step) + vec.push_back(*p); + return vec; + } })); return; } template -enable_if_t...>::value> vector_buffer(Class_&) {} +void vector_buffer_impl(Class_&, std::false_type) {} + +template +void vector_buffer(Class_& cl) { + vector_buffer_impl(cl, detail::any_of...>{}); +} PYBIND11_NAMESPACE_END(detail) diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 5b2f83d5cd..ad65420893 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -from ._version import version_info, __version__ # noqa: F401 imported but unused +from ._version import version_info, __version__ +from .commands import get_include, get_cmake_dir -def get_include(user=False): - import os - d = os.path.dirname(__file__) - if os.path.exists(os.path.join(d, "include")): - # Package is installed - return os.path.join(d, "include") - else: - # Package is from a source directory - return os.path.join(os.path.dirname(d), "include") + +__all__ = ( + "version_info", + "__version__", + "get_include", + "get_cmake_dir", +) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 5e393cc8f1..020988c671 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -5,33 +5,48 @@ import sys import sysconfig -from . import get_include +from .commands import get_include, get_cmake_dir def print_includes(): - dirs = [sysconfig.get_path('include'), - sysconfig.get_path('platinclude'), - get_include()] + # type: () -> None + dirs = [ + sysconfig.get_path("include"), + sysconfig.get_path("platinclude"), + get_include(), + ] # Make unique but preserve order unique_dirs = [] for d in dirs: - if d not in unique_dirs: + if d and d not in unique_dirs: unique_dirs.append(d) - print(' '.join('-I' + d for d in unique_dirs)) + print(" ".join("-I" + d for d in unique_dirs)) def main(): - parser = argparse.ArgumentParser(prog='python -m pybind11') - parser.add_argument('--includes', action='store_true', - help='Include flags for both pybind11 and Python headers.') + # type: () -> None + + parser = argparse.ArgumentParser() + parser.add_argument( + "--includes", + action="store_true", + help="Include flags for both pybind11 and Python headers.", + ) + parser.add_argument( + "--cmakedir", + action="store_true", + help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() if args.includes: print_includes() + if args.cmakedir: + print(get_cmake_dir()) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/pybind11/_version.py b/pybind11/_version.py index 1f2f254ce5..ffadcca5a8 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -1,3 +1,12 @@ # -*- coding: utf-8 -*- -version_info = (2, 5, 'dev1') -__version__ = '.'.join(map(str, version_info)) + + +def _to_int(s): + try: + return int(s) + except ValueError: + return s + + +__version__ = "2.6.3.dev1" +version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/pybind11/_version.pyi b/pybind11/_version.pyi new file mode 100644 index 0000000000..970184c770 --- /dev/null +++ b/pybind11/_version.pyi @@ -0,0 +1,6 @@ +from typing import Union, Tuple + +def _to_int(s: str) -> Union[int, str]: ... + +__version__: str +version_info: Tuple[Union[int, str], ...] diff --git a/pybind11/commands.py b/pybind11/commands.py new file mode 100644 index 0000000000..34dbaf8a89 --- /dev/null +++ b/pybind11/commands.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +import os + + +DIR = os.path.abspath(os.path.dirname(__file__)) + + +def get_include(user=False): + # type: (bool) -> str + installed_path = os.path.join(DIR, "include") + source_path = os.path.join(os.path.dirname(DIR), "include") + return installed_path if os.path.exists(installed_path) else source_path + + +def get_cmake_dir(): + # type: () -> str + cmake_installed_path = os.path.join(DIR, "share", "cmake", "pybind11") + if os.path.exists(cmake_installed_path): + return cmake_installed_path + else: + msg = "pybind11 not installed, installation required to access the CMake files" + raise ImportError(msg) diff --git a/pybind11/py.typed b/pybind11/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py new file mode 100644 index 0000000000..c69064ca50 --- /dev/null +++ b/pybind11/setup_helpers.py @@ -0,0 +1,444 @@ +# -*- coding: utf-8 -*- + +""" +This module provides helpers for C++11+ projects using pybind11. + +LICENSE: + +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +# IMPORTANT: If you change this file in the pybind11 repo, also review +# setup_helpers.pyi for matching changes. +# +# If you copy this file in, you don't +# need the .pyi file; it's just an interface file for static type checkers. + +import contextlib +import os +import shutil +import sys +import tempfile +import threading +import platform +import warnings + +try: + from setuptools.command.build_ext import build_ext as _build_ext + from setuptools import Extension as _Extension +except ImportError: + from distutils.command.build_ext import build_ext as _build_ext + from distutils.extension import Extension as _Extension + +import distutils.errors +import distutils.ccompiler + + +WIN = sys.platform.startswith("win32") +PY2 = sys.version_info[0] < 3 +MACOS = sys.platform.startswith("darwin") +STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" + + +# It is recommended to use PEP 518 builds if using this module. However, this +# file explicitly supports being copied into a user's project directory +# standalone, and pulling pybind11 with the deprecated setup_requires feature. +# If you copy the file, remember to add it to your MANIFEST.in, and add the current +# directory into your path if it sits beside your setup.py. + + +class Pybind11Extension(_Extension): + """ + Build a C++11+ Extension module with pybind11. This automatically adds the + recommended flags when you init the extension and assumes C++ sources - you + can further modify the options yourself. + + The customizations are: + + * ``/EHsc`` and ``/bigobj`` on Windows + * ``stdlib=libc++`` on macOS + * ``visibility=hidden`` and ``-g0`` on Unix + + Finally, you can set ``cxx_std`` via constructor or afterwords to enable + flags for C++ std, and a few extra helper flags related to the C++ standard + level. It is _highly_ recommended you either set this, or use the provided + ``build_ext``, which will search for the highest supported extension for + you if the ``cxx_std`` property is not set. Do not set the ``cxx_std`` + property more than once, as flags are added when you set it. Set the + property to None to disable the addition of C++ standard flags. + + If you want to add pybind11 headers manually, for example for an exact + git checkout, then set ``include_pybind11=False``. + + Warning: do not use property-based access to the instance on Python 2 - + this is an ugly old-style class due to Distutils. + """ + + # flags are prepended, so that they can be further overridden, e.g. by + # ``extra_compile_args=["-g"]``. + + def _add_cflags(self, flags): + self.extra_compile_args[:0] = flags + + def _add_ldflags(self, flags): + self.extra_link_args[:0] = flags + + def __init__(self, *args, **kwargs): + + self._cxx_level = 0 + cxx_std = kwargs.pop("cxx_std", 0) + + if "language" not in kwargs: + kwargs["language"] = "c++" + + include_pybind11 = kwargs.pop("include_pybind11", True) + + # Can't use super here because distutils has old-style classes in + # Python 2! + _Extension.__init__(self, *args, **kwargs) + + # Include the installed package pybind11 headers + if include_pybind11: + # If using setup_requires, this fails the first time - that's okay + try: + import pybind11 + + pyinc = pybind11.get_include() + + if pyinc not in self.include_dirs: + self.include_dirs.append(pyinc) + except ImportError: + pass + + # Have to use the accessor manually to support Python 2 distutils + Pybind11Extension.cxx_std.__set__(self, cxx_std) + + cflags = [] + ldflags = [] + if WIN: + cflags += ["/EHsc", "/bigobj"] + else: + cflags += ["-fvisibility=hidden", "-g0"] + if MACOS: + cflags += ["-stdlib=libc++"] + ldflags += ["-stdlib=libc++"] + self._add_cflags(cflags) + self._add_ldflags(ldflags) + + @property + def cxx_std(self): + """ + The CXX standard level. If set, will add the required flags. If left + at 0, it will trigger an automatic search when pybind11's build_ext + is used. If None, will have no effect. Besides just the flags, this + may add a register warning/error fix for Python 2 or macos-min 10.9 + or 10.14. + """ + return self._cxx_level + + @cxx_std.setter + def cxx_std(self, level): + + if self._cxx_level: + warnings.warn("You cannot safely change the cxx_level after setting it!") + + # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so + # force a valid flag here. + if WIN and level == 11: + level = 14 + + self._cxx_level = level + + if not level: + return + + cflags = [STD_TMPL.format(level)] + ldflags = [] + + if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ: + # C++17 requires a higher min version of macOS. An earlier version + # (10.12 or 10.13) can be set manually via environment variable if + # you are careful in your feature usage, but 10.14 is the safest + # setting for general use. However, never set higher than the + # current macOS version! + current_macos = tuple(int(x) for x in platform.mac_ver()[0].split(".")[:2]) + desired_macos = (10, 9) if level < 17 else (10, 14) + macos_string = ".".join(str(x) for x in min(current_macos, desired_macos)) + macosx_min = "-mmacosx-version-min=" + macos_string + cflags += [macosx_min] + ldflags += [macosx_min] + + if PY2: + if WIN: + # Will be ignored on MSVC 2015, where C++17 is not supported so + # this flag is not valid. + cflags += ["/wd5033"] + elif level >= 17: + cflags += ["-Wno-register"] + elif level >= 14: + cflags += ["-Wno-deprecated-register"] + + self._add_cflags(cflags) + self._add_ldflags(ldflags) + + +# Just in case someone clever tries to multithread +tmp_chdir_lock = threading.Lock() +cpp_cache_lock = threading.Lock() + + +@contextlib.contextmanager +def tmp_chdir(): + "Prepare and enter a temporary directory, cleanup when done" + + # Threadsafe + with tmp_chdir_lock: + olddir = os.getcwd() + try: + tmpdir = tempfile.mkdtemp() + os.chdir(tmpdir) + yield tmpdir + finally: + os.chdir(olddir) + shutil.rmtree(tmpdir) + + +# cf http://bugs.python.org/issue26689 +def has_flag(compiler, flag): + """ + Return the flag if a flag name is supported on the + specified compiler, otherwise None (can be used as a boolean). + If multiple flags are passed, return the first that matches. + """ + + with tmp_chdir(): + fname = "flagcheck.cpp" + with open(fname, "w") as f: + # Don't trigger -Wunused-parameter. + f.write("int main (int, char **) { return 0; }") + + try: + compiler.compile([fname], extra_postargs=[flag]) + except distutils.errors.CompileError: + return False + return True + + +# Every call will cache the result +cpp_flag_cache = None + + +def auto_cpp_level(compiler): + """ + Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows. + """ + + if WIN: + return "latest" + + global cpp_flag_cache + + # If this has been previously calculated with the same args, return that + with cpp_cache_lock: + if cpp_flag_cache: + return cpp_flag_cache + + levels = [17, 14, 11] + + for level in levels: + if has_flag(compiler, STD_TMPL.format(level)): + with cpp_cache_lock: + cpp_flag_cache = level + return level + + msg = "Unsupported compiler -- at least C++11 support is needed!" + raise RuntimeError(msg) + + +class build_ext(_build_ext): # noqa: N801 + """ + Customized build_ext that allows an auto-search for the highest supported + C++ level for Pybind11Extension. This is only needed for the auto-search + for now, and is completely optional otherwise. + """ + + def build_extensions(self): + """ + Build extensions, injecting C++ std for Pybind11Extension if needed. + """ + + for ext in self.extensions: + if hasattr(ext, "_cxx_level") and ext._cxx_level == 0: + # Python 2 syntax - old-style distutils class + ext.__class__.cxx_std.__set__(ext, auto_cpp_level(self.compiler)) + + # Python 2 doesn't allow super here, since distutils uses old-style + # classes! + _build_ext.build_extensions(self) + + +def naive_recompile(obj, src): + """ + This will recompile only if the source file changes. It does not check + header files, so a more advanced function or Ccache is better if you have + editable header files in your package. + """ + return os.stat(obj).st_mtime < os.stat(src).st_mtime + + +def no_recompile(obg, src): + """ + This is the safest but slowest choice (and is the default) - will always + recompile sources. + """ + return True + + +# Optional parallel compile utility +# inspired by: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils +# and: https://github.com/tbenthompson/cppimport/blob/stable/cppimport/build_module.py +# and NumPy's parallel distutils module: +# https://github.com/numpy/numpy/blob/master/numpy/distutils/ccompiler.py +class ParallelCompile(object): + """ + Make a parallel compile function. Inspired by + numpy.distutils.ccompiler.CCompiler_compile and cppimport. + + This takes several arguments that allow you to customize the compile + function created: + + envvar: + Set an environment variable to control the compilation threads, like + NPY_NUM_BUILD_JOBS + default: + 0 will automatically multithread, or 1 will only multithread if the + envvar is set. + max: + The limit for automatic multithreading if non-zero + needs_recompile: + A function of (obj, src) that returns True when recompile is needed. No + effect in isolated mode; use ccache instead, see + https://github.com/matplotlib/matplotlib/issues/1507/ + + To use:: + + ParallelCompile("NPY_NUM_BUILD_JOBS").install() + + or:: + + with ParallelCompile("NPY_NUM_BUILD_JOBS"): + setup(...) + + By default, this assumes all files need to be recompiled. A smarter + function can be provided via needs_recompile. If the output has not yet + been generated, the compile will always run, and this function is not + called. + """ + + __slots__ = ("envvar", "default", "max", "_old", "needs_recompile") + + def __init__(self, envvar=None, default=0, max=0, needs_recompile=no_recompile): + self.envvar = envvar + self.default = default + self.max = max + self.needs_recompile = needs_recompile + self._old = [] + + def function(self): + """ + Builds a function object usable as distutils.ccompiler.CCompiler.compile. + """ + + def compile_function( + compiler, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): + + # These lines are directly from distutils.ccompiler.CCompiler + macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs + ) + cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) + + # The number of threads; start with default. + threads = self.default + + # Determine the number of compilation threads, unless set by an environment variable. + if self.envvar is not None: + threads = int(os.environ.get(self.envvar, self.default)) + + def _single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + + if not os.path.exists(obj) or self.needs_recompile(obj, src): + compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + try: + import multiprocessing + from multiprocessing.pool import ThreadPool + except ImportError: + threads = 1 + + if threads == 0: + try: + threads = multiprocessing.cpu_count() + threads = self.max if self.max and self.max < threads else threads + except NotImplementedError: + threads = 1 + + if threads > 1: + for _ in ThreadPool(threads).imap_unordered(_single_compile, objects): + pass + else: + for ob in objects: + _single_compile(ob) + + return objects + + return compile_function + + def install(self): + distutils.ccompiler.CCompiler.compile = self.function() + return self + + def __enter__(self): + self._old.append(distutils.ccompiler.CCompiler.compile) + return self.install() + + def __exit__(self, *args): + distutils.ccompiler.CCompiler.compile = self._old.pop() diff --git a/pybind11/setup_helpers.pyi b/pybind11/setup_helpers.pyi new file mode 100644 index 0000000000..23232e1fda --- /dev/null +++ b/pybind11/setup_helpers.pyi @@ -0,0 +1,61 @@ +# IMPORTANT: Should stay in sync with setup_helpers.py (mostly checked by CI / +# pre-commit). + +from typing import Any, Callable, Iterator, Optional, Type, TypeVar, Union +from types import TracebackType + +from distutils.command.build_ext import build_ext as _build_ext # type: ignore +from distutils.extension import Extension as _Extension +import distutils.ccompiler +import contextlib + +WIN: bool +PY2: bool +MACOS: bool +STD_TMPL: str + +class Pybind11Extension(_Extension): + def _add_cflags(self, *flags: str) -> None: ... + def _add_lflags(self, *flags: str) -> None: ... + def __init__( + self, *args: Any, cxx_std: int = 0, language: str = "c++", **kwargs: Any + ) -> None: ... + @property + def cxx_std(self) -> int: ... + @cxx_std.setter + def cxx_std(self, level: int) -> None: ... + +@contextlib.contextmanager +def tmp_chdir() -> Iterator[str]: ... +def has_flag(compiler: distutils.ccompiler.CCompiler, flag: str) -> bool: ... +def auto_cpp_level(compiler: distutils.ccompiler.CCompiler) -> Union[int, str]: ... + +class build_ext(_build_ext): # type: ignore + def build_extensions(self) -> None: ... + +def no_recompile(obj: str, src: str) -> bool: ... +def naive_recompile(obj: str, src: str) -> bool: ... + +T = TypeVar("T", bound="ParallelCompile") + +class ParallelCompile: + envvar: Optional[str] + default: int + max: int + needs_recompile: Callable[[str, str], bool] + def __init__( + self, + envvar: Optional[str] = None, + default: int = 0, + max: int = 0, + needs_recompile: Callable[[str, str], bool] = no_recompile, + ) -> None: ... + def function(self) -> Any: ... + def install(self: T) -> T: ... + def __enter__(self: T) -> T: ... + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: ... diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..5c9d153541 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "cmake>=3.18", "ninja"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 002f38d10e..041e62dae6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,54 @@ +[metadata] +long_description = file: README.rst +long_description_content_type = text/x-rst +description = Seamless operability between C++11 and Python +author = Wenzel Jakob +author_email = wenzel.jakob@epfl.ch +url = https://github.com/pybind/pybind11 +license = BSD + +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + Programming Language :: C++ + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + License :: OSI Approved :: BSD License + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Python :: Implementation :: CPython + Programming Language :: C++ + Topic :: Software Development :: Libraries :: Python Modules + +keywords = + C++11 + Python bindings + +[options] +python_requires = >=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4 +zip_safe = False + [bdist_wheel] universal=1 +[check-manifest] +ignore = + tests/** + docs/** + tools/** + include/** + .* + pybind11/include/** + pybind11/share/** + CMakeLists.txt + + [flake8] max-line-length = 99 show_source = True @@ -10,3 +58,28 @@ ignore = E201, E241, W504, # camelcase 'cPickle' imported as lowercase 'pickle' N813 + # Black conflict + W503, E203 + +[mypy] +files = pybind11 +python_version = 2.7 +warn_unused_configs = True + +# Currently (0.800) identical to --strict +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +no_implicit_optional = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_return_any = True +no_implicit_reexport = True +strict_equality = True + +[tool:pytest] +timeout = 300 diff --git a/setup.py b/setup.py index 577a6b6c37..3a0327982a 100644 --- a/setup.py +++ b/setup.py @@ -3,128 +3,113 @@ # Setup script for PyPI; use CMakeFile.txt to build extension modules -from setuptools import setup -from distutils.command.install_headers import install_headers -from distutils.command.build_py import build_py -from pybind11 import __version__ +import contextlib import os +import re +import shutil +import string +import subprocess +import sys +import tempfile -package_data = [ - 'include/pybind11/detail/class.h', - 'include/pybind11/detail/common.h', - 'include/pybind11/detail/descr.h', - 'include/pybind11/detail/init.h', - 'include/pybind11/detail/internals.h', - 'include/pybind11/detail/typeid.h', - 'include/pybind11/attr.h', - 'include/pybind11/buffer_info.h', - 'include/pybind11/cast.h', - 'include/pybind11/chrono.h', - 'include/pybind11/common.h', - 'include/pybind11/complex.h', - 'include/pybind11/eigen.h', - 'include/pybind11/embed.h', - 'include/pybind11/eval.h', - 'include/pybind11/functional.h', - 'include/pybind11/iostream.h', - 'include/pybind11/numpy.h', - 'include/pybind11/operators.h', - 'include/pybind11/options.h', - 'include/pybind11/pybind11.h', - 'include/pybind11/pytypes.h', - 'include/pybind11/stl.h', - 'include/pybind11/stl_bind.h', -] - -# Prevent installation of pybind11 headers by setting -# PYBIND11_USE_CMAKE. -if os.environ.get('PYBIND11_USE_CMAKE'): - headers = [] -else: - headers = package_data - - -class InstallHeaders(install_headers): - """Use custom header installer because the default one flattens subdirectories""" - def run(self): - if not self.distribution.headers: - return - - for header in self.distribution.headers: - subdir = os.path.dirname(os.path.relpath(header, 'include/pybind11')) - install_dir = os.path.join(self.install_dir, subdir) - self.mkpath(install_dir) - - (out, _) = self.copy_file(header, install_dir) - self.outfiles.append(out) - - -# Install the headers inside the package as well -class BuildPy(build_py): - def build_package_data(self): - build_py.build_package_data(self) - for header in package_data: - target = os.path.join(self.build_lib, 'pybind11', header) - self.mkpath(os.path.dirname(target)) - self.copy_file(header, target, preserve_mode=False) - - def get_outputs(self, include_bytecode=1): - outputs = build_py.get_outputs(self, include_bytecode=include_bytecode) - for header in package_data: - target = os.path.join(self.build_lib, 'pybind11', header) - outputs.append(target) - return outputs - - -setup( - name='pybind11', - version=__version__, - description='Seamless operability between C++11 and Python', - author='Wenzel Jakob', - author_email='wenzel.jakob@epfl.ch', - url='https://github.com/pybind/pybind11', - download_url='https://github.com/pybind/pybind11/tarball/v' + __version__, - packages=['pybind11'], - license='BSD', - headers=headers, - zip_safe=False, - cmdclass=dict(install_headers=InstallHeaders, build_py=BuildPy), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Utilities', - 'Programming Language :: C++', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'License :: OSI Approved :: BSD License' - ], - keywords='C++11, Python bindings', - long_description="""pybind11 is a lightweight header-only library that -exposes C++ types in Python and vice versa, mainly to create Python bindings of -existing C++ code. Its goals and syntax are similar to the excellent -Boost.Python by David Abrahams: to minimize boilerplate code in traditional -extension modules by inferring type information using compile-time -introspection. - -The main issue with Boost.Python-and the reason for creating such a similar -project-is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. - -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations.""") +import setuptools.command.sdist + +DIR = os.path.abspath(os.path.dirname(__file__)) +VERSION_REGEX = re.compile( + r"^\s*#\s*define\s+PYBIND11_VERSION_([A-Z]+)\s+(.*)$", re.MULTILINE +) + +# PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers +# files, and the sys.prefix files (CMake and headers). + +global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) + +setup_py = "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" +extra_cmd = 'cmdclass["sdist"] = SDist\n' + +to_src = ( + ("pyproject.toml", "tools/pyproject.toml"), + ("setup.py", setup_py), +) + +# Read the listed version +with open("pybind11/_version.py") as f: + code = compile(f.read(), "pybind11/_version.py", "exec") +loc = {} +exec(code, loc) +version = loc["__version__"] + +# Verify that the version matches the one in C++ +with open("include/pybind11/detail/common.h") as f: + matches = dict(VERSION_REGEX.findall(f.read())) +cpp_version = "{MAJOR}.{MINOR}.{PATCH}".format(**matches) +if version != cpp_version: + msg = "Python version {} does not match C++ version {}!".format( + version, cpp_version + ) + raise RuntimeError(msg) + + +def get_and_replace(filename, binary=False, **opts): + with open(filename, "rb" if binary else "r") as f: + contents = f.read() + # Replacement has to be done on text in Python 3 (both work in Python 2) + if binary: + return string.Template(contents.decode()).substitute(opts).encode() + else: + return string.Template(contents).substitute(opts) + + +# Use our input files instead when making the SDist (and anything that depends +# on it, like a wheel) +class SDist(setuptools.command.sdist.sdist): + def make_release_tree(self, base_dir, files): + setuptools.command.sdist.sdist.make_release_tree(self, base_dir, files) + + for to, src in to_src: + txt = get_and_replace(src, binary=True, version=version, extra_cmd="") + + dest = os.path.join(base_dir, to) + + # This is normally linked, so unlink before writing! + os.unlink(dest) + with open(dest, "wb") as f: + f.write(txt) + + +# Backport from Python 3 +@contextlib.contextmanager +def TemporaryDirectory(): # noqa: N802 + "Prepare a temporary directory, cleanup when done" + try: + tmpdir = tempfile.mkdtemp() + yield tmpdir + finally: + shutil.rmtree(tmpdir) + + +# Remove the CMake install directory when done +@contextlib.contextmanager +def remove_output(*sources): + try: + yield + finally: + for src in sources: + shutil.rmtree(src) + + +with remove_output("pybind11/include", "pybind11/share"): + # Generate the files if they are not present. + with TemporaryDirectory() as tmpdir: + cmd = ["cmake", "-S", ".", "-B", tmpdir] + [ + "-DCMAKE_INSTALL_PREFIX=pybind11", + "-DBUILD_TESTING=OFF", + "-DPYBIND11_NOPYTHON=ON", + ] + cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call(cmd, **cmake_opts) + subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) + + txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) + code = compile(txt, setup_py, "exec") + exec(code, {"SDist": SDist}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a7a9d94078..5f5617fa89 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,24 +5,69 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.4) -# VERSION 3.7...3.18, but some versions of VS have a patched CMake 3.11 -# that do not work properly with this syntax, so using the following workaround: +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: if(${CMAKE_VERSION} VERSION_LESS 3.18) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() cmake_policy(VERSION 3.18) endif() +# Only needed for CMake < 3.5 support +include(CMakeParseArguments) + +# Filter out items; print an optional message if any items filtered +# +# Usage: +# pybind11_filter_tests(LISTNAME file1.cpp file2.cpp ... MESSAGE "") +# +macro(pybind11_filter_tests LISTNAME) + cmake_parse_arguments(ARG "" "MESSAGE" "" ${ARGN}) + set(PYBIND11_FILTER_TESTS_FOUND OFF) + foreach(filename IN LISTS ARG_UNPARSED_ARGUMENTS) + list(FIND ${LISTNAME} ${filename} _FILE_FOUND) + if(_FILE_FOUND GREATER -1) + list(REMOVE_AT ${LISTNAME} ${_FILE_FOUND}) + set(PYBIND11_FILTER_TESTS_FOUND ON) + endif() + endforeach() + if(PYBIND11_FILTER_TESTS_FOUND AND ARG_MESSAGE) + message(STATUS "${ARG_MESSAGE}") + endif() +endmacro() + +macro(possibly_uninitialized) + foreach(VARNAME ${ARGN}) + if(NOT DEFINED "${VARNAME}") + set("${VARNAME}" "") + endif() + endforeach() +endmacro() + +# New Python support +if(DEFINED Python_EXECUTABLE) + set(PYTHON_EXECUTABLE "${Python_EXECUTABLE}") + set(PYTHON_VERSION "${Python_VERSION}") +endif() + # There's no harm in including a project in a project project(pybind11_tests CXX) +# Access FindCatch and more +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/../tools") + option(PYBIND11_WERROR "Report all warnings as errors" OFF) option(DOWNLOAD_EIGEN "Download EIGEN (requires CMake 3.11+)" OFF) +option(PYBIND11_CUDA_TESTS "Enable building CUDA tests (requires CMake 3.12+)" OFF) set(PYBIND11_TEST_OVERRIDE "" CACHE STRING "Tests from ;-separated list of *.cpp files will be built instead of all tests") +set(PYBIND11_TEST_FILTER + "" + CACHE STRING "Tests from ;-separated list of *.cpp files will be removed from all tests") if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) # We're being loaded directly, i.e. not via add_subdirectory, so make this @@ -30,7 +75,7 @@ if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) find_package(pybind11 REQUIRED CONFIG) endif() -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) +if(NOT CMAKE_BUILD_TYPE AND NOT DEFINED CMAKE_CONFIGURATION_TYPES) message(STATUS "Setting tests build type to MinSizeRel as none was specified") set(CMAKE_BUILD_TYPE MinSizeRel @@ -39,6 +84,14 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) "RelWithDebInfo") endif() +if(PYBIND11_CUDA_TESTS) + enable_language(CUDA) + if(DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) + endif() + set(CMAKE_CUDA_STANDARD_REQUIRED ON) +endif() + # Full set of test files (you can override these; see below) set(PYBIND11_TEST_FILES test_async.cpp @@ -81,22 +134,30 @@ set(PYBIND11_TEST_FILES test_virtual_functions.cpp) # Invoking cmake with something like: -# cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_picking.cpp" .. +# cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp" .. # lets you override the tests that get compiled and run. You can restore to all tests with: # cmake -DPYBIND11_TEST_OVERRIDE= .. if(PYBIND11_TEST_OVERRIDE) set(PYBIND11_TEST_FILES ${PYBIND11_TEST_OVERRIDE}) endif() -# Skip test_async for Python < 3.5 -list(FIND PYBIND11_TEST_FILES test_async.cpp PYBIND11_TEST_FILES_ASYNC_I) -if((PYBIND11_TEST_FILES_ASYNC_I GREATER -1) AND ("${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}" - VERSION_LESS 3.5)) - message( - STATUS - "Skipping test_async because Python version ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} < 3.5" - ) - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_ASYNC_I}) +# You can also filter tests: +if(PYBIND11_TEST_FILTER) + pybind11_filter_tests(PYBIND11_TEST_FILES ${PYBIND11_TEST_FILTER}) +endif() + +if(PYTHON_VERSION VERSION_LESS 3.5) + pybind11_filter_tests(PYBIND11_TEST_FILES test_async.cpp MESSAGE + "Skipping test_async on Python 2") +endif() + +# Skip tests for CUDA check: +# /pybind11/tests/test_constants_and_functions.cpp(125): +# error: incompatible exception specifications +if(PYBIND11_CUDA_TESTS) + pybind11_filter_tests( + PYBIND11_TEST_FILES test_constants_and_functions.cpp MESSAGE + "Skipping test_constants_and_functions due to incompatible exception specifications") endif() string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}") @@ -122,7 +183,7 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) message(FATAL_ERROR "CMake 3.11+ required when using DOWNLOAD_EIGEN") endif() - set(EIGEN3_VERSION_STRING "3.3.7") + set(EIGEN3_VERSION_STRING "3.3.8") include(FetchContent) FetchContent_Declare( @@ -138,13 +199,9 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR}) set(EIGEN3_FOUND TRUE) + else() find_package(Eigen3 3.2.7 QUIET CONFIG) - if(EIGEN3_FOUND) - if(EIGEN3_VERSION_STRING AND NOT EIGEN3_VERSION_STRING VERSION_LESS 3.3.1) - set(PYBIND11_EIGEN_VIA_TARGET TRUE) - endif() - endif() if(NOT EIGEN3_FOUND) # Couldn't load via target, so fall back to allowing module mode finding, which will pick up @@ -154,6 +211,12 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) endif() if(EIGEN3_FOUND) + if(NOT TARGET Eigen3::Eigen) + add_library(Eigen3::Eigen IMPORTED INTERFACE) + set_property(TARGET Eigen3::Eigen PROPERTY INTERFACE_INCLUDE_DIRECTORIES + "${EIGEN3_INCLUDE_DIR}") + endif() + # Eigen 3.3.1+ cmake sets EIGEN3_VERSION_STRING (and hard codes the version when installed # rather than looking it up in the cmake script); older versions, and the # tools/FindEigen3.cmake, set EIGEN3_VERSION instead. @@ -163,25 +226,49 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") else() list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) - message(STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN on CMake 3.11+ to download") + message( + STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") endif() endif() # Optional dependency for some tests (boost::variant is only supported with version >= 1.56) find_package(Boost 1.56) +if(Boost_FOUND) + if(NOT TARGET Boost::headers) + add_library(Boost::headers IMPORTED INTERFACE) + if(TARGET Boost::boost) + # Classic FindBoost + set_property(TARGET Boost::boost PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost) + else() + # Very old FindBoost, or newer Boost than CMake in older CMakes + set_property(TARGET Boost::headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES + ${Boost_INCLUDE_DIRS}) + endif() + endif() +endif() + # Compile with compiler warnings turned on function(pybind11_enable_warnings target_name) if(MSVC) target_compile_options(${target_name} PRIVATE /W4) - elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)") - target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion -Wcast-qual - -Wdeprecated) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)" AND NOT PYBIND11_CUDA_TESTS) + target_compile_options( + ${target_name} + PRIVATE -Wall + -Wextra + -Wconversion + -Wcast-qual + -Wdeprecated + -Wundef + -Wnon-virtual-dtor) endif() if(PYBIND11_WERROR) if(MSVC) target_compile_options(${target_name} PRIVATE /WX) + elseif(PYBIND11_CUDA_TESTS) + target_compile_options(${target_name} PRIVATE "SHELL:-Werror all-warnings") elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)") target_compile_options(${target_name} PRIVATE -Werror) endif() @@ -218,70 +305,95 @@ foreach(t ${PYBIND11_CROSS_MODULE_GIL_TESTS}) endif() endforeach() -set(testdir ${CMAKE_CURRENT_SOURCE_DIR}) +# Support CUDA testing by forcing the target file to compile with NVCC +if(PYBIND11_CUDA_TESTS) + set_property(SOURCE ${PYBIND11_TEST_FILES} PROPERTY LANGUAGE CUDA) +endif() + foreach(target ${test_targets}) set(test_files ${PYBIND11_TEST_FILES}) if(NOT "${target}" STREQUAL "pybind11_tests") set(test_files "") endif() + # Support CUDA testing by forcing the target file to compile with NVCC + if(PYBIND11_CUDA_TESTS) + set_property(SOURCE ${target}.cpp PROPERTY LANGUAGE CUDA) + endif() + # Create the binding library pybind11_add_module(${target} THIN_LTO ${target}.cpp ${test_files} ${PYBIND11_HEADERS}) pybind11_enable_warnings(${target}) + if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + get_property( + suffix + TARGET ${target} + PROPERTY SUFFIX) + set(source_output "${CMAKE_CURRENT_SOURCE_DIR}/${target}${suffix}") + if(suffix AND EXISTS "${source_output}") + message(WARNING "Output file also in source directory; " + "please remove to avoid confusion: ${source_output}") + endif() + endif() + if(MSVC) target_compile_options(${target} PRIVATE /utf-8) endif() if(EIGEN3_FOUND) - if(PYBIND11_EIGEN_VIA_TARGET) - target_link_libraries(${target} PRIVATE Eigen3::Eigen) - else() - target_include_directories(${target} PRIVATE ${EIGEN3_INCLUDE_DIR}) - endif() + target_link_libraries(${target} PRIVATE Eigen3::Eigen) target_compile_definitions(${target} PRIVATE -DPYBIND11_TEST_EIGEN) endif() if(Boost_FOUND) - target_include_directories(${target} PRIVATE ${Boost_INCLUDE_DIRS}) + target_link_libraries(${target} PRIVATE Boost::headers) target_compile_definitions(${target} PRIVATE -DPYBIND11_TEST_BOOST) endif() # Always write the output file directly into the 'tests' directory (even on MSVC) if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) - set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${testdir}") - foreach(config ${CMAKE_CONFIGURATION_TYPES}) - string(TOUPPER ${config} config) - set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} "${testdir}") - endforeach() + set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY + "${CMAKE_CURRENT_BINARY_DIR}") + + if(DEFINED CMAKE_CONFIGURATION_TYPES) + foreach(config ${CMAKE_CONFIGURATION_TYPES}) + string(TOUPPER ${config} config) + set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} + "${CMAKE_CURRENT_BINARY_DIR}") + endforeach() + endif() endif() endforeach() -# Make sure pytest is found or produce a fatal error -if(NOT PYBIND11_PYTEST_FOUND) - execute_process( - COMMAND ${PYTHON_EXECUTABLE} -c "import pytest; print(pytest.__version__)" - RESULT_VARIABLE pytest_not_found - OUTPUT_VARIABLE pytest_version - ERROR_QUIET) - if(pytest_not_found) - message(FATAL_ERROR "Running the tests requires pytest. Please install it manually" - " (try: ${PYTHON_EXECUTABLE} -m pip install pytest)") - elseif(pytest_version VERSION_LESS 3.0) - message(FATAL_ERROR "Running the tests requires pytest >= 3.0. Found: ${pytest_version}" - "Please update it (try: ${PYTHON_EXECUTABLE} -m pip install -U pytest)") - endif() - set(PYBIND11_PYTEST_FOUND - TRUE - CACHE INTERNAL "") +# Make sure pytest is found or produce a warning +pybind11_find_import(pytest VERSION 3.1) + +if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + # This is not used later in the build, so it's okay to regenerate each time. + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/pytest.ini" "${CMAKE_CURRENT_BINARY_DIR}/pytest.ini" + COPYONLY) + file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/pytest.ini" + "\ntestpaths = \"${CMAKE_CURRENT_SOURCE_DIR}\"") + endif() +# cmake 3.12 added list(transform prepend +# but we can't use it yet +string(REPLACE "test_" "${CMAKE_CURRENT_SOURCE_DIR}/test_" PYBIND11_ABS_PYTEST_FILES + "${PYBIND11_PYTEST_FILES}") + +set(PYBIND11_TEST_PREFIX_COMMAND + "" + CACHE STRING "Put this before pytest, use for checkers and such") + # A single command to compile and run the tests add_custom_target( pytest - COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_PYTEST_FILES} + COMMAND ${PYBIND11_TEST_PREFIX_COMMAND} ${PYTHON_EXECUTABLE} -m pytest + ${PYBIND11_ABS_PYTEST_FILES} DEPENDS ${test_targets} - WORKING_DIRECTORY ${testdir} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" USES_TERMINAL) if(PYBIND11_TEST_OVERRIDE) @@ -292,6 +404,27 @@ if(PYBIND11_TEST_OVERRIDE) "Note: not all tests run: -DPYBIND11_TEST_OVERRIDE is in effect") endif() +# cmake-format: off +add_custom_target( + memcheck + COMMAND + PYTHONMALLOC=malloc + valgrind + --leak-check=full + --show-leak-kinds=definite,indirect + --errors-for-leak-kinds=definite,indirect + --error-exitcode=1 + --read-var-info=yes + --track-origins=yes + --suppressions="${CMAKE_CURRENT_SOURCE_DIR}/valgrind-python.supp" + --suppressions="${CMAKE_CURRENT_SOURCE_DIR}/valgrind-numpy-scipy.supp" + --gen-suppressions=all + ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_ABS_PYTEST_FILES} + DEPENDS ${test_targets} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + USES_TERMINAL) +# cmake-format: on + # Add a check target to run all the tests, starting with pytest (we add dependencies to this below) add_custom_target(check DEPENDS pytest) @@ -310,8 +443,10 @@ add_custom_command( $ ${CMAKE_CURRENT_BINARY_DIR}/sosize-$.txt) -# Test embedding the interpreter. Provides the `cpptest` target. -add_subdirectory(test_embed) +if(NOT PYBIND11_CUDA_TESTS) + # Test embedding the interpreter. Provides the `cpptest` target. + add_subdirectory(test_embed) -# Test CMake build using functions and targets from subdirectory or installed location -add_subdirectory(test_cmake_build) + # Test CMake build using functions and targets from subdirectory or installed location + add_subdirectory(test_cmake_build) +endif() diff --git a/tests/conftest.py b/tests/conftest.py index d317c49dbc..362eb80691 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,28 +5,32 @@ Adds docstring and exceptions message sanitizers: ignore Python 2 vs 3 differences. """ -import pytest -import textwrap -import difflib -import re -import sys import contextlib -import platform +import difflib import gc +import re +import textwrap + +import pytest -_unicode_marker = re.compile(r'u(\'[^\']*\')') -_long_marker = re.compile(r'([0-9])L') -_hexadecimal = re.compile(r'0x[0-9a-fA-F]+') +import env -# test_async.py requires support for async and await +# Early diagnostic for failed imports +import pybind11_tests # noqa: F401 + +_unicode_marker = re.compile(r"u(\'[^\']*\')") +_long_marker = re.compile(r"([0-9])L") +_hexadecimal = re.compile(r"0x[0-9a-fA-F]+") + +# Avoid collecting Python3 only files collect_ignore = [] -if sys.version_info[:2] < (3, 5): +if env.PY2: collect_ignore.append("test_async.py") def _strip_and_dedent(s): """For triple-quote strings""" - return textwrap.dedent(s.lstrip('\n').rstrip()) + return textwrap.dedent(s.lstrip("\n").rstrip()) def _split_and_sort(s): @@ -36,11 +40,14 @@ def _split_and_sort(s): def _make_explanation(a, b): """Explanation for a failed assert -- the a and b arguments are List[str]""" - return ["--- actual / +++ expected"] + [line.strip('\n') for line in difflib.ndiff(a, b)] + return ["--- actual / +++ expected"] + [ + line.strip("\n") for line in difflib.ndiff(a, b) + ] class Output(object): """Basic output post-processing and comparison""" + def __init__(self, string): self.string = string self.explanation = [] @@ -50,7 +57,11 @@ def __str__(self): def __eq__(self, other): # Ignore constructor/destructor output which is prefixed with "###" - a = [line for line in self.string.strip().splitlines() if not line.startswith("###")] + a = [ + line + for line in self.string.strip().splitlines() + if not line.startswith("###") + ] b = _strip_and_dedent(other).splitlines() if a == b: return True @@ -61,6 +72,7 @@ def __eq__(self, other): class Unordered(Output): """Custom comparison for output without strict line ordering""" + def __eq__(self, other): a = _split_and_sort(self.string) b = _split_and_sort(other) @@ -171,7 +183,7 @@ def msg(): # noinspection PyUnusedLocal def pytest_assertrepr_compare(op, left, right): """Hook to insert custom failure explanation""" - if hasattr(left, 'explanation'): + if hasattr(left, "explanation"): return left.explanation @@ -185,66 +197,12 @@ def suppress(exception): def gc_collect(): - ''' Run the garbage collector twice (needed when running - reference counting tests with PyPy) ''' + """Run the garbage collector twice (needed when running + reference counting tests with PyPy)""" gc.collect() gc.collect() def pytest_configure(): - """Add import suppression and test requirements to `pytest` namespace""" - try: - import numpy as np - except ImportError: - np = None - try: - import scipy - except ImportError: - scipy = None - try: - from pybind11_tests.eigen import have_eigen - except ImportError: - have_eigen = False - pypy = platform.python_implementation() == "PyPy" - - skipif = pytest.mark.skipif pytest.suppress = suppress - pytest.requires_numpy = skipif(not np, reason="numpy is not installed") - pytest.requires_scipy = skipif(not np, reason="scipy is not installed") - pytest.requires_eigen_and_numpy = skipif(not have_eigen or not np, - reason="eigen and/or numpy are not installed") - pytest.requires_eigen_and_scipy = skipif( - not have_eigen or not scipy, reason="eigen and/or scipy are not installed") - pytest.unsupported_on_pypy = skipif(pypy, reason="unsupported on PyPy") - pytest.bug_in_pypy = pytest.mark.xfail(pypy, reason="bug in PyPy") - pytest.unsupported_on_pypy3 = skipif(pypy and sys.version_info.major >= 3, - reason="unsupported on PyPy3") - pytest.unsupported_on_pypy_lt_6 = skipif(pypy and sys.pypy_version_info[0] < 6, - reason="unsupported on PyPy<6") - pytest.unsupported_on_py2 = skipif(sys.version_info.major < 3, - reason="unsupported on Python 2.x") pytest.gc_collect = gc_collect - - -def _test_import_pybind11(): - """Early diagnostic for test module initialization errors - - When there is an error during initialization, the first import will report the - real error while all subsequent imports will report nonsense. This import test - is done early (in the pytest configuration file, before any tests) in order to - avoid the noise of having all tests fail with identical error messages. - - Any possible exception is caught here and reported manually *without* the stack - trace. This further reduces noise since the trace would only show pytest internals - which are not useful for debugging pybind11 module issues. - """ - # noinspection PyBroadException - try: - import pybind11_tests # noqa: F401 imported but unused - except Exception as e: - print("Failed to import pybind11_tests from pytest:") - print(" {}: {}".format(type(e).__name__, e)) - sys.exit(1) - - -_test_import_pybind11() diff --git a/tests/constructor_stats.h b/tests/constructor_stats.h index abfaf91614..805968a09b 100644 --- a/tests/constructor_stats.h +++ b/tests/constructor_stats.h @@ -120,7 +120,7 @@ class ConstructorStats { throw py::error_already_set(); Py_DECREF(result); #else - py::module::import("gc").attr("collect")(); + py::module_::import("gc").attr("collect")(); #endif } diff --git a/tests/env.py b/tests/env.py new file mode 100644 index 0000000000..5cded44127 --- /dev/null +++ b/tests/env.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +import platform +import sys + +LINUX = sys.platform.startswith("linux") +MACOS = sys.platform.startswith("darwin") +WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") + +CPYTHON = platform.python_implementation() == "CPython" +PYPY = platform.python_implementation() == "PyPy" + +PY2 = sys.version_info.major == 2 + +PY = sys.version_info diff --git a/tests/extra_python_package/pytest.ini b/tests/extra_python_package/pytest.ini new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py new file mode 100644 index 0000000000..064a3e12fe --- /dev/null +++ b/tests/extra_python_package/test_files.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +import contextlib +import os +import string +import subprocess +import sys +import tarfile +import zipfile + +# These tests must be run explicitly +# They require CMake 3.15+ (--install) + +DIR = os.path.abspath(os.path.dirname(__file__)) +MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) + + +main_headers = { + "include/pybind11/attr.h", + "include/pybind11/buffer_info.h", + "include/pybind11/cast.h", + "include/pybind11/chrono.h", + "include/pybind11/common.h", + "include/pybind11/complex.h", + "include/pybind11/eigen.h", + "include/pybind11/embed.h", + "include/pybind11/eval.h", + "include/pybind11/functional.h", + "include/pybind11/gil.h", + "include/pybind11/iostream.h", + "include/pybind11/numpy.h", + "include/pybind11/operators.h", + "include/pybind11/options.h", + "include/pybind11/pybind11.h", + "include/pybind11/pytypes.h", + "include/pybind11/stl.h", + "include/pybind11/stl_bind.h", +} + +detail_headers = { + "include/pybind11/detail/class.h", + "include/pybind11/detail/common.h", + "include/pybind11/detail/descr.h", + "include/pybind11/detail/init.h", + "include/pybind11/detail/internals.h", + "include/pybind11/detail/type_caster_base.h", + "include/pybind11/detail/typeid.h", +} + +cmake_files = { + "share/cmake/pybind11/FindPythonLibsNew.cmake", + "share/cmake/pybind11/pybind11Common.cmake", + "share/cmake/pybind11/pybind11Config.cmake", + "share/cmake/pybind11/pybind11ConfigVersion.cmake", + "share/cmake/pybind11/pybind11NewTools.cmake", + "share/cmake/pybind11/pybind11Targets.cmake", + "share/cmake/pybind11/pybind11Tools.cmake", +} + +py_files = { + "__init__.py", + "__main__.py", + "_version.py", + "_version.pyi", + "commands.py", + "py.typed", + "setup_helpers.py", + "setup_helpers.pyi", +} + +headers = main_headers | detail_headers +src_files = headers | cmake_files +all_files = src_files | py_files + + +sdist_files = { + "pybind11", + "pybind11/include", + "pybind11/include/pybind11", + "pybind11/include/pybind11/detail", + "pybind11/share", + "pybind11/share/cmake", + "pybind11/share/cmake/pybind11", + "pyproject.toml", + "setup.cfg", + "setup.py", + "LICENSE", + "MANIFEST.in", + "README.rst", + "PKG-INFO", +} + +local_sdist_files = { + ".egg-info", + ".egg-info/PKG-INFO", + ".egg-info/SOURCES.txt", + ".egg-info/dependency_links.txt", + ".egg-info/not-zip-safe", + ".egg-info/top_level.txt", +} + + +def test_build_sdist(monkeypatch, tmpdir): + + monkeypatch.chdir(MAIN_DIR) + + out = subprocess.check_output( + [ + sys.executable, + "setup.py", + "sdist", + "--formats=tar", + "--dist-dir", + str(tmpdir), + ] + ) + if hasattr(out, "decode"): + out = out.decode() + + (sdist,) = tmpdir.visit("*.tar") + + with tarfile.open(str(sdist)) as tar: + start = tar.getnames()[0] + "/" + version = start[9:-1] + simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "setup.py")) + ) as f: + setup_py = f.read() + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pyproject.toml")) + ) as f: + pyproject_toml = f.read() + + files = set("pybind11/{}".format(n) for n in all_files) + files |= sdist_files + files |= set("pybind11{}".format(n) for n in local_sdist_files) + files.add("pybind11.egg-info/entry_points.txt") + files.add("pybind11.egg-info/requires.txt") + assert simpler == files + + with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: + contents = ( + string.Template(f.read().decode()) + .substitute(version=version, extra_cmd="") + .encode() + ) + assert setup_py == contents + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + contents = f.read() + assert pyproject_toml == contents + + +def test_build_global_dist(monkeypatch, tmpdir): + + monkeypatch.chdir(MAIN_DIR) + monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") + + out = subprocess.check_output( + [ + sys.executable, + "setup.py", + "sdist", + "--formats=tar", + "--dist-dir", + str(tmpdir), + ] + ) + if hasattr(out, "decode"): + out = out.decode() + + (sdist,) = tmpdir.visit("*.tar") + + with tarfile.open(str(sdist)) as tar: + start = tar.getnames()[0] + "/" + version = start[16:-1] + simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "setup.py")) + ) as f: + setup_py = f.read() + + with contextlib.closing( + tar.extractfile(tar.getmember(start + "pyproject.toml")) + ) as f: + pyproject_toml = f.read() + + files = set("pybind11/{}".format(n) for n in all_files) + files |= sdist_files + files |= set("pybind11_global{}".format(n) for n in local_sdist_files) + assert simpler == files + + with open(os.path.join(MAIN_DIR, "tools", "setup_global.py.in"), "rb") as f: + contents = ( + string.Template(f.read().decode()) + .substitute(version=version, extra_cmd="") + .encode() + ) + assert setup_py == contents + + with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: + contents = f.read() + assert pyproject_toml == contents + + +def tests_build_wheel(monkeypatch, tmpdir): + monkeypatch.chdir(MAIN_DIR) + + subprocess.check_output( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + ) + + (wheel,) = tmpdir.visit("*.whl") + + files = set("pybind11/{}".format(n) for n in all_files) + files |= { + "dist-info/LICENSE", + "dist-info/METADATA", + "dist-info/RECORD", + "dist-info/WHEEL", + "dist-info/entry_points.txt", + "dist-info/top_level.txt", + } + + with zipfile.ZipFile(str(wheel)) as z: + names = z.namelist() + + trimmed = set(n for n in names if "dist-info" not in n) + trimmed |= set( + "dist-info/{}".format(n.split("/", 1)[-1]) for n in names if "dist-info" in n + ) + assert files == trimmed + + +def tests_build_global_wheel(monkeypatch, tmpdir): + monkeypatch.chdir(MAIN_DIR) + monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") + + subprocess.check_output( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + ) + + (wheel,) = tmpdir.visit("*.whl") + + files = set("data/data/{}".format(n) for n in src_files) + files |= set("data/headers/{}".format(n[8:]) for n in headers) + files |= { + "dist-info/LICENSE", + "dist-info/METADATA", + "dist-info/WHEEL", + "dist-info/top_level.txt", + "dist-info/RECORD", + } + + with zipfile.ZipFile(str(wheel)) as z: + names = z.namelist() + + beginning = names[0].split("/", 1)[0].rsplit(".", 1)[0] + trimmed = set(n[len(beginning) + 1 :] for n in names) + + assert files == trimmed diff --git a/tests/extra_setuptools/pytest.ini b/tests/extra_setuptools/pytest.ini new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py new file mode 100644 index 0000000000..0d8bd0e4d6 --- /dev/null +++ b/tests/extra_setuptools/test_setuphelper.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +import os +import sys +import subprocess +from textwrap import dedent + +import pytest + +DIR = os.path.abspath(os.path.dirname(__file__)) +MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) + + +@pytest.mark.parametrize("parallel", [False, True]) +@pytest.mark.parametrize("std", [11, 0]) +def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + u"""\ + import sys + sys.path.append({MAIN_DIR!r}) + + from setuptools import setup, Extension + from pybind11.setup_helpers import build_ext, Pybind11Extension + + std = {std} + + ext_modules = [ + Pybind11Extension( + "simple_setup", + sorted(["main.cpp"]), + cxx_std=std, + ), + ] + + cmdclass = dict() + if std == 0: + cmdclass["build_ext"] = build_ext + + + parallel = {parallel} + if parallel: + from pybind11.setup_helpers import ParallelCompile + ParallelCompile().install() + + setup( + name="simple_setup_package", + cmdclass=cmdclass, + ext_modules=ext_modules, + ) + """ + ).format(MAIN_DIR=MAIN_DIR, std=std, parallel=parallel), + encoding="ascii", + ) + + (tmpdir / "main.cpp").write_text( + dedent( + u"""\ + #include + + int f(int x) { + return x * 3; + } + PYBIND11_MODULE(simple_setup, m) { + m.def("f", &f); + } + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "--inplace"], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + # Debug helper printout, normally hidden + for item in tmpdir.listdir(): + print(item.basename) + + assert ( + len([f for f in tmpdir.listdir() if f.basename.startswith("simple_setup")]) == 1 + ) + assert len(list(tmpdir.listdir())) == 4 # two files + output + build_dir + + (tmpdir / "test.py").write_text( + dedent( + u"""\ + import simple_setup + assert simple_setup.f(3) == 9 + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) diff --git a/tests/local_bindings.h b/tests/local_bindings.h index b6afb80866..22537b13ad 100644 --- a/tests/local_bindings.h +++ b/tests/local_bindings.h @@ -58,7 +58,7 @@ class Pet { std::string name_; const std::string &name() { return name_; } }; -} +} // namespace pets struct MixGL { int i; MixGL(int i) : i{i} {} }; struct MixGL2 { int i; MixGL2(int i) : i{i} {} }; diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index bc7d2c3e7a..439cd40129 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -26,23 +26,23 @@ productively. Instead, see the "How can I reduce the build time?" question in the "Frequently asked questions" section of the documentation for good practice on splitting binding code over multiple files. */ -std::list> &initializers() { - static std::list> inits; +std::list> &initializers() { + static std::list> inits; return inits; } test_initializer::test_initializer(Initializer init) { - initializers().push_back(init); + initializers().emplace_back(init); } test_initializer::test_initializer(const char *submodule_name, Initializer init) { - initializers().push_back([=](py::module &parent) { + initializers().emplace_back([=](py::module_ &parent) { auto m = parent.def_submodule(submodule_name); init(m); }); } -void bind_ConstructorStats(py::module &m) { +void bind_ConstructorStats(py::module_ &m) { py::class_(m, "ConstructorStats") .def("alive", &ConstructorStats::alive) .def("values", &ConstructorStats::values) @@ -88,6 +88,4 @@ PYBIND11_MODULE(pybind11_tests, m) { for (const auto &initializer : initializers()) initializer(m); - - if (!py::hasattr(m, "have_eigen")) m.attr("have_eigen") = false; } diff --git a/tests/pybind11_tests.h b/tests/pybind11_tests.h index 1e47416270..ccb0529787 100644 --- a/tests/pybind11_tests.h +++ b/tests/pybind11_tests.h @@ -1,5 +1,6 @@ #pragma once #include +#include #if defined(_MSC_VER) && _MSC_VER < 1910 // We get some really long type names here which causes MSVC 2015 to emit warnings @@ -10,7 +11,7 @@ namespace py = pybind11; using namespace pybind11::literals; class test_initializer { - using Initializer = void (*)(py::module &); + using Initializer = void (*)(py::module_ &); public: test_initializer(Initializer init); @@ -18,9 +19,9 @@ class test_initializer { }; #define TEST_SUBMODULE(name, variable) \ - void test_submodule_##name(py::module &); \ + void test_submodule_##name(py::module_ &); \ test_initializer name(#name, test_submodule_##name); \ - void test_submodule_##name(py::module &variable) + void test_submodule_##name(py::module_ &variable) /// Dummy type which is not exported anywhere -- something to trigger a conversion error @@ -50,6 +51,12 @@ class IncType : public UserType { IncType &operator=(IncType &&) = delete; }; +/// A simple union for basic testing +union IntFloat { + int i; + float f; +}; + /// Custom cast-only type that casts to a string "rvalue" or "lvalue" depending on the cast context. /// Used to test recursive casters (e.g. std::tuple, stl containers). struct RValueCaster {}; @@ -63,3 +70,15 @@ template<> class type_caster { }; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) + +template +void ignoreOldStyleInitWarnings(F &&body) { + py::exec(R"( + message = "pybind11-bound class '.+' is using an old-style placement-new '(?:__init__|__setstate__)' which has been deprecated" + + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=message, category=FutureWarning) + body() + )", py::dict(py::arg("body") = py::cpp_function(body))); +} diff --git a/tests/pytest.ini b/tests/pytest.ini index f209964a47..a3871d6c3a 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,6 +1,7 @@ [pytest] -minversion = 3.0 -norecursedirs = test_cmake_build test_embed +minversion = 3.1 +norecursedirs = test_* extra_* +xfail_strict = True addopts = # show summary of skipped tests -rs @@ -9,6 +10,8 @@ addopts = filterwarnings = # make warnings into errors but ignore certain third-party extension issues error + # somehow, some DeprecationWarnings do not get turned into errors + always::DeprecationWarning # importing scipy submodules on some version of Python ignore::ImportWarning # bogus numpy ABI warning (see numpy/#432) diff --git a/tests/requirements.txt b/tests/requirements.txt index c9750e473f..257d8d85fb 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,8 +1,11 @@ --extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/ -numpy==1.16.6; python_version<"3.6" +numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6" -numpy==1.19.1; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version>="3.6" +numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version=="3.6" +numpy==1.20.0; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10" pytest==4.6.9; python_version<"3.5" -pytest==5.4.3; python_version>="3.5" -scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version<"3.6" -scipy==1.5.2; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version>="3.6" and python_version<"3.9" +pytest==6.1.2; python_version=="3.5" +pytest==6.2.1; python_version>="3.6" +pytest-timeout +scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6" +scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10" diff --git a/tests/test_async.cpp b/tests/test_async.cpp index f0ad0d5350..e6e01d72c9 100644 --- a/tests/test_async.cpp +++ b/tests/test_async.cpp @@ -18,7 +18,7 @@ TEST_SUBMODULE(async_module, m) { .def(py::init<>()) .def("__await__", [](const SupportsAsync& self) -> py::object { static_cast(self); - py::object loop = py::module::import("asyncio.events").attr("get_event_loop")(); + py::object loop = py::module_::import("asyncio.events").attr("get_event_loop")(); py::object f = loop.attr("create_future")(); f.attr("set_result")(5); return f.attr("__await__")(); diff --git a/tests/test_async.py b/tests/test_async.py index e9292c9d9c..df4489c499 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -import asyncio import pytest -from pybind11_tests import async_module as m + +asyncio = pytest.importorskip("asyncio") +m = pytest.importorskip("pybind11_tests.async_module") @pytest.fixture diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 1bc67ff7b6..46eabf3963 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -9,12 +9,13 @@ #include "pybind11_tests.h" #include "constructor_stats.h" +#include TEST_SUBMODULE(buffers, m) { // test_from_python / test_to_python: class Matrix { public: - Matrix(ssize_t rows, ssize_t cols) : m_rows(rows), m_cols(cols) { + Matrix(py::ssize_t rows, py::ssize_t cols) : m_rows(rows), m_cols(cols) { print_created(this, std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); m_data = new float[(size_t) (rows*cols)]; memset(m_data, 0, sizeof(float) * (size_t) (rows * cols)); @@ -58,25 +59,25 @@ TEST_SUBMODULE(buffers, m) { return *this; } - float operator()(ssize_t i, ssize_t j) const { + float operator()(py::ssize_t i, py::ssize_t j) const { return m_data[(size_t) (i*m_cols + j)]; } - float &operator()(ssize_t i, ssize_t j) { + float &operator()(py::ssize_t i, py::ssize_t j) { return m_data[(size_t) (i*m_cols + j)]; } float *data() { return m_data; } - ssize_t rows() const { return m_rows; } - ssize_t cols() const { return m_cols; } + py::ssize_t rows() const { return m_rows; } + py::ssize_t cols() const { return m_cols; } private: - ssize_t m_rows; - ssize_t m_cols; + py::ssize_t m_rows; + py::ssize_t m_cols; float *m_data; }; py::class_(m, "Matrix", py::buffer_protocol()) - .def(py::init()) + .def(py::init()) /// Construct from a buffer .def(py::init([](py::buffer const b) { py::buffer_info info = b.request(); @@ -92,12 +93,12 @@ TEST_SUBMODULE(buffers, m) { .def("cols", &Matrix::cols) /// Bare bones interface - .def("__getitem__", [](const Matrix &m, std::pair i) { + .def("__getitem__", [](const Matrix &m, std::pair i) { if (i.first >= m.rows() || i.second >= m.cols()) throw py::index_error(); return m(i.first, i.second); }) - .def("__setitem__", [](Matrix &m, std::pair i, float v) { + .def("__setitem__", [](Matrix &m, std::pair i, float v) { if (i.first >= m.rows() || i.second >= m.cols()) throw py::index_error(); m(i.first, i.second) = v; @@ -117,11 +118,11 @@ TEST_SUBMODULE(buffers, m) { // test_inherited_protocol class SquareMatrix : public Matrix { public: - SquareMatrix(ssize_t n) : Matrix(n, n) { } + SquareMatrix(py::ssize_t n) : Matrix(n, n) { } }; // Derived classes inherit the buffer protocol and the buffer access function py::class_(m, "SquareMatrix") - .def(py::init()); + .def(py::init()); // test_pointer_to_member_fn @@ -192,4 +193,22 @@ TEST_SUBMODULE(buffers, m) { .def_readwrite("readonly", &BufferReadOnlySelect::readonly) .def_buffer(&BufferReadOnlySelect::get_buffer_info); + // Expose buffer_info for testing. + py::class_(m, "buffer_info") + .def(py::init<>()) + .def_readonly("itemsize", &py::buffer_info::itemsize) + .def_readonly("size", &py::buffer_info::size) + .def_readonly("format", &py::buffer_info::format) + .def_readonly("ndim", &py::buffer_info::ndim) + .def_readonly("shape", &py::buffer_info::shape) + .def_readonly("strides", &py::buffer_info::strides) + .def_readonly("readonly", &py::buffer_info::readonly) + .def("__repr__", [](py::handle self) { + return py::str("itemsize={0.itemsize!r}, size={0.size!r}, format={0.format!r}, ndim={0.ndim!r}, shape={0.shape!r}, strides={0.strides!r}, readonly={0.readonly!r}").format(self); + }) + ; + + m.def("get_buffer_info", [](py::buffer buffer) { + return buffer.request(); + }); } diff --git a/tests/test_buffers.py b/tests/test_buffers.py index e264311d7c..50845758b8 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -1,19 +1,16 @@ # -*- coding: utf-8 -*- import io import struct -import sys +import ctypes import pytest +import env # noqa: F401 + from pybind11_tests import buffers as m from pybind11_tests import ConstructorStats -PY3 = sys.version_info[0] >= 3 - -pytestmark = pytest.requires_numpy - -with pytest.suppress(ImportError): - import numpy as np +np = pytest.importorskip("numpy") def test_from_python(): @@ -39,9 +36,7 @@ def test_from_python(): assert cstats.move_assignments == 0 -# PyPy: Memory leak in the "np.array(m, copy=False)" call -# https://bitbucket.org/pypy/pypy/issues/2444 -@pytest.unsupported_on_pypy +# https://foss.heptapod.net/pypy/pypy/-/issues/2444 def test_to_python(): mat = m.Matrix(5, 4) assert memoryview(mat).shape == (5, 4) @@ -51,8 +46,8 @@ def test_to_python(): mat[3, 2] = 7.0 assert mat[2, 3] == 4 assert mat[3, 2] == 7 - assert struct.unpack_from('f', mat, (3 * 4 + 2) * 4) == (7, ) - assert struct.unpack_from('f', mat, (2 * 4 + 3) * 4) == (4, ) + assert struct.unpack_from("f", mat, (3 * 4 + 2) * 4) == (7,) + assert struct.unpack_from("f", mat, (2 * 4 + 3) * 4) == (4,) mat2 = np.array(mat, copy=False) assert mat2.shape == (5, 4) @@ -76,7 +71,6 @@ def test_to_python(): assert cstats.move_assignments == 0 -@pytest.unsupported_on_pypy def test_inherited_protocol(): """SquareMatrix is derived from Matrix and inherits the buffer protocol""" @@ -85,35 +79,86 @@ def test_inherited_protocol(): assert np.asarray(matrix).shape == (5, 5) -@pytest.unsupported_on_pypy def test_pointer_to_member_fn(): for cls in [m.Buffer, m.ConstBuffer, m.DerivedBuffer]: buf = cls() buf.value = 0x12345678 - value = struct.unpack('i', bytearray(buf))[0] + value = struct.unpack("i", bytearray(buf))[0] assert value == 0x12345678 -@pytest.unsupported_on_pypy def test_readonly_buffer(): buf = m.BufferReadOnly(0x64) view = memoryview(buf) - assert view[0] == 0x64 if PY3 else b'd' + assert view[0] == b"d" if env.PY2 else 0x64 assert view.readonly + with pytest.raises(TypeError): + view[0] = b"\0" if env.PY2 else 0 -@pytest.unsupported_on_pypy def test_selective_readonly_buffer(): buf = m.BufferReadOnlySelect() - memoryview(buf)[0] = 0x64 if PY3 else b'd' + memoryview(buf)[0] = b"d" if env.PY2 else 0x64 assert buf.value == 0x64 - io.BytesIO(b'A').readinto(buf) - assert buf.value == ord(b'A') + io.BytesIO(b"A").readinto(buf) + assert buf.value == ord(b"A") buf.readonly = True with pytest.raises(TypeError): - memoryview(buf)[0] = 0 if PY3 else b'\0' + memoryview(buf)[0] = b"\0" if env.PY2 else 0 with pytest.raises(TypeError): - io.BytesIO(b'1').readinto(buf) + io.BytesIO(b"1").readinto(buf) + + +def test_ctypes_array_1d(): + char1d = (ctypes.c_char * 10)() + int1d = (ctypes.c_int * 15)() + long1d = (ctypes.c_long * 7)() + + for carray in (char1d, int1d, long1d): + info = m.get_buffer_info(carray) + assert info.itemsize == ctypes.sizeof(carray._type_) + assert info.size == len(carray) + assert info.ndim == 1 + assert info.shape == [info.size] + assert info.strides == [info.itemsize] + assert not info.readonly + + +def test_ctypes_array_2d(): + char2d = ((ctypes.c_char * 10) * 4)() + int2d = ((ctypes.c_int * 15) * 3)() + long2d = ((ctypes.c_long * 7) * 2)() + + for carray in (char2d, int2d, long2d): + info = m.get_buffer_info(carray) + assert info.itemsize == ctypes.sizeof(carray[0]._type_) + assert info.size == len(carray) * len(carray[0]) + assert info.ndim == 2 + assert info.shape == [len(carray), len(carray[0])] + assert info.strides == [info.itemsize * len(carray[0]), info.itemsize] + assert not info.readonly + + +@pytest.mark.skipif( + "env.PYPY and env.PY2", reason="PyPy2 bytes buffer not reported as readonly" +) +def test_ctypes_from_buffer(): + test_pystr = b"0123456789" + for pyarray in (test_pystr, bytearray(test_pystr)): + pyinfo = m.get_buffer_info(pyarray) + + if pyinfo.readonly: + cbytes = (ctypes.c_char * len(pyarray)).from_buffer_copy(pyarray) + cinfo = m.get_buffer_info(cbytes) + else: + cbytes = (ctypes.c_char * len(pyarray)).from_buffer(pyarray) + cinfo = m.get_buffer_info(cbytes) + + assert cinfo.size == pyinfo.size + assert cinfo.ndim == pyinfo.ndim + assert cinfo.shape == pyinfo.shape + assert cinfo.strides == pyinfo.strides + assert not cinfo.readonly diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index c782870b14..dfcbc0fc27 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -15,6 +15,49 @@ # pragma warning(disable: 4127) // warning C4127: Conditional expression is constant #endif +struct ConstRefCasted { + int tag; +}; + +PYBIND11_NAMESPACE_BEGIN(pybind11) +PYBIND11_NAMESPACE_BEGIN(detail) +template <> +class type_caster { + public: + static constexpr auto name = _(); + + // Input is unimportant, a new value will always be constructed based on the + // cast operator. + bool load(handle, bool) { return true; } + + operator ConstRefCasted&&() { value = {1}; return std::move(value); } + operator ConstRefCasted&() { value = {2}; return value; } + operator ConstRefCasted*() { value = {3}; return &value; } + + operator const ConstRefCasted&() { value = {4}; return value; } + operator const ConstRefCasted*() { value = {5}; return &value; } + + // custom cast_op to explicitly propagate types to the conversion operators. + template + using cast_op_type = + /// const + conditional_t< + std::is_same, const ConstRefCasted*>::value, const ConstRefCasted*, + conditional_t< + std::is_same::value, const ConstRefCasted&, + /// non-const + conditional_t< + std::is_same, ConstRefCasted*>::value, ConstRefCasted*, + conditional_t< + std::is_same::value, ConstRefCasted&, + /* else */ConstRefCasted&&>>>>; + + private: + ConstRefCasted value = {0}; +}; +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(pybind11) + TEST_SUBMODULE(builtin_casters, m) { // test_simple_string m.def("string_roundtrip", [](const char *s) { return s; }); @@ -98,6 +141,10 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("i64_str", [](std::int64_t v) { return std::to_string(v); }); m.def("u64_str", [](std::uint64_t v) { return std::to_string(v); }); + // test_int_convert + m.def("int_passthrough", [](int arg) { return arg; }); + m.def("int_passthrough_noconvert", [](int arg) { return arg; }, py::arg{}.noconvert()); + // test_tuple m.def("pair_passthrough", [](std::pair input) { return std::make_pair(input.second, input.first); @@ -140,13 +187,35 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("load_nullptr_t", [](std::nullptr_t) {}); // not useful, but it should still compile m.def("cast_nullptr_t", []() { return std::nullptr_t{}; }); + // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. + // test_bool_caster m.def("bool_passthrough", [](bool arg) { return arg; }); - m.def("bool_passthrough_noconvert", [](bool arg) { return arg; }, py::arg().noconvert()); + m.def("bool_passthrough_noconvert", [](bool arg) { return arg; }, py::arg{}.noconvert()); + + // TODO: This should be disabled and fixed in future Intel compilers +#if !defined(__INTEL_COMPILER) + // Test "bool_passthrough_noconvert" again, but using () instead of {} to construct py::arg + // When compiled with the Intel compiler, this results in segmentation faults when importing + // the module. Tested with icc (ICC) 2021.1 Beta 20200827, this should be tested again when + // a newer version of icc is available. + m.def("bool_passthrough_noconvert2", [](bool arg) { return arg; }, py::arg().noconvert()); +#endif // test_reference_wrapper m.def("refwrap_builtin", [](std::reference_wrapper p) { return 10 * p.get(); }); m.def("refwrap_usertype", [](std::reference_wrapper p) { return p.get().value(); }); + m.def("refwrap_usertype_const", [](std::reference_wrapper p) { return p.get().value(); }); + + m.def("refwrap_lvalue", []() -> std::reference_wrapper { + static UserType x(1); + return std::ref(x); + }); + m.def("refwrap_lvalue_const", []() -> std::reference_wrapper { + static UserType x(1); + return std::cref(x); + }); + // Not currently supported (std::pair caster has return-by-value cast operator); // triggers static_assert failure. //m.def("refwrap_pair", [](std::reference_wrapper>) { }); @@ -190,6 +259,16 @@ TEST_SUBMODULE(builtin_casters, m) { return py::cast(o) == v; }); + // Tests const/non-const propagation in cast_op. + m.def("takes", [](ConstRefCasted x) { return x.tag; }); + m.def("takes_move", [](ConstRefCasted&& x) { return x.tag; }); + m.def("takes_ptr", [](ConstRefCasted* x) { return x->tag; }); + m.def("takes_ref", [](ConstRefCasted& x) { return x.tag; }); + m.def("takes_ref_wrap", [](std::reference_wrapper x) { return x.get().tag; }); + m.def("takes_const_ptr", [](const ConstRefCasted* x) { return x->tag; }); + m.def("takes_const_ref", [](const ConstRefCasted& x) { return x.tag; }); + m.def("takes_const_ref_wrap", [](std::reference_wrapper x) { return x.get().tag; }); + // For Drake issue: https://github.com/RobotLocomotion/drake/issues/9398 m.def("test_pointer_caster", []() -> bool { UserType a; diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index e4a5235c04..029f291444 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import pytest +import env # noqa: F401 + from pybind11_tests import builtin_casters as m from pybind11_tests import UserType, IncType @@ -35,79 +37,85 @@ def test_unicode_conversion(): with pytest.raises(UnicodeDecodeError): m.bad_utf8_u8string() - assert m.u8_Z() == 'Z' - assert m.u8_eacute() == u'é' - assert m.u16_ibang() == u'‽' - assert m.u32_mathbfA() == u'𝐀' - assert m.wchar_heart() == u'♥' + assert m.u8_Z() == "Z" + assert m.u8_eacute() == u"é" + assert m.u16_ibang() == u"‽" + assert m.u32_mathbfA() == u"𝐀" + assert m.wchar_heart() == u"♥" if hasattr(m, "has_u8string"): - assert m.u8_char8_Z() == 'Z' + assert m.u8_char8_Z() == "Z" def test_single_char_arguments(): """Tests failures for passing invalid inputs to char-accepting functions""" + def toobig_message(r): return "Character code point not in range({0:#x})".format(r) + toolong_message = "Expected a character, but multi-character string found" - assert m.ord_char(u'a') == 0x61 # simple ASCII - assert m.ord_char_lv(u'b') == 0x62 - assert m.ord_char(u'é') == 0xE9 # requires 2 bytes in utf-8, but can be stuffed in a char + assert m.ord_char(u"a") == 0x61 # simple ASCII + assert m.ord_char_lv(u"b") == 0x62 + assert ( + m.ord_char(u"é") == 0xE9 + ) # requires 2 bytes in utf-8, but can be stuffed in a char with pytest.raises(ValueError) as excinfo: - assert m.ord_char(u'Ā') == 0x100 # requires 2 bytes, doesn't fit in a char + assert m.ord_char(u"Ā") == 0x100 # requires 2 bytes, doesn't fit in a char assert str(excinfo.value) == toobig_message(0x100) with pytest.raises(ValueError) as excinfo: - assert m.ord_char(u'ab') + assert m.ord_char(u"ab") assert str(excinfo.value) == toolong_message - assert m.ord_char16(u'a') == 0x61 - assert m.ord_char16(u'é') == 0xE9 - assert m.ord_char16_lv(u'ê') == 0xEA - assert m.ord_char16(u'Ā') == 0x100 - assert m.ord_char16(u'‽') == 0x203d - assert m.ord_char16(u'♥') == 0x2665 - assert m.ord_char16_lv(u'♡') == 0x2661 + assert m.ord_char16(u"a") == 0x61 + assert m.ord_char16(u"é") == 0xE9 + assert m.ord_char16_lv(u"ê") == 0xEA + assert m.ord_char16(u"Ā") == 0x100 + assert m.ord_char16(u"‽") == 0x203D + assert m.ord_char16(u"♥") == 0x2665 + assert m.ord_char16_lv(u"♡") == 0x2661 with pytest.raises(ValueError) as excinfo: - assert m.ord_char16(u'🎂') == 0x1F382 # requires surrogate pair + assert m.ord_char16(u"🎂") == 0x1F382 # requires surrogate pair assert str(excinfo.value) == toobig_message(0x10000) with pytest.raises(ValueError) as excinfo: - assert m.ord_char16(u'aa') + assert m.ord_char16(u"aa") assert str(excinfo.value) == toolong_message - assert m.ord_char32(u'a') == 0x61 - assert m.ord_char32(u'é') == 0xE9 - assert m.ord_char32(u'Ā') == 0x100 - assert m.ord_char32(u'‽') == 0x203d - assert m.ord_char32(u'♥') == 0x2665 - assert m.ord_char32(u'🎂') == 0x1F382 + assert m.ord_char32(u"a") == 0x61 + assert m.ord_char32(u"é") == 0xE9 + assert m.ord_char32(u"Ā") == 0x100 + assert m.ord_char32(u"‽") == 0x203D + assert m.ord_char32(u"♥") == 0x2665 + assert m.ord_char32(u"🎂") == 0x1F382 with pytest.raises(ValueError) as excinfo: - assert m.ord_char32(u'aa') + assert m.ord_char32(u"aa") assert str(excinfo.value) == toolong_message - assert m.ord_wchar(u'a') == 0x61 - assert m.ord_wchar(u'é') == 0xE9 - assert m.ord_wchar(u'Ā') == 0x100 - assert m.ord_wchar(u'‽') == 0x203d - assert m.ord_wchar(u'♥') == 0x2665 + assert m.ord_wchar(u"a") == 0x61 + assert m.ord_wchar(u"é") == 0xE9 + assert m.ord_wchar(u"Ā") == 0x100 + assert m.ord_wchar(u"‽") == 0x203D + assert m.ord_wchar(u"♥") == 0x2665 if m.wchar_size == 2: with pytest.raises(ValueError) as excinfo: - assert m.ord_wchar(u'🎂') == 0x1F382 # requires surrogate pair + assert m.ord_wchar(u"🎂") == 0x1F382 # requires surrogate pair assert str(excinfo.value) == toobig_message(0x10000) else: - assert m.ord_wchar(u'🎂') == 0x1F382 + assert m.ord_wchar(u"🎂") == 0x1F382 with pytest.raises(ValueError) as excinfo: - assert m.ord_wchar(u'aa') + assert m.ord_wchar(u"aa") assert str(excinfo.value) == toolong_message if hasattr(m, "has_u8string"): - assert m.ord_char8(u'a') == 0x61 # simple ASCII - assert m.ord_char8_lv(u'b') == 0x62 - assert m.ord_char8(u'é') == 0xE9 # requires 2 bytes in utf-8, but can be stuffed in a char + assert m.ord_char8(u"a") == 0x61 # simple ASCII + assert m.ord_char8_lv(u"b") == 0x62 + assert ( + m.ord_char8(u"é") == 0xE9 + ) # requires 2 bytes in utf-8, but can be stuffed in a char with pytest.raises(ValueError) as excinfo: - assert m.ord_char8(u'Ā') == 0x100 # requires 2 bytes, doesn't fit in a char + assert m.ord_char8(u"Ā") == 0x100 # requires 2 bytes, doesn't fit in a char assert str(excinfo.value) == toobig_message(0x100) with pytest.raises(ValueError) as excinfo: - assert m.ord_char8(u'ab') + assert m.ord_char8(u"ab") assert str(excinfo.value) == toolong_message @@ -115,28 +123,31 @@ def test_bytes_to_string(): """Tests the ability to pass bytes to C++ string-accepting functions. Note that this is one-way: the only way to return bytes to Python is via the pybind11::bytes class.""" # Issue #816 - import sys - byte = bytes if sys.version_info[0] < 3 else str - assert m.strlen(byte("hi")) == 2 - assert m.string_length(byte("world")) == 5 - assert m.string_length(byte("a\x00b")) == 3 - assert m.strlen(byte("a\x00b")) == 1 # C-string limitation + def to_bytes(s): + b = s if env.PY2 else s.encode("utf8") + assert isinstance(b, bytes) + return b + + assert m.strlen(to_bytes("hi")) == 2 + assert m.string_length(to_bytes("world")) == 5 + assert m.string_length(to_bytes("a\x00b")) == 3 + assert m.strlen(to_bytes("a\x00b")) == 1 # C-string limitation # passing in a utf8 encoded string should work - assert m.string_length(u'💩'.encode("utf8")) == 4 + assert m.string_length(u"💩".encode("utf8")) == 4 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") def test_string_view(capture): """Tests support for C++17 string_view arguments and return values""" assert m.string_view_chars("Hi") == [72, 105] - assert m.string_view_chars("Hi 🎂") == [72, 105, 32, 0xf0, 0x9f, 0x8e, 0x82] - assert m.string_view16_chars(u"Hi 🎂") == [72, 105, 32, 0xd83c, 0xdf82] + assert m.string_view_chars("Hi 🎂") == [72, 105, 32, 0xF0, 0x9F, 0x8E, 0x82] + assert m.string_view16_chars(u"Hi 🎂") == [72, 105, 32, 0xD83C, 0xDF82] assert m.string_view32_chars(u"Hi 🎂") == [72, 105, 32, 127874] if hasattr(m, "has_u8string"): assert m.string_view8_chars("Hi") == [72, 105] - assert m.string_view8_chars(u"Hi 🎂") == [72, 105, 32, 0xf0, 0x9f, 0x8e, 0x82] + assert m.string_view8_chars(u"Hi 🎂") == [72, 105, 32, 0xF0, 0x9F, 0x8E, 0x82] assert m.string_view_return() == u"utf8 secret 🎂" assert m.string_view16_return() == u"utf16 secret 🎂" @@ -149,54 +160,71 @@ def test_string_view(capture): m.string_view_print("utf8 🎂") m.string_view16_print(u"utf16 🎂") m.string_view32_print(u"utf32 🎂") - assert capture == u""" + assert ( + capture + == u""" Hi 2 utf8 🎂 9 utf16 🎂 8 utf32 🎂 7 """ + ) if hasattr(m, "has_u8string"): with capture: m.string_view8_print("Hi") m.string_view8_print(u"utf8 🎂") - assert capture == u""" + assert ( + capture + == u""" Hi 2 utf8 🎂 9 """ + ) with capture: m.string_view_print("Hi, ascii") m.string_view_print("Hi, utf8 🎂") m.string_view16_print(u"Hi, utf16 🎂") m.string_view32_print(u"Hi, utf32 🎂") - assert capture == u""" + assert ( + capture + == u""" Hi, ascii 9 Hi, utf8 🎂 13 Hi, utf16 🎂 12 Hi, utf32 🎂 11 """ + ) if hasattr(m, "has_u8string"): with capture: m.string_view8_print("Hi, ascii") m.string_view8_print(u"Hi, utf8 🎂") - assert capture == u""" + assert ( + capture + == u""" Hi, ascii 9 Hi, utf8 🎂 13 """ + ) def test_integer_casting(): """Issue #929 - out-of-range integer values shouldn't be accepted""" - import sys assert m.i32_str(-1) == "-1" assert m.i64_str(-1) == "-1" assert m.i32_str(2000000000) == "2000000000" assert m.u32_str(2000000000) == "2000000000" - if sys.version_info < (3,): + if env.PY2: assert m.i32_str(long(-1)) == "-1" # noqa: F821 undefined name 'long' assert m.i64_str(long(-1)) == "-1" # noqa: F821 undefined name 'long' - assert m.i64_str(long(-999999999999)) == "-999999999999" # noqa: F821 undefined name - assert m.u64_str(long(999999999999)) == "999999999999" # noqa: F821 undefined name 'long' + assert ( + m.i64_str(long(-999999999999)) # noqa: F821 undefined name 'long' + == "-999999999999" + ) + assert ( + m.u64_str(long(999999999999)) # noqa: F821 undefined name 'long' + == "999999999999" + ) else: assert m.i64_str(-999999999999) == "-999999999999" assert m.u64_str(999999999999) == "999999999999" @@ -214,7 +242,7 @@ def test_integer_casting(): m.i32_str(3000000000) assert "incompatible function arguments" in str(excinfo.value) - if sys.version_info < (3,): + if env.PY2: with pytest.raises(TypeError) as excinfo: m.u32_str(long(-1)) # noqa: F821 undefined name 'long' assert "incompatible function arguments" in str(excinfo.value) @@ -223,6 +251,98 @@ def test_integer_casting(): assert "incompatible function arguments" in str(excinfo.value) +def test_int_convert(): + class Int(object): + def __int__(self): + return 42 + + class NotInt(object): + pass + + class Float(object): + def __float__(self): + return 41.99999 + + class Index(object): + def __index__(self): + return 42 + + class IntAndIndex(object): + def __int__(self): + return 42 + + def __index__(self): + return 0 + + class RaisingTypeErrorOnIndex(object): + def __index__(self): + raise TypeError + + def __int__(self): + return 42 + + class RaisingValueErrorOnIndex(object): + def __index__(self): + raise ValueError + + def __int__(self): + return 42 + + convert, noconvert = m.int_passthrough, m.int_passthrough_noconvert + + def requires_conversion(v): + pytest.raises(TypeError, noconvert, v) + + def cant_convert(v): + pytest.raises(TypeError, convert, v) + + assert convert(7) == 7 + assert noconvert(7) == 7 + cant_convert(3.14159) + # TODO: Avoid DeprecationWarning in `PyLong_AsLong` (and similar) + if (3, 8) <= env.PY < (3, 10): + with pytest.deprecated_call(): + assert convert(Int()) == 42 + else: + assert convert(Int()) == 42 + requires_conversion(Int()) + cant_convert(NotInt()) + cant_convert(Float()) + + # Before Python 3.8, `PyLong_AsLong` does not pick up on `obj.__index__`, + # but pybind11 "backports" this behavior. + assert convert(Index()) == 42 + assert noconvert(Index()) == 42 + assert convert(IntAndIndex()) == 0 # Fishy; `int(DoubleThought)` == 42 + assert noconvert(IntAndIndex()) == 0 + assert convert(RaisingTypeErrorOnIndex()) == 42 + requires_conversion(RaisingTypeErrorOnIndex()) + assert convert(RaisingValueErrorOnIndex()) == 42 + requires_conversion(RaisingValueErrorOnIndex()) + + +def test_numpy_int_convert(): + np = pytest.importorskip("numpy") + + convert, noconvert = m.int_passthrough, m.int_passthrough_noconvert + + def require_implicit(v): + pytest.raises(TypeError, noconvert, v) + + # `np.intc` is an alias that corresponds to a C++ `int` + assert convert(np.intc(42)) == 42 + assert noconvert(np.intc(42)) == 42 + + # The implicit conversion from np.float32 is undesirable but currently accepted. + # TODO: Avoid DeprecationWarning in `PyLong_AsLong` (and similar) + if (3, 8) <= env.PY < (3, 10): + with pytest.deprecated_call(): + assert convert(np.float32(3.14159)) == 3 + else: + assert convert(np.float32(3.14159)) == 3 + require_implicit(np.float32(3.14159)) + + def test_tuple(doc): """std::pair <-> tuple & std::tuple <-> tuple""" assert m.pair_passthrough((True, "test")) == ("test", True) @@ -232,16 +352,22 @@ def test_tuple(doc): assert m.tuple_passthrough([True, "test", 5]) == (5, "test", True) assert m.empty_tuple() == () - assert doc(m.pair_passthrough) == """ + assert ( + doc(m.pair_passthrough) + == """ pair_passthrough(arg0: Tuple[bool, str]) -> Tuple[str, bool] Return a pair in reversed order """ - assert doc(m.tuple_passthrough) == """ + ) + assert ( + doc(m.tuple_passthrough) + == """ tuple_passthrough(arg0: Tuple[bool, str, int]) -> Tuple[int, str, bool] Return a triple in reversed order """ + ) assert m.rvalue_pair() == ("rvalue", "rvalue") assert m.lvalue_pair() == ("lvalue", "lvalue") @@ -281,6 +407,7 @@ def test_reference_wrapper(): """std::reference_wrapper for builtin and user types""" assert m.refwrap_builtin(42) == 420 assert m.refwrap_usertype(UserType(42)) == 42 + assert m.refwrap_usertype_const(UserType(42)) == 42 with pytest.raises(TypeError) as excinfo: m.refwrap_builtin(None) @@ -290,6 +417,9 @@ def test_reference_wrapper(): m.refwrap_usertype(None) assert "incompatible function arguments" in str(excinfo.value) + assert m.refwrap_lvalue().value == 1 + assert m.refwrap_lvalue_const().value == 1 + a1 = m.refwrap_list(copy=True) a2 = m.refwrap_list(copy=True) assert [x.value for x in a1] == [2, 3] @@ -355,9 +485,9 @@ class B(object): assert convert(A(False)) is False -@pytest.requires_numpy def test_numpy_bool(): - import numpy as np + np = pytest.importorskip("numpy") + convert, noconvert = m.bool_passthrough, m.bool_passthrough_noconvert def cant_convert(v): @@ -368,7 +498,7 @@ def cant_convert(v): assert convert(np.bool_(False)) is False assert noconvert(np.bool_(True)) is True assert noconvert(np.bool_(False)) is False - cant_convert(np.zeros(2, dtype='int')) + cant_convert(np.zeros(2, dtype="int")) def test_int_long(): @@ -378,7 +508,8 @@ def test_int_long(): long.""" import sys - must_be_long = type(getattr(sys, 'maxint', 1) + 1) + + must_be_long = type(getattr(sys, "maxint", 1) + 1) assert isinstance(m.int_cast(), int) assert isinstance(m.long_cast(), int) assert isinstance(m.longlong_cast(), must_be_long) @@ -388,5 +519,23 @@ def test_void_caster_2(): assert m.test_void_caster() +def test_const_ref_caster(): + """Verifies that const-ref is propagated through type_caster cast_op. + The returned ConstRefCasted type is a mimimal type that is constructed to + reference the casting mode used. + """ + x = False + assert m.takes(x) == 1 + assert m.takes_move(x) == 1 + + assert m.takes_ptr(x) == 3 + assert m.takes_ref(x) == 2 + assert m.takes_ref_wrap(x) == 2 + + assert m.takes_const_ptr(x) == 5 + assert m.takes_const_ref(x) == 4 + assert m.takes_const_ref_wrap(x) == 4 + + def test_pointer_caster(): assert m.test_pointer_caster() diff --git a/tests/test_call_policies.py b/tests/test_call_policies.py index 0e3230c573..e0413d1451 100644 --- a/tests/test_call_policies.py +++ b/tests/test_call_policies.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- import pytest + +import env # noqa: F401 + from pybind11_tests import call_policies as m from pybind11_tests import ConstructorStats +@pytest.mark.xfail("env.PYPY", reason="sometimes comes out 1 off on PyPy", strict=False) def test_keep_alive_argument(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -12,10 +16,13 @@ def test_keep_alive_argument(capture): with capture: p.addChild(m.Child()) assert ConstructorStats.detail_reg_inst() == n_inst + 1 - assert capture == """ + assert ( + capture + == """ Allocating child. Releasing child. """ + ) with capture: del p assert ConstructorStats.detail_reg_inst() == n_inst @@ -31,10 +38,13 @@ def test_keep_alive_argument(capture): with capture: del p assert ConstructorStats.detail_reg_inst() == n_inst - assert capture == """ + assert ( + capture + == """ Releasing parent. Releasing child. """ + ) def test_keep_alive_return_value(capture): @@ -45,10 +55,13 @@ def test_keep_alive_return_value(capture): with capture: p.returnChild() assert ConstructorStats.detail_reg_inst() == n_inst + 1 - assert capture == """ + assert ( + capture + == """ Allocating child. Releasing child. """ + ) with capture: del p assert ConstructorStats.detail_reg_inst() == n_inst @@ -64,28 +77,34 @@ def test_keep_alive_return_value(capture): with capture: del p assert ConstructorStats.detail_reg_inst() == n_inst - assert capture == """ + assert ( + capture + == """ Releasing parent. Releasing child. """ + ) -# https://bitbucket.org/pypy/pypy/issues/2447 -@pytest.unsupported_on_pypy +# https://foss.heptapod.net/pypy/pypy/-/issues/2447 +@pytest.mark.xfail("env.PYPY", reason="_PyObject_GetDictPtr is unimplemented") def test_alive_gc(capture): n_inst = ConstructorStats.detail_reg_inst() p = m.ParentGC() p.addChildKeepAlive(m.Child()) assert ConstructorStats.detail_reg_inst() == n_inst + 2 lst = [p] - lst.append(lst) # creates a circular reference + lst.append(lst) # creates a circular reference with capture: del p, lst assert ConstructorStats.detail_reg_inst() == n_inst - assert capture == """ + assert ( + capture + == """ Releasing parent. Releasing child. """ + ) def test_alive_gc_derived(capture): @@ -97,14 +116,17 @@ class Derived(m.Parent): p.addChildKeepAlive(m.Child()) assert ConstructorStats.detail_reg_inst() == n_inst + 2 lst = [p] - lst.append(lst) # creates a circular reference + lst.append(lst) # creates a circular reference with capture: del p, lst assert ConstructorStats.detail_reg_inst() == n_inst - assert capture == """ + assert ( + capture + == """ Releasing parent. Releasing child. """ + ) def test_alive_gc_multi_derived(capture): @@ -119,15 +141,18 @@ def __init__(self): # +3 rather than +2 because Derived corresponds to two registered instances assert ConstructorStats.detail_reg_inst() == n_inst + 3 lst = [p] - lst.append(lst) # creates a circular reference + lst.append(lst) # creates a circular reference with capture: del p, lst assert ConstructorStats.detail_reg_inst() == n_inst - assert capture == """ + assert ( + capture + == """ Releasing parent. Releasing child. Releasing child. """ + ) def test_return_none(capture): @@ -163,17 +188,23 @@ def test_keep_alive_constructor(capture): with capture: p = m.Parent(m.Child()) assert ConstructorStats.detail_reg_inst() == n_inst + 2 - assert capture == """ + assert ( + capture + == """ Allocating child. Allocating parent. """ + ) with capture: del p assert ConstructorStats.detail_reg_inst() == n_inst - assert capture == """ + assert ( + capture + == """ Releasing parent. Releasing child. """ + ) def test_call_guard(): diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp index 71b88c44c7..dffe538fc5 100644 --- a/tests/test_callbacks.cpp +++ b/tests/test_callbacks.cpp @@ -117,7 +117,14 @@ TEST_SUBMODULE(callbacks, m) { } }); - class AbstractBase { public: virtual unsigned int func() = 0; }; + class AbstractBase { + public: + // [workaround(intel)] = default does not work here + // Defaulting this destructor results in linking errors with the Intel compiler + // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) + virtual ~AbstractBase() {}; // NOLINT(modernize-use-equals-default) + virtual unsigned int func() = 0; + }; m.def("func_accepting_func_accepting_base", [](std::function) { }); struct MovableObject { diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index d5d0e045d2..039b877ced 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -42,17 +42,19 @@ def double(self, val): def test_keyword_args_and_generalized_unpacking(): - def f(*args, **kwargs): return args, kwargs assert m.test_tuple_unpacking(f) == (("positional", 1, 2, 3, 4, 5, 6), {}) - assert m.test_dict_unpacking(f) == (("positional", 1), {"key": "value", "a": 1, "b": 2}) + assert m.test_dict_unpacking(f) == ( + ("positional", 1), + {"key": "value", "a": 1, "b": 2}, + ) assert m.test_keyword_args(f) == ((), {"x": 10, "y": 20}) assert m.test_unpacking_and_keywords1(f) == ((1, 2), {"c": 3, "d": 4}) assert m.test_unpacking_and_keywords2(f) == ( ("positional", 1, 2, 3, 4, 5), - {"key": "value", "a": 1, "b": 2, "c": 3, "d": 4, "e": 5} + {"key": "value", "a": 1, "b": 2, "c": 3, "d": 4, "e": 5}, ) with pytest.raises(TypeError) as excinfo: @@ -83,12 +85,18 @@ def test_lambda_closure_cleanup(): def test_cpp_function_roundtrip(): """Test if passing a function pointer from C++ -> Python -> C++ yields the original pointer""" - assert m.test_dummy_function(m.dummy_function) == "matches dummy_function: eval(1) = 2" - assert (m.test_dummy_function(m.roundtrip(m.dummy_function)) == - "matches dummy_function: eval(1) = 2") + assert ( + m.test_dummy_function(m.dummy_function) == "matches dummy_function: eval(1) = 2" + ) + assert ( + m.test_dummy_function(m.roundtrip(m.dummy_function)) + == "matches dummy_function: eval(1) = 2" + ) assert m.roundtrip(None, expect_none=True) is None - assert (m.test_dummy_function(lambda x: x + 2) == - "can't convert to function pointer: eval(1) = 3") + assert ( + m.test_dummy_function(lambda x: x + 2) + == "can't convert to function pointer: eval(1) = 3" + ) with pytest.raises(TypeError) as excinfo: m.test_dummy_function(m.dummy_function2) @@ -96,8 +104,10 @@ def test_cpp_function_roundtrip(): with pytest.raises(TypeError) as excinfo: m.test_dummy_function(lambda x, y: x + y) - assert any(s in str(excinfo.value) for s in ("missing 1 required positional argument", - "takes exactly 2 arguments")) + assert any( + s in str(excinfo.value) + for s in ("missing 1 required positional argument", "takes exactly 2 arguments") + ) def test_function_signatures(doc): @@ -127,6 +137,7 @@ def gen_f(): m.test_async_callback(gen_f(), work) # wait until work is done from time import sleep + sleep(0.5) assert sum(res) == sum([x + 3 for x in work]) diff --git a/tests/test_chrono.cpp b/tests/test_chrono.cpp index 899d08d8d8..6537050803 100644 --- a/tests/test_chrono.cpp +++ b/tests/test_chrono.cpp @@ -10,6 +10,25 @@ #include "pybind11_tests.h" #include +#include + +struct different_resolutions { + using time_point_h = std::chrono::time_point< + std::chrono::system_clock, std::chrono::hours>; + using time_point_m = std::chrono::time_point< + std::chrono::system_clock, std::chrono::minutes>; + using time_point_s = std::chrono::time_point< + std::chrono::system_clock, std::chrono::seconds>; + using time_point_ms = std::chrono::time_point< + std::chrono::system_clock, std::chrono::milliseconds>; + using time_point_us = std::chrono::time_point< + std::chrono::system_clock, std::chrono::microseconds>; + time_point_h timestamp_h; + time_point_m timestamp_m; + time_point_s timestamp_s; + time_point_ms timestamp_ms; + time_point_us timestamp_us; +}; TEST_SUBMODULE(chrono, m) { using system_time = std::chrono::system_clock::time_point; @@ -52,4 +71,14 @@ TEST_SUBMODULE(chrono, m) { m.def("test_nano_timepoint", [](timestamp start, timespan delta) -> timestamp { return start + delta; }); + + // Test different resolutions + py::class_(m, "different_resolutions") + .def(py::init<>()) + .def_readwrite("timestamp_h", &different_resolutions::timestamp_h) + .def_readwrite("timestamp_m", &different_resolutions::timestamp_m) + .def_readwrite("timestamp_s", &different_resolutions::timestamp_s) + .def_readwrite("timestamp_ms", &different_resolutions::timestamp_ms) + .def_readwrite("timestamp_us", &different_resolutions::timestamp_us) + ; } diff --git a/tests/test_chrono.py b/tests/test_chrono.py index f1817e44f6..e9e24e0826 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- from pybind11_tests import chrono as m import datetime +import pytest + +import env # noqa: F401 def test_chrono_system_clock(): # Get the time from both c++ and datetime + date0 = datetime.datetime.today() date1 = m.test_chrono1() date2 = datetime.datetime.today() @@ -13,16 +17,15 @@ def test_chrono_system_clock(): assert isinstance(date1, datetime.datetime) # The numbers should vary by a very small amount (time it took to execute) + diff_python = abs(date2 - date0) diff = abs(date1 - date2) - # There should never be a days/seconds difference + # There should never be a days difference assert diff.days == 0 - assert diff.seconds == 0 - # We test that no more than about 0.5 seconds passes here - # This makes sure that the dates created are very close to the same - # but if the testing system is incredibly overloaded this should still pass - assert diff.microseconds < 500000 + # Since datetime.datetime.today() calls time.time(), and on some platforms + # that has 1 second accuracy, we compare this way + assert diff.seconds <= diff_python.seconds def test_chrono_system_clock_roundtrip(): @@ -72,8 +75,36 @@ def test_chrono_system_clock_roundtrip_date(): assert time2.microsecond == 0 -def test_chrono_system_clock_roundtrip_time(): - time1 = datetime.datetime.today().time() +SKIP_TZ_ENV_ON_WIN = pytest.mark.skipif( + "env.WIN", reason="TZ environment variable only supported on POSIX" +) + + +@pytest.mark.parametrize( + "time1", + [ + datetime.datetime.today().time(), + datetime.time(0, 0, 0), + datetime.time(0, 0, 0, 1), + datetime.time(0, 28, 45, 109827), + datetime.time(0, 59, 59, 999999), + datetime.time(1, 0, 0), + datetime.time(5, 59, 59, 0), + datetime.time(5, 59, 59, 1), + ], +) +@pytest.mark.parametrize( + "tz", + [ + None, + pytest.param("Europe/Brussels", marks=SKIP_TZ_ENV_ON_WIN), + pytest.param("Asia/Pyongyang", marks=SKIP_TZ_ENV_ON_WIN), + pytest.param("America/New_York", marks=SKIP_TZ_ENV_ON_WIN), + ], +) +def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): + if tz is not None: + monkeypatch.setenv("TZ", "/usr/share/zoneinfo/{}".format(tz)) # Roundtrip the time datetime2 = m.test_chrono2(time1) @@ -174,4 +205,14 @@ def test_floating_point_duration(): def test_nano_timepoint(): time = datetime.datetime.now() time1 = m.test_nano_timepoint(time, datetime.timedelta(seconds=60)) - assert(time1 == time + datetime.timedelta(seconds=60)) + assert time1 == time + datetime.timedelta(seconds=60) + + +def test_chrono_different_resolutions(): + resolutions = m.different_resolutions() + time = datetime.datetime.now() + resolutions.timestamp_h = time + resolutions.timestamp_m = time + resolutions.timestamp_s = time + resolutions.timestamp_ms = time + resolutions.timestamp_us = time diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 77dc3bf1a5..ee2212327f 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -7,6 +7,13 @@ BSD-style license that can be found in the LICENSE file. */ +#if defined(__INTEL_COMPILER) && __cplusplus >= 201703L +// Intel compiler requires a separate header file to support aligned new operators +// and does not set the __cpp_aligned_new feature macro. +// This header needs to be included before pybind11. +#include +#endif + #include "pybind11_tests.h" #include "constructor_stats.h" #include "local_bindings.h" @@ -103,7 +110,7 @@ TEST_SUBMODULE(class_, m) { BaseClass() = default; BaseClass(const BaseClass &) = default; BaseClass(BaseClass &&) = default; - virtual ~BaseClass() {} + virtual ~BaseClass() = default; }; struct DerivedClass1 : BaseClass { }; struct DerivedClass2 : BaseClass { }; @@ -134,6 +141,32 @@ TEST_SUBMODULE(class_, m) { ); }); + struct Invalid {}; + + // test_type + m.def("check_type", [](int category) { + // Currently not supported (via a fail at compile time) + // See https://github.com/pybind/pybind11/issues/2486 + // if (category == 2) + // return py::type::of(); + if (category == 1) + return py::type::of(); + else + return py::type::of(); + }); + + m.def("get_type_of", [](py::object ob) { + return py::type::of(ob); + }); + + m.def("get_type_classic", [](py::handle h) { + return h.get_type(); + }); + + m.def("as_type", [](py::object ob) { + return py::type(ob); + }); + // test_mismatched_holder struct MismatchBase1 { }; struct MismatchDerived1 : MismatchBase1 { }; @@ -142,12 +175,12 @@ TEST_SUBMODULE(class_, m) { struct MismatchDerived2 : MismatchBase2 { }; m.def("mismatched_holder_1", []() { - auto mod = py::module::import("__main__"); + auto mod = py::module_::import("__main__"); py::class_>(mod, "MismatchBase1"); py::class_(mod, "MismatchDerived1"); }); m.def("mismatched_holder_2", []() { - auto mod = py::module::import("__main__"); + auto mod = py::module_::import("__main__"); py::class_(mod, "MismatchBase2"); py::class_, MismatchBase2>(mod, "MismatchDerived2"); @@ -205,7 +238,8 @@ TEST_SUBMODULE(class_, m) { }; auto def = new PyMethodDef{"f", f, METH_VARARGS, nullptr}; - return py::reinterpret_steal(PyCFunction_NewEx(def, nullptr, m.ptr())); + py::capsule def_capsule(def, [](void *ptr) { delete reinterpret_cast(ptr); }); + return py::reinterpret_steal(PyCFunction_NewEx(def, def_capsule.ptr(), m.ptr())); }()); // test_operator_new_delete @@ -291,11 +325,15 @@ TEST_SUBMODULE(class_, m) { class TrampolineB : public ProtectedB { public: - int foo() const override { PYBIND11_OVERLOAD(int, ProtectedB, foo, ); } + int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); } }; class PublicistB : public ProtectedB { public: + // [workaround(intel)] = default does not work here + // Removing or defaulting this destructor results in linking errors with the Intel compiler + // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) + ~PublicistB() override {}; // NOLINT(modernize-use-equals-default) using ProtectedB::foo; }; @@ -327,7 +365,7 @@ TEST_SUBMODULE(class_, m) { // test_reentrant_implicit_conversion_failure // #1035: issue with runaway reentrant implicit conversion struct BogusImplicitConversion { - BogusImplicitConversion(const BogusImplicitConversion &) { } + BogusImplicitConversion(const BogusImplicitConversion &) = default; }; py::class_(m, "BogusImplicitConversion") @@ -380,8 +418,9 @@ TEST_SUBMODULE(class_, m) { struct IsNonFinalFinal {}; py::class_(m, "IsNonFinalFinal", py::is_final()); + // test_exception_rvalue_abort struct PyPrintDestructor { - PyPrintDestructor() {} + PyPrintDestructor() = default; ~PyPrintDestructor() { py::print("Print from destructor"); } @@ -391,10 +430,58 @@ TEST_SUBMODULE(class_, m) { .def(py::init<>()) .def("throw_something", &PyPrintDestructor::throw_something); + // test_multiple_instances_with_same_pointer + struct SamePointer {}; + static SamePointer samePointer; + py::class_>(m, "SamePointer") + .def(py::init([]() { return &samePointer; })); + + struct Empty {}; + py::class_(m, "Empty") + .def(py::init<>()); + + // test_base_and_derived_nested_scope + struct BaseWithNested { + struct Nested {}; + }; + + struct DerivedWithNested : BaseWithNested { + struct Nested {}; + }; + + py::class_ baseWithNested_class(m, "BaseWithNested"); + py::class_ derivedWithNested_class(m, "DerivedWithNested"); + py::class_(baseWithNested_class, "Nested") + .def_static("get_name", []() { return "BaseWithNested::Nested"; }); + py::class_(derivedWithNested_class, "Nested") + .def_static("get_name", []() { return "DerivedWithNested::Nested"; }); + + // test_register_duplicate_class + struct Duplicate {}; + struct OtherDuplicate {}; + struct DuplicateNested {}; + struct OtherDuplicateNested {}; + m.def("register_duplicate_class_name", [](py::module_ m) { + py::class_(m, "Duplicate"); + py::class_(m, "Duplicate"); + }); + m.def("register_duplicate_class_type", [](py::module_ m) { + py::class_(m, "OtherDuplicate"); + py::class_(m, "YetAnotherDuplicate"); + }); + m.def("register_duplicate_nested_class_name", [](py::object gt) { + py::class_(gt, "DuplicateNested"); + py::class_(gt, "DuplicateNested"); + }); + m.def("register_duplicate_nested_class_type", [](py::object gt) { + py::class_(gt, "OtherDuplicateNested"); + py::class_(gt, "YetAnotherDuplicateNested"); + }); + // Test #1922 (drake#11424). class ExampleVirt2 { public: - virtual ~ExampleVirt2() {} + virtual ~ExampleVirt2() = default; virtual std::string get_name() const { return "ExampleVirt2"; } }; class PyExampleVirt2 : public ExampleVirt2 { @@ -417,14 +504,14 @@ template class BreaksBase { public: }; template class BreaksTramp : public BreaksBase {}; // These should all compile just fine: -typedef py::class_, std::unique_ptr>, BreaksTramp<1>> DoesntBreak1; -typedef py::class_, BreaksTramp<2>, std::unique_ptr>> DoesntBreak2; -typedef py::class_, std::unique_ptr>> DoesntBreak3; -typedef py::class_, BreaksTramp<4>> DoesntBreak4; -typedef py::class_> DoesntBreak5; -typedef py::class_, std::shared_ptr>, BreaksTramp<6>> DoesntBreak6; -typedef py::class_, BreaksTramp<7>, std::shared_ptr>> DoesntBreak7; -typedef py::class_, std::shared_ptr>> DoesntBreak8; +using DoesntBreak1 = py::class_, std::unique_ptr>, BreaksTramp<1>>; +using DoesntBreak2 = py::class_, BreaksTramp<2>, std::unique_ptr>>; +using DoesntBreak3 = py::class_, std::unique_ptr>>; +using DoesntBreak4 = py::class_, BreaksTramp<4>>; +using DoesntBreak5 = py::class_>; +using DoesntBreak6 = py::class_, std::shared_ptr>, BreaksTramp<6>>; +using DoesntBreak7 = py::class_, BreaksTramp<7>, std::shared_ptr>>; +using DoesntBreak8 = py::class_, std::shared_ptr>>; #define CHECK_BASE(N) static_assert(std::is_same>::value, \ "DoesntBreak" #N " has wrong type!") CHECK_BASE(1); CHECK_BASE(2); CHECK_BASE(3); CHECK_BASE(4); CHECK_BASE(5); CHECK_BASE(6); CHECK_BASE(7); CHECK_BASE(8); diff --git a/tests/test_class.py b/tests/test_class.py index 5054774f4b..69c8832f96 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -2,6 +2,8 @@ import pytest import weakref +import env # noqa: F401 + from pybind11_tests import class_ as m from pybind11_tests import UserType, ConstructorStats @@ -25,6 +27,48 @@ def test_instance(msg): assert cstats.alive() == 0 +def test_type(): + assert m.check_type(1) == m.DerivedClass1 + with pytest.raises(RuntimeError) as execinfo: + m.check_type(0) + + assert "pybind11::detail::get_type_info: unable to find type info" in str( + execinfo.value + ) + assert "Invalid" in str(execinfo.value) + + # Currently not supported + # See https://github.com/pybind/pybind11/issues/2486 + # assert m.check_type(2) == int + + +def test_type_of_py(): + assert m.get_type_of(1) == int + assert m.get_type_of(m.DerivedClass1()) == m.DerivedClass1 + assert m.get_type_of(int) == type + + +def test_type_of_classic(): + assert m.get_type_classic(1) == int + assert m.get_type_classic(m.DerivedClass1()) == m.DerivedClass1 + assert m.get_type_classic(int) == type + + +def test_type_of_py_nodelete(): + # If the above test deleted the class, this will segfault + assert m.get_type_of(m.DerivedClass1()) == m.DerivedClass1 + + +def test_as_type_py(): + assert m.as_type(int) == int + + with pytest.raises(TypeError): + assert m.as_type(1) == int + + with pytest.raises(TypeError): + assert m.as_type(m.DerivedClass1()) == m.DerivedClass1 + + def test_docstrings(doc): assert doc(UserType) == "A `py::class_` type for testing" assert UserType.__name__ == "UserType" @@ -32,18 +76,24 @@ def test_docstrings(doc): assert UserType.get_value.__name__ == "get_value" assert UserType.get_value.__module__ == "pybind11_tests" - assert doc(UserType.get_value) == """ + assert ( + doc(UserType.get_value) + == """ get_value(self: m.UserType) -> int Get value using a method """ + ) assert doc(UserType.value) == "Get/set value using a property" - assert doc(m.NoConstructor.new_instance) == """ + assert ( + doc(m.NoConstructor.new_instance) + == """ new_instance() -> m.class_.NoConstructor Return an instance """ + ) def test_qualname(doc): @@ -52,51 +102,69 @@ def test_qualname(doc): assert m.NestBase.__qualname__ == "NestBase" assert m.NestBase.Nested.__qualname__ == "NestBase.Nested" - assert doc(m.NestBase.__init__) == """ + assert ( + doc(m.NestBase.__init__) + == """ __init__(self: m.class_.NestBase) -> None """ - assert doc(m.NestBase.g) == """ + ) + assert ( + doc(m.NestBase.g) + == """ g(self: m.class_.NestBase, arg0: m.class_.NestBase.Nested) -> None """ - assert doc(m.NestBase.Nested.__init__) == """ + ) + assert ( + doc(m.NestBase.Nested.__init__) + == """ __init__(self: m.class_.NestBase.Nested) -> None """ - assert doc(m.NestBase.Nested.fn) == """ + ) + assert ( + doc(m.NestBase.Nested.fn) + == """ fn(self: m.class_.NestBase.Nested, arg0: int, arg1: m.class_.NestBase, arg2: m.class_.NestBase.Nested) -> None """ # noqa: E501 line too long - assert doc(m.NestBase.Nested.fa) == """ + ) + assert ( + doc(m.NestBase.Nested.fa) + == """ fa(self: m.class_.NestBase.Nested, a: int, b: m.class_.NestBase, c: m.class_.NestBase.Nested) -> None """ # noqa: E501 line too long + ) assert m.NestBase.__module__ == "pybind11_tests.class_" assert m.NestBase.Nested.__module__ == "pybind11_tests.class_" def test_inheritance(msg): - roger = m.Rabbit('Rabbit') + roger = m.Rabbit("Rabbit") assert roger.name() + " is a " + roger.species() == "Rabbit is a parrot" assert m.pet_name_species(roger) == "Rabbit is a parrot" - polly = m.Pet('Polly', 'parrot') + polly = m.Pet("Polly", "parrot") assert polly.name() + " is a " + polly.species() == "Polly is a parrot" assert m.pet_name_species(polly) == "Polly is a parrot" - molly = m.Dog('Molly') + molly = m.Dog("Molly") assert molly.name() + " is a " + molly.species() == "Molly is a dog" assert m.pet_name_species(molly) == "Molly is a dog" - fred = m.Hamster('Fred') + fred = m.Hamster("Fred") assert fred.name() + " is a " + fred.species() == "Fred is a rodent" assert m.dog_bark(molly) == "Woof!" with pytest.raises(TypeError) as excinfo: m.dog_bark(polly) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ dog_bark(): incompatible function arguments. The following argument types are supported: 1. (arg0: m.class_.Dog) -> str Invoked with: """ + ) with pytest.raises(TypeError) as excinfo: m.Chimera("lion", "goat") @@ -109,12 +177,11 @@ def test_inheritance_init(msg): class Python(m.Pet): def __init__(self): pass + with pytest.raises(TypeError) as exc_info: Python() - expected = ["m.class_.Pet.__init__() must be called when overriding __init__", - "Pet.__init__() must be called when overriding __init__"] # PyPy? - # TODO: fix PyPy error message wrt. tp_name/__qualname__? - assert msg(exc_info.value) in expected + expected = "m.class_.Pet.__init__() must be called when overriding __init__" + assert msg(exc_info.value) == expected # Multiple bases class RabbitHamster(m.Rabbit, m.Hamster): @@ -123,9 +190,8 @@ def __init__(self): with pytest.raises(TypeError) as exc_info: RabbitHamster() - expected = ["m.class_.Hamster.__init__() must be called when overriding __init__", - "Hamster.__init__() must be called when overriding __init__"] # PyPy - assert msg(exc_info.value) in expected + expected = "m.class_.Hamster.__init__() must be called when overriding __init__" + assert msg(exc_info.value) == expected def test_automatic_upcasting(): @@ -153,13 +219,19 @@ def test_mismatched_holder(): with pytest.raises(RuntimeError) as excinfo: m.mismatched_holder_1() - assert re.match('generic_type: type ".*MismatchDerived1" does not have a non-default ' - 'holder type while its base ".*MismatchBase1" does', str(excinfo.value)) + assert re.match( + 'generic_type: type ".*MismatchDerived1" does not have a non-default ' + 'holder type while its base ".*MismatchBase1" does', + str(excinfo.value), + ) with pytest.raises(RuntimeError) as excinfo: m.mismatched_holder_2() - assert re.match('generic_type: type ".*MismatchDerived2" has a non-default holder type ' - 'while its base ".*MismatchBase2" does not', str(excinfo.value)) + assert re.match( + 'generic_type: type ".*MismatchDerived2" has a non-default holder type ' + 'while its base ".*MismatchBase2" does not', + str(excinfo.value), + ) def test_override_static(): @@ -191,20 +263,20 @@ class SubAliased(m.AliasedHasOpNewDelSize): a = m.HasOpNewDel() b = m.HasOpNewDelSize() d = m.HasOpNewDelBoth() - assert capture == """ + assert ( + capture + == """ A new 8 B new 4 D new 32 """ + ) sz_alias = str(m.AliasedHasOpNewDelSize.size_alias) sz_noalias = str(m.AliasedHasOpNewDelSize.size_noalias) with capture: c = m.AliasedHasOpNewDelSize() c2 = SubAliased() - assert capture == ( - "C new " + sz_noalias + "\n" + - "C new " + sz_alias + "\n" - ) + assert capture == ("C new " + sz_noalias + "\n" + "C new " + sz_alias + "\n") with capture: del a @@ -213,21 +285,21 @@ class SubAliased(m.AliasedHasOpNewDelSize): pytest.gc_collect() del d pytest.gc_collect() - assert capture == """ + assert ( + capture + == """ A delete B delete 4 D delete """ + ) with capture: del c pytest.gc_collect() del c2 pytest.gc_collect() - assert capture == ( - "C delete " + sz_noalias + "\n" + - "C delete " + sz_alias + "\n" - ) + assert capture == ("C delete " + sz_noalias + "\n" + "C delete " + sz_alias + "\n") def test_bind_protected_functions(): @@ -262,7 +334,7 @@ def test_brace_initialization(): assert b.vec == [123, 456] -@pytest.unsupported_on_pypy +@pytest.mark.xfail("env.PYPY") def test_class_refcount(): """Instances must correctly increase/decrease the reference count of their types (#1029)""" from sys import getrefcount @@ -297,19 +369,23 @@ def test_reentrant_implicit_conversion_failure(msg): # ensure that there is no runaway reentrant implicit conversion (#1035) with pytest.raises(TypeError) as excinfo: m.BogusImplicitConversion(0) - assert msg(excinfo.value) == ''' + assert ( + msg(excinfo.value) + == """ __init__(): incompatible constructor arguments. The following argument types are supported: 1. m.class_.BogusImplicitConversion(arg0: m.class_.BogusImplicitConversion) Invoked with: 0 - ''' + """ + ) def test_error_after_conversions(): with pytest.raises(TypeError) as exc_info: m.test_error_after_conversions("hello") assert str(exc_info.value).startswith( - "Unable to convert function return value to a Python type!") + "Unable to convert function return value to a Python type!" + ) def test_aligned(): @@ -318,21 +394,25 @@ def test_aligned(): assert p % 1024 == 0 -# https://bitbucket.org/pypy/pypy/issues/2742 -@pytest.unsupported_on_pypy +# https://foss.heptapod.net/pypy/pypy/-/issues/2742 +@pytest.mark.xfail("env.PYPY") def test_final(): with pytest.raises(TypeError) as exc_info: + class PyFinalChild(m.IsFinal): pass + assert str(exc_info.value).endswith("is not an acceptable base type") -# https://bitbucket.org/pypy/pypy/issues/2742 -@pytest.unsupported_on_pypy +# https://foss.heptapod.net/pypy/pypy/-/issues/2742 +@pytest.mark.xfail("env.PYPY") def test_non_final_final(): with pytest.raises(TypeError) as exc_info: + class PyNonFinalFinalChild(m.IsNonFinalFinal): pass + assert str(exc_info.value).endswith("is not an acceptable base type") @@ -342,9 +422,65 @@ def test_exception_rvalue_abort(): m.PyPrintDestructor().throw_something() +# https://github.com/pybind/pybind11/issues/1568 +def test_multiple_instances_with_same_pointer(capture): + n = 100 + instances = [m.SamePointer() for _ in range(n)] + for i in range(n): + # We need to reuse the same allocated memory for with a different type, + # to ensure the bug in `deregister_instance_impl` is detected. Otherwise + # `Py_TYPE(self) == Py_TYPE(it->second)` will still succeed, even though + # the `instance` is already deleted. + instances[i] = m.Empty() + # No assert: if this does not trigger the error + # pybind11_fail("pybind11_object_dealloc(): Tried to deallocate unregistered instance!"); + # and just completes without crashing, we're good. + + +# https://github.com/pybind/pybind11/issues/1624 +def test_base_and_derived_nested_scope(): + assert issubclass(m.DerivedWithNested, m.BaseWithNested) + assert m.BaseWithNested.Nested != m.DerivedWithNested.Nested + assert m.BaseWithNested.Nested.get_name() == "BaseWithNested::Nested" + assert m.DerivedWithNested.Nested.get_name() == "DerivedWithNested::Nested" + + +def test_register_duplicate_class(): + import types + + module_scope = types.ModuleType("module_scope") + with pytest.raises(RuntimeError) as exc_info: + m.register_duplicate_class_name(module_scope) + expected = ( + 'generic_type: cannot initialize type "Duplicate": ' + "an object with that name is already defined" + ) + assert str(exc_info.value) == expected + with pytest.raises(RuntimeError) as exc_info: + m.register_duplicate_class_type(module_scope) + expected = 'generic_type: type "YetAnotherDuplicate" is already registered!' + assert str(exc_info.value) == expected + + class ClassScope: + pass + + with pytest.raises(RuntimeError) as exc_info: + m.register_duplicate_nested_class_name(ClassScope) + expected = ( + 'generic_type: cannot initialize type "DuplicateNested": ' + "an object with that name is already defined" + ) + assert str(exc_info.value) == expected + with pytest.raises(RuntimeError) as exc_info: + m.register_duplicate_nested_class_type(ClassScope) + expected = 'generic_type: type "YetAnotherDuplicateNested" is already registered!' + assert str(exc_info.value) == expected + + @pytest.mark.skip( reason="Generally reproducible in CPython, Python 3, non-debug, on Linux. " - "However, hard to pin this down for CI.") + "However, hard to pin this down for CI." +) def test_1922(): # Test #1922 (drake#11424). # Define a derived class which *does not* overload the method. diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 53228f0eb4..8bfaa386ae 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -1,23 +1,37 @@ +# Built-in in CMake 3.5+ +include(CMakeParseArguments) + add_custom_target(test_cmake_build) function(pybind11_add_build_test name) cmake_parse_arguments(ARG "INSTALL" "" "" ${ARGN}) - set(build_options - "-DCMAKE_PREFIX_PATH=${pybind11_BINARY_DIR}/mock_install" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" - "-DPYTHON_EXECUTABLE:FILEPATH=${PYTHON_EXECUTABLE}") + set(build_options "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") + + if(PYBIND11_FINDPYTHON) + list(APPEND build_options "-DPYBIND11_FINDPYTHON=${PYBIND11_FINDPYTHON}") - if(CMAKE_CXX_STANDARD) + if(DEFINED Python_ROOT_DIR) + list(APPEND build_options "-DPython_ROOT_DIR=${Python_ROOT_DIR}") + endif() + + list(APPEND build_options "-DPython_EXECUTABLE=${Python_EXECUTABLE}") + else() + list(APPEND build_options "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}") + endif() + + if(DEFINED CMAKE_CXX_STANDARD) list(APPEND build_options "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}") endif() if(NOT ARG_INSTALL) - list(APPEND build_options "-DPYBIND11_PROJECT_DIR=${pybind11_SOURCE_DIR}") + list(APPEND build_options "-Dpybind11_SOURCE_DIR=${pybind11_SOURCE_DIR}") + else() + list(APPEND build_options "-DCMAKE_PREFIX_PATH=${pybind11_BINARY_DIR}/mock_install") endif() add_custom_target( - test_${name} + test_build_${name} ${CMAKE_CTEST_COMMAND} --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/${name}" @@ -32,18 +46,22 @@ function(pybind11_add_build_test name) --build-makeprogram ${CMAKE_MAKE_PROGRAM} --build-target - check + check_${name} --build-options ${build_options}) if(ARG_INSTALL) - add_dependencies(test_${name} mock_install) + add_dependencies(test_build_${name} mock_install) endif() - add_dependencies(test_cmake_build test_${name}) + add_dependencies(test_cmake_build test_build_${name}) endfunction() +possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) + pybind11_add_build_test(subdirectory_function) pybind11_add_build_test(subdirectory_target) -if(NOT ${PYTHON_MODULE_EXTENSION} MATCHES "pypy") +if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy") + message(STATUS "Skipping embed test on PyPy") +else() pybind11_add_build_test(subdirectory_embed) endif() @@ -54,9 +72,13 @@ if(PYBIND11_INSTALL) pybind11_add_build_test(installed_function INSTALL) pybind11_add_build_test(installed_target INSTALL) - if(NOT ${PYTHON_MODULE_EXTENSION} MATCHES "pypy") + if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + )) pybind11_add_build_test(installed_embed INSTALL) endif() endif() add_dependencies(check test_cmake_build) + +add_subdirectory(subdirectory_target EXCLUDE_FROM_ALL) +add_subdirectory(subdirectory_embed EXCLUDE_FROM_ALL) diff --git a/tests/test_cmake_build/embed.cpp b/tests/test_cmake_build/embed.cpp index b9581d2fdb..a3abc8a84d 100644 --- a/tests/test_cmake_build/embed.cpp +++ b/tests/test_cmake_build/embed.cpp @@ -12,10 +12,10 @@ int main(int argc, char *argv[]) { py::scoped_interpreter guard{}; - auto m = py::module::import("test_cmake_build"); + auto m = py::module_::import("test_cmake_build"); if (m.attr("add")(1, 2).cast() != 3) throw std::runtime_error("embed.cpp failed"); - py::module::import("sys").attr("argv") = py::make_tuple("test.py", "embed.cpp"); + py::module_::import("sys").attr("argv") = py::make_tuple("test.py", "embed.cpp"); py::eval_file(test_py_file, py::globals()); } diff --git a/tests/test_cmake_build/installed_embed/CMakeLists.txt b/tests/test_cmake_build/installed_embed/CMakeLists.txt index 78855afa26..64ae5c4bff 100644 --- a/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,15 +1,26 @@ -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.4) + +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.18) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.18) +endif() + project(test_installed_embed CXX) -set(CMAKE_MODULE_PATH "") find_package(pybind11 CONFIG REQUIRED) message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") -add_executable(test_cmake_build ../embed.cpp) -target_link_libraries(test_cmake_build PRIVATE pybind11::embed) +add_executable(test_installed_embed ../embed.cpp) +target_link_libraries(test_installed_embed PRIVATE pybind11::embed) +set_target_properties(test_installed_embed PROPERTIES OUTPUT_NAME test_cmake_build) # Do not treat includes from IMPORTED target as SYSTEM (Python headers in pybind11::embed). # This may be needed to resolve header conflicts, e.g. between Python release and debug headers. -set_target_properties(test_cmake_build PROPERTIES NO_SYSTEM_FROM_IMPORTED ON) +set_target_properties(test_installed_embed PROPERTIES NO_SYSTEM_FROM_IMPORTED ON) -add_custom_target(check $ ${PROJECT_SOURCE_DIR}/../test.py) +add_custom_target(check_installed_embed $ + ${PROJECT_SOURCE_DIR}/../test.py) diff --git a/tests/test_cmake_build/installed_function/CMakeLists.txt b/tests/test_cmake_build/installed_function/CMakeLists.txt index 3ad5445e3f..1a502863c0 100644 --- a/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,19 +1,38 @@ -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.4) project(test_installed_module CXX) -set(CMAKE_MODULE_PATH "") +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.18) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.18) +endif() + +project(test_installed_function CXX) find_package(pybind11 CONFIG REQUIRED) -message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") +message( + STATUS "Found pybind11 v${pybind11_VERSION} ${pybind11_VERSION_TYPE}: ${pybind11_INCLUDE_DIRS}") + +pybind11_add_module(test_installed_function SHARED NO_EXTRAS ../main.cpp) +set_target_properties(test_installed_function PROPERTIES OUTPUT_NAME test_cmake_build) -pybind11_add_module(test_cmake_build SHARED NO_EXTRAS ../main.cpp) +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() add_custom_target( - check + check_installed_function ${CMAKE_COMMAND} -E env - PYTHONPATH=$ - ${PYTHON_EXECUTABLE} + PYTHONPATH=$ + ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/tests/test_cmake_build/installed_target/CMakeLists.txt b/tests/test_cmake_build/installed_target/CMakeLists.txt index 348c419cd3..b38eb77470 100644 --- a/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,29 +1,45 @@ -cmake_minimum_required(VERSION 3.7) -project(test_installed_target CXX) +cmake_minimum_required(VERSION 3.4) + +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.18) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.18) +endif() -set(CMAKE_MODULE_PATH "") +project(test_installed_target CXX) find_package(pybind11 CONFIG REQUIRED) message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") -add_library(test_cmake_build MODULE ../main.cpp) +add_library(test_installed_target MODULE ../main.cpp) -target_link_libraries(test_cmake_build PRIVATE pybind11::module) +target_link_libraries(test_installed_target PRIVATE pybind11::module) +set_target_properties(test_installed_target PROPERTIES OUTPUT_NAME test_cmake_build) -# make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib -set_target_properties(test_cmake_build PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") +# Make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib +pybind11_extension(test_installed_target) # Do not treat includes from IMPORTED target as SYSTEM (Python headers in pybind11::module). # This may be needed to resolve header conflicts, e.g. between Python release and debug headers. -set_target_properties(test_cmake_build PROPERTIES NO_SYSTEM_FROM_IMPORTED ON) +set_target_properties(test_installed_target PROPERTIES NO_SYSTEM_FROM_IMPORTED ON) + +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() add_custom_target( - check + check_installed_target ${CMAKE_COMMAND} -E env - PYTHONPATH=$ - ${PYTHON_EXECUTABLE} + PYTHONPATH=$ + ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index eea0eeea3d..dfb9cb8f63 100644 --- a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,4 +1,14 @@ -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.4) + +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.18) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.18) +endif() + project(test_subdirectory_embed CXX) set(PYBIND11_INSTALL @@ -6,13 +16,15 @@ set(PYBIND11_INSTALL CACHE BOOL "") set(PYBIND11_EXPORT_NAME test_export) -add_subdirectory(${PYBIND11_PROJECT_DIR} pybind11) +add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) # Test basic target functionality -add_executable(test_cmake_build ../embed.cpp) -target_link_libraries(test_cmake_build PRIVATE pybind11::embed) +add_executable(test_subdirectory_embed ../embed.cpp) +target_link_libraries(test_subdirectory_embed PRIVATE pybind11::embed) +set_target_properties(test_subdirectory_embed PROPERTIES OUTPUT_NAME test_cmake_build) -add_custom_target(check $ ${PROJECT_SOURCE_DIR}/../test.py) +add_custom_target(check_subdirectory_embed $ + "${PROJECT_SOURCE_DIR}/../test.py") # Test custom export group -- PYBIND11_EXPORT_NAME add_library(test_embed_lib ../embed.cpp) diff --git a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index e4518044ed..34aedcf6a8 100644 --- a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,15 +1,34 @@ -cmake_minimum_required(VERSION 3.7) -project(test_subdirectory_module CXX) +cmake_minimum_required(VERSION 3.4) -add_subdirectory(${PYBIND11_PROJECT_DIR} pybind11) -pybind11_add_module(test_cmake_build THIN_LTO ../main.cpp) +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.18) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.18) +endif() + +project(test_subdirectory_function CXX) + +add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) +pybind11_add_module(test_subdirectory_function ../main.cpp) +set_target_properties(test_subdirectory_function PROPERTIES OUTPUT_NAME test_cmake_build) + +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() add_custom_target( - check + check_subdirectory_function ${CMAKE_COMMAND} -E env - PYTHONPATH=$ - ${PYTHON_EXECUTABLE} + PYTHONPATH=$ + ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index f84140ce04..31d862f6ad 100644 --- a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,22 +1,40 @@ -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.4) + +# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.18) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.18) +endif() + project(test_subdirectory_target CXX) -add_subdirectory(${PYBIND11_PROJECT_DIR} pybind11) +add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) + +add_library(test_subdirectory_target MODULE ../main.cpp) +set_target_properties(test_subdirectory_target PROPERTIES OUTPUT_NAME test_cmake_build) -add_library(test_cmake_build MODULE ../main.cpp) +target_link_libraries(test_subdirectory_target PRIVATE pybind11::module) -target_link_libraries(test_cmake_build PRIVATE pybind11::module) +# Make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib +pybind11_extension(test_subdirectory_target) -# make sure result is, for example, test_installed_target.so, not libtest_installed_target.dylib -set_target_properties(test_cmake_build PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() add_custom_target( - check + check_subdirectory_target ${CMAKE_COMMAND} -E env - PYTHONPATH=$ - ${PYTHON_EXECUTABLE} + PYTHONPATH=$ + ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}) diff --git a/tests/test_constants_and_functions.cpp b/tests/test_constants_and_functions.cpp index e8ec74b7bc..8855dd7d7c 100644 --- a/tests/test_constants_and_functions.cpp +++ b/tests/test_constants_and_functions.cpp @@ -46,7 +46,14 @@ std::string print_bytes(py::bytes bytes) { // Test that we properly handle C++17 exception specifiers (which are part of the function signature // in C++17). These should all still work before C++17, but don't affect the function signature. namespace test_exc_sp { +// [workaround(intel)] Unable to use noexcept instead of noexcept(true) +// Make the f1 test basically the same as the f2 test in C++17 mode for the Intel compiler as +// it fails to compile with a plain noexcept (tested with icc (ICC) 2021.1 Beta 20200827). +#if defined(__INTEL_COMPILER) && defined(PYBIND11_CPP17) +int f1(int x) noexcept(true) { return x+1; } +#else int f1(int x) noexcept { return x+1; } +#endif int f2(int x) noexcept(true) { return x+2; } int f3(int x) noexcept(false) { return x+3; } #if defined(__GNUG__) @@ -74,7 +81,7 @@ struct C { # pragma GCC diagnostic pop #endif }; -} +} // namespace test_exc_sp TEST_SUBMODULE(constants_and_functions, m) { @@ -124,4 +131,19 @@ TEST_SUBMODULE(constants_and_functions, m) { m.def("f2", f2); m.def("f3", f3); m.def("f4", f4); + + // test_function_record_leaks + struct LargeCapture { + // This should always be enough to trigger the alternative branch + // where `sizeof(capture) > sizeof(rec->data)` + uint64_t zeros[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + }; + m.def("register_large_capture_with_invalid_arguments", [](py::module_ m) { + LargeCapture capture; // VS 2015's MSVC is acting up if we create the array here + m.def("should_raise", [capture](int) { return capture.zeros[9] + 33; }, py::kw_only(), py::arg()); + }); + m.def("register_with_raising_repr", [](py::module_ m, py::object default_value) { + m.def("should_raise", [](int, int, py::object) { return 42; }, "some docstring", + py::arg_v("x", 42), py::arg_v("y", 42, ""), py::arg_v("z", default_value)); + }); } diff --git a/tests/test_constants_and_functions.py b/tests/test_constants_and_functions.py index 36b1aa64b1..ff13bd0f26 100644 --- a/tests/test_constants_and_functions.py +++ b/tests/test_constants_and_functions.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- -from pybind11_tests import constants_and_functions as m +import pytest + +m = pytest.importorskip("pybind11_tests.constants_and_functions") def test_constants(): @@ -38,3 +40,14 @@ def test_exception_specifiers(): assert m.f2(53) == 55 assert m.f3(86) == 89 assert m.f4(140) == 144 + + +def test_function_record_leaks(): + class RaisingRepr: + def __repr__(self): + raise RuntimeError("Surprise!") + + with pytest.raises(RuntimeError): + m.register_large_capture_with_invalid_arguments(m) + with pytest.raises(RuntimeError): + m.register_with_raising_repr(m, RaisingRepr()) diff --git a/tests/test_copy_move.cpp b/tests/test_copy_move.cpp index 0f698bdf05..322e9bb85d 100644 --- a/tests/test_copy_move.cpp +++ b/tests/test_copy_move.cpp @@ -19,14 +19,14 @@ struct empty { }; struct lacking_copy_ctor : public empty { - lacking_copy_ctor() {} + lacking_copy_ctor() = default; lacking_copy_ctor(const lacking_copy_ctor& other) = delete; }; template <> lacking_copy_ctor empty::instance_ = {}; struct lacking_move_ctor : public empty { - lacking_move_ctor() {} + lacking_move_ctor() = default; lacking_move_ctor(const lacking_move_ctor& other) = delete; lacking_move_ctor(lacking_move_ctor&& other) = delete; }; @@ -116,9 +116,9 @@ TEST_SUBMODULE(copy_move_policies, m) { r += py::cast(o).value; /* moves */ r += py::cast(o).value; /* moves */ r += py::cast(o).value; /* copies */ - MoveOrCopyInt m1(py::cast(o)); /* moves */ - MoveOnlyInt m2(py::cast(o)); /* moves */ - CopyOnlyInt m3(py::cast(o)); /* copies */ + auto m1(py::cast(o)); /* moves */ + auto m2(py::cast(o)); /* moves */ + auto m3(py::cast(o)); /* copies */ r += m1.value + m2.value + m3.value; return r; @@ -175,14 +175,20 @@ TEST_SUBMODULE(copy_move_policies, m) { m.attr("has_optional") = false; #endif - // #70 compilation issue if operator new is not public + // #70 compilation issue if operator new is not public - simple body added + // but not needed on most compilers; MSVC and nvcc don't like a local + // struct not having a method defined when declared, since it can not be + // added later. struct PrivateOpNew { int value = 1; private: -#if defined(_MSC_VER) -# pragma warning(disable: 4822) // warning C4822: local class member function does not have a body -#endif - void *operator new(size_t bytes); + void *operator new(size_t bytes) { + void *ptr = std::malloc(bytes); + if (ptr) + return ptr; + else + throw std::bad_alloc{}; + } }; py::class_(m, "PrivateOpNew").def_readonly("value", &PrivateOpNew::value); m.def("private_op_new_value", []() { return PrivateOpNew(); }); @@ -208,6 +214,7 @@ TEST_SUBMODULE(copy_move_policies, m) { }; py::class_(m, "MoveIssue2").def(py::init()).def_readwrite("value", &MoveIssue2::v); - m.def("get_moveissue1", [](int i) { return new MoveIssue1(i); }, py::return_value_policy::move); + // #2742: Don't expect ownership of raw pointer to `new`ed object to be transferred with `py::return_value_policy::move` + m.def("get_moveissue1", [](int i) { return std::unique_ptr(new MoveIssue1(i)); }, py::return_value_policy::move); m.def("get_moveissue2", [](int i) { return MoveIssue2(i); }, py::return_value_policy::move); } diff --git a/tests/test_copy_move.py b/tests/test_copy_move.py index 6b53993a91..1d98952200 100644 --- a/tests/test_copy_move.py +++ b/tests/test_copy_move.py @@ -19,7 +19,11 @@ def test_move_and_copy_casts(): """Cast some values in C++ via custom type casters and count the number of moves/copies.""" cstats = m.move_and_copy_cstats() - c_m, c_mc, c_c = cstats["MoveOnlyInt"], cstats["MoveOrCopyInt"], cstats["CopyOnlyInt"] + c_m, c_mc, c_c = ( + cstats["MoveOnlyInt"], + cstats["MoveOrCopyInt"], + cstats["CopyOnlyInt"], + ) # The type move constructions/assignments below each get incremented: the move assignment comes # from the type_caster load; the move construction happens when extracting that via a cast or @@ -43,7 +47,11 @@ def test_move_and_copy_loads(): moves/copies.""" cstats = m.move_and_copy_cstats() - c_m, c_mc, c_c = cstats["MoveOnlyInt"], cstats["MoveOrCopyInt"], cstats["CopyOnlyInt"] + c_m, c_mc, c_c = ( + cstats["MoveOnlyInt"], + cstats["MoveOrCopyInt"], + cstats["CopyOnlyInt"], + ) assert m.move_only(10) == 10 # 1 move, c_m assert m.move_or_copy(11) == 11 # 1 move, c_mc @@ -66,12 +74,16 @@ def test_move_and_copy_loads(): assert c_m.alive() + c_mc.alive() + c_c.alive() == 0 -@pytest.mark.skipif(not m.has_optional, reason='no ') +@pytest.mark.skipif(not m.has_optional, reason="no ") def test_move_and_copy_load_optional(): """Tests move/copy loads of std::optional arguments""" cstats = m.move_and_copy_cstats() - c_m, c_mc, c_c = cstats["MoveOnlyInt"], cstats["MoveOrCopyInt"], cstats["CopyOnlyInt"] + c_m, c_mc, c_c = ( + cstats["MoveOnlyInt"], + cstats["MoveOrCopyInt"], + cstats["CopyOnlyInt"], + ) # The extra move/copy constructions below come from the std::optional move (which has to move # its arguments): @@ -107,7 +119,7 @@ def test_private_op_new(): def test_move_fallback(): """#389: rvp::move should fall-through to copy on non-movable objects""" - m2 = m.get_moveissue2(2) - assert m2.value == 2 m1 = m.get_moveissue1(1) assert m1.value == 1 + m2 = m.get_moveissue2(2) + assert m2.value == 2 diff --git a/tests/test_custom_type_casters.cpp b/tests/test_custom_type_casters.cpp index 9485d3cdb2..3fe910d491 100644 --- a/tests/test_custom_type_casters.cpp +++ b/tests/test_custom_type_casters.cpp @@ -58,7 +58,8 @@ template <> struct type_caster { return py::none().release(); } }; -}} +} // namespace detail +} // namespace pybind11 // test_custom_caster_destruction class DestructionTester { @@ -79,7 +80,8 @@ template <> struct type_caster { return py::bool_(true).release(); } }; -}} +} // namespace detail +} // namespace pybind11 TEST_SUBMODULE(custom_type_casters, m) { // test_custom_type_casters @@ -97,19 +99,20 @@ TEST_SUBMODULE(custom_type_casters, m) { } static ArgInspector2 h(ArgInspector2 a, ArgAlwaysConverts) { return a; } }; + // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. py::class_(m, "ArgInspector") .def(py::init<>()) .def("f", &ArgInspector::f, py::arg(), py::arg() = ArgAlwaysConverts()) .def("g", &ArgInspector::g, "a"_a.noconvert(), "b"_a, "c"_a.noconvert()=13, "d"_a=ArgInspector2(), py::arg() = ArgAlwaysConverts()) - .def_static("h", &ArgInspector::h, py::arg().noconvert(), py::arg() = ArgAlwaysConverts()) + .def_static("h", &ArgInspector::h, py::arg{}.noconvert(), py::arg() = ArgAlwaysConverts()) ; m.def("arg_inspect_func", [](ArgInspector2 a, ArgInspector1 b, ArgAlwaysConverts) { return a.arg + "\n" + b.arg; }, - py::arg().noconvert(false), py::arg_v(nullptr, ArgInspector1()).noconvert(true), py::arg() = ArgAlwaysConverts()); + py::arg{}.noconvert(false), py::arg_v(nullptr, ArgInspector1()).noconvert(true), py::arg() = ArgAlwaysConverts()); - m.def("floats_preferred", [](double f) { return 0.5 * f; }, py::arg("f")); - m.def("floats_only", [](double f) { return 0.5 * f; }, py::arg("f").noconvert()); - m.def("ints_preferred", [](int i) { return i / 2; }, py::arg("i")); - m.def("ints_only", [](int i) { return i / 2; }, py::arg("i").noconvert()); + m.def("floats_preferred", [](double f) { return 0.5 * f; }, "f"_a); + m.def("floats_only", [](double f) { return 0.5 * f; }, "f"_a.noconvert()); + m.def("ints_preferred", [](int i) { return i / 2; }, "i"_a); + m.def("ints_only", [](int i) { return i / 2; }, "i"_a.noconvert()); // test_custom_caster_destruction // Test that `take_ownership` works on types with a custom type caster when given a pointer diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index 9475c45168..bb74d54eb3 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -5,65 +5,91 @@ def test_noconvert_args(msg): a = m.ArgInspector() - assert msg(a.f("hi")) == """ + assert ( + msg(a.f("hi")) + == """ loading ArgInspector1 argument WITH conversion allowed. Argument value = hi """ - assert msg(a.g("this is a", "this is b")) == """ + ) + assert ( + msg(a.g("this is a", "this is b")) + == """ loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = this is a loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b 13 loading ArgInspector2 argument WITH conversion allowed. Argument value = (default arg inspector 2) """ # noqa: E501 line too long - assert msg(a.g("this is a", "this is b", 42)) == """ + ) + assert ( + msg(a.g("this is a", "this is b", 42)) + == """ loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = this is a loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b 42 loading ArgInspector2 argument WITH conversion allowed. Argument value = (default arg inspector 2) """ # noqa: E501 line too long - assert msg(a.g("this is a", "this is b", 42, "this is d")) == """ + ) + assert ( + msg(a.g("this is a", "this is b", 42, "this is d")) + == """ loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = this is a loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b 42 loading ArgInspector2 argument WITH conversion allowed. Argument value = this is d """ - assert (a.h("arg 1") == - "loading ArgInspector2 argument WITHOUT conversion allowed. Argument value = arg 1") - assert msg(m.arg_inspect_func("A1", "A2")) == """ + ) + assert ( + a.h("arg 1") + == "loading ArgInspector2 argument WITHOUT conversion allowed. Argument value = arg 1" + ) + assert ( + msg(m.arg_inspect_func("A1", "A2")) + == """ loading ArgInspector2 argument WITH conversion allowed. Argument value = A1 loading ArgInspector1 argument WITHOUT conversion allowed. Argument value = A2 """ + ) assert m.floats_preferred(4) == 2.0 assert m.floats_only(4.0) == 2.0 with pytest.raises(TypeError) as excinfo: m.floats_only(4) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ floats_only(): incompatible function arguments. The following argument types are supported: 1. (f: float) -> float Invoked with: 4 """ + ) assert m.ints_preferred(4) == 2 assert m.ints_preferred(True) == 0 with pytest.raises(TypeError) as excinfo: m.ints_preferred(4.0) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ ints_preferred(): incompatible function arguments. The following argument types are supported: 1. (i: int) -> int Invoked with: 4.0 """ # noqa: E501 line too long + ) assert m.ints_only(4) == 2 with pytest.raises(TypeError) as excinfo: m.ints_only(4.0) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ ints_only(): incompatible function arguments. The following argument types are supported: 1. (i: int) -> int Invoked with: 4.0 """ + ) def test_custom_caster_destruction(): diff --git a/tests/test_docstring_options.cpp b/tests/test_docstring_options.cpp index 8c8f79fd5f..8a97af55fc 100644 --- a/tests/test_docstring_options.cpp +++ b/tests/test_docstring_options.cpp @@ -45,6 +45,14 @@ TEST_SUBMODULE(docstring_options, m) { m.def("test_function7", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + { + py::options options; + options.disable_user_defined_docstrings(); + options.disable_function_signatures(); + + m.def("test_function8", []() {}); + } + { py::options options; options.disable_user_defined_docstrings(); diff --git a/tests/test_docstring_options.py b/tests/test_docstring_options.py index 80ade0f158..8ee6613884 100644 --- a/tests/test_docstring_options.py +++ b/tests/test_docstring_options.py @@ -18,10 +18,10 @@ def test_docstring_options(): assert m.test_overloaded3.__doc__ == "Overload docstr" # options.enable_function_signatures() - assert m.test_function3.__doc__ .startswith("test_function3(a: int, b: int) -> None") + assert m.test_function3.__doc__.startswith("test_function3(a: int, b: int) -> None") - assert m.test_function4.__doc__ .startswith("test_function4(a: int, b: int) -> None") - assert m.test_function4.__doc__ .endswith("A custom docstring\n") + assert m.test_function4.__doc__.startswith("test_function4(a: int, b: int) -> None") + assert m.test_function4.__doc__.endswith("A custom docstring\n") # options.disable_function_signatures() # options.disable_user_defined_docstrings() @@ -31,8 +31,11 @@ def test_docstring_options(): assert m.test_function6.__doc__ == "A custom docstring" # RAII destructor - assert m.test_function7.__doc__ .startswith("test_function7(a: int, b: int) -> None") - assert m.test_function7.__doc__ .endswith("A custom docstring\n") + assert m.test_function7.__doc__.startswith("test_function7(a: int, b: int) -> None") + assert m.test_function7.__doc__.endswith("A custom docstring\n") + + # when all options are disabled, no docstring (instead of an empty one) should be generated + assert m.test_function8.__doc__ is None # Suppression of user-defined docstrings for non-function objects assert not m.DocstringTestFoo.__doc__ diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index 38673fb5a4..baff69a126 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -81,8 +81,9 @@ double get_elem(Eigen::Ref m) { return m(2, 1); }; // reference is referencing rows/columns correctly). template Eigen::MatrixXd adjust_matrix(MatrixArgType m) { Eigen::MatrixXd ret(m); - for (int c = 0; c < m.cols(); c++) for (int r = 0; r < m.rows(); r++) - ret(r, c) += 10*r + 100*c; + for (int c = 0; c < m.cols(); c++) + for (int r = 0; r < m.rows(); r++) + ret(r, c) += 10*r + 100*c; // NOLINT(clang-analyzer-core.uninitialized.Assign) return ret; } @@ -109,8 +110,6 @@ TEST_SUBMODULE(eigen, m) { using SparseMatrixR = Eigen::SparseMatrix; using SparseMatrixC = Eigen::SparseMatrix; - m.attr("have_eigen") = true; - // various tests m.def("double_col", [](const Eigen::VectorXf &x) -> Eigen::VectorXf { return 2.0f * x; }); m.def("double_adscalar_col", [](const VectorXADScalar &x) -> VectorXADScalar { return 2.0f * x; }); @@ -330,7 +329,7 @@ TEST_SUBMODULE(eigen, m) { m.def("dense_copy_r", [](const DenseMatrixR &m) -> DenseMatrixR { return m; }); m.def("dense_copy_c", [](const DenseMatrixC &m) -> DenseMatrixC { return m; }); // test_sparse, test_sparse_signature - m.def("sparse_r", [mat]() -> SparseMatrixR { return Eigen::SparseView(mat); }); + m.def("sparse_r", [mat]() -> SparseMatrixR { return Eigen::SparseView(mat); }); //NOLINT(clang-analyzer-core.uninitialized.UndefReturn) m.def("sparse_c", [mat]() -> SparseMatrixC { return Eigen::SparseView(mat); }); m.def("sparse_copy_r", [](const SparseMatrixR &m) -> SparseMatrixR { return m; }); m.def("sparse_copy_c", [](const SparseMatrixC &m) -> SparseMatrixC { return m; }); @@ -347,6 +346,7 @@ TEST_SUBMODULE(eigen, m) { m.def("cpp_ref_r", [](py::handle m) { return m.cast>()(1, 0); }); m.def("cpp_ref_any", [](py::handle m) { return m.cast>()(1, 0); }); + // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. // test_nocopy_wrapper // Test that we can prevent copying into an argument that would normally copy: First a version @@ -354,10 +354,10 @@ TEST_SUBMODULE(eigen, m) { m.def("get_elem", &get_elem); // Now this alternative that calls the tells pybind to fail rather than copy: m.def("get_elem_nocopy", [](Eigen::Ref m) -> double { return get_elem(m); }, - py::arg().noconvert()); + py::arg{}.noconvert()); // Also test a row-major-only no-copy const ref: m.def("get_elem_rm_nocopy", [](Eigen::Ref> &m) -> long { return m(2, 1); }, - py::arg().noconvert()); + py::arg{}.noconvert()); // test_issue738_issue2038 // Issue #738: 1xN or Nx1 2D matrices were neither accepted nor properly copied with an @@ -408,11 +408,11 @@ TEST_SUBMODULE(eigen, m) { // a new array (np.ones(10)) increases the chances that the temp array will be garbage // collected and/or that its memory will be overridden with different values. m.def("get_elem_direct", [](Eigen::Ref v) { - py::module::import("numpy").attr("ones")(10); + py::module_::import("numpy").attr("ones")(10); return v(5); }); m.def("get_elem_indirect", [](std::vector> v) { - py::module::import("numpy").attr("ones")(10); + py::module_::import("numpy").attr("ones")(10); return v[0](5); }); } diff --git a/tests/test_eigen.py b/tests/test_eigen.py index 76191f6b70..0c336deaf0 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen.py @@ -2,17 +2,19 @@ import pytest from pybind11_tests import ConstructorStats -pytestmark = pytest.requires_eigen_and_numpy +np = pytest.importorskip("numpy") +m = pytest.importorskip("pybind11_tests.eigen") -with pytest.suppress(ImportError): - from pybind11_tests import eigen as m - import numpy as np - ref = np.array([[ 0., 3, 0, 0, 0, 11], - [22, 0, 0, 0, 17, 11], - [ 7, 5, 0, 1, 0, 11], - [ 0, 0, 0, 0, 0, 11], - [ 0, 0, 14, 0, 8, 11]]) +ref = np.array( + [ + [0.0, 3, 0, 0, 0, 11], + [22, 0, 0, 0, 17, 11], + [7, 5, 0, 1, 0, 11], + [0, 0, 0, 0, 0, 11], + [0, 0, 14, 0, 8, 11], + ] +) def assert_equal_ref(mat): @@ -42,28 +44,37 @@ def test_dense(): def test_partially_fixed(): - ref2 = np.array([[0., 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]]) + ref2 = np.array([[0.0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]]) np.testing.assert_array_equal(m.partial_copy_four_rm_r(ref2), ref2) np.testing.assert_array_equal(m.partial_copy_four_rm_c(ref2), ref2) np.testing.assert_array_equal(m.partial_copy_four_rm_r(ref2[:, 1]), ref2[:, [1]]) np.testing.assert_array_equal(m.partial_copy_four_rm_c(ref2[0, :]), ref2[[0], :]) - np.testing.assert_array_equal(m.partial_copy_four_rm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)]) np.testing.assert_array_equal( - m.partial_copy_four_rm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :]) + m.partial_copy_four_rm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)] + ) + np.testing.assert_array_equal( + m.partial_copy_four_rm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :] + ) np.testing.assert_array_equal(m.partial_copy_four_cm_r(ref2), ref2) np.testing.assert_array_equal(m.partial_copy_four_cm_c(ref2), ref2) np.testing.assert_array_equal(m.partial_copy_four_cm_r(ref2[:, 1]), ref2[:, [1]]) np.testing.assert_array_equal(m.partial_copy_four_cm_c(ref2[0, :]), ref2[[0], :]) - np.testing.assert_array_equal(m.partial_copy_four_cm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)]) np.testing.assert_array_equal( - m.partial_copy_four_cm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :]) + m.partial_copy_four_cm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)] + ) + np.testing.assert_array_equal( + m.partial_copy_four_cm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :] + ) # TypeError should be raise for a shape mismatch - functions = [m.partial_copy_four_rm_r, m.partial_copy_four_rm_c, - m.partial_copy_four_cm_r, m.partial_copy_four_cm_c] - matrix_with_wrong_shape = [[1, 2], - [3, 4]] + functions = [ + m.partial_copy_four_rm_r, + m.partial_copy_four_rm_c, + m.partial_copy_four_cm_r, + m.partial_copy_four_cm_c, + ] + matrix_with_wrong_shape = [[1, 2], [3, 4]] for f in functions: with pytest.raises(TypeError) as excinfo: f(matrix_with_wrong_shape) @@ -71,7 +82,7 @@ def test_partially_fixed(): def test_mutator_descriptors(): - zr = np.arange(30, dtype='float32').reshape(5, 6) # row-major + zr = np.arange(30, dtype="float32").reshape(5, 6) # row-major zc = zr.reshape(6, 5).transpose() # column-major m.fixed_mutator_r(zr) @@ -80,18 +91,21 @@ def test_mutator_descriptors(): m.fixed_mutator_a(zc) with pytest.raises(TypeError) as excinfo: m.fixed_mutator_r(zc) - assert ('(arg0: numpy.ndarray[numpy.float32[5, 6],' - ' flags.writeable, flags.c_contiguous]) -> None' - in str(excinfo.value)) + assert ( + "(arg0: numpy.ndarray[numpy.float32[5, 6]," + " flags.writeable, flags.c_contiguous]) -> None" in str(excinfo.value) + ) with pytest.raises(TypeError) as excinfo: m.fixed_mutator_c(zr) - assert ('(arg0: numpy.ndarray[numpy.float32[5, 6],' - ' flags.writeable, flags.f_contiguous]) -> None' - in str(excinfo.value)) + assert ( + "(arg0: numpy.ndarray[numpy.float32[5, 6]," + " flags.writeable, flags.f_contiguous]) -> None" in str(excinfo.value) + ) with pytest.raises(TypeError) as excinfo: - m.fixed_mutator_a(np.array([[1, 2], [3, 4]], dtype='float32')) - assert ('(arg0: numpy.ndarray[numpy.float32[5, 6], flags.writeable]) -> None' - in str(excinfo.value)) + m.fixed_mutator_a(np.array([[1, 2], [3, 4]], dtype="float32")) + assert "(arg0: numpy.ndarray[numpy.float32[5, 6], flags.writeable]) -> None" in str( + excinfo.value + ) zr.flags.writeable = False with pytest.raises(TypeError): m.fixed_mutator_r(zr) @@ -100,26 +114,26 @@ def test_mutator_descriptors(): def test_cpp_casting(): - assert m.cpp_copy(m.fixed_r()) == 22. - assert m.cpp_copy(m.fixed_c()) == 22. - z = np.array([[5., 6], [7, 8]]) - assert m.cpp_copy(z) == 7. - assert m.cpp_copy(m.get_cm_ref()) == 21. - assert m.cpp_copy(m.get_rm_ref()) == 21. - assert m.cpp_ref_c(m.get_cm_ref()) == 21. - assert m.cpp_ref_r(m.get_rm_ref()) == 21. + assert m.cpp_copy(m.fixed_r()) == 22.0 + assert m.cpp_copy(m.fixed_c()) == 22.0 + z = np.array([[5.0, 6], [7, 8]]) + assert m.cpp_copy(z) == 7.0 + assert m.cpp_copy(m.get_cm_ref()) == 21.0 + assert m.cpp_copy(m.get_rm_ref()) == 21.0 + assert m.cpp_ref_c(m.get_cm_ref()) == 21.0 + assert m.cpp_ref_r(m.get_rm_ref()) == 21.0 with pytest.raises(RuntimeError) as excinfo: # Can't reference m.fixed_c: it contains floats, m.cpp_ref_any wants doubles m.cpp_ref_any(m.fixed_c()) - assert 'Unable to cast Python instance' in str(excinfo.value) + assert "Unable to cast Python instance" in str(excinfo.value) with pytest.raises(RuntimeError) as excinfo: # Can't reference m.fixed_r: it contains floats, m.cpp_ref_any wants doubles m.cpp_ref_any(m.fixed_r()) - assert 'Unable to cast Python instance' in str(excinfo.value) - assert m.cpp_ref_any(m.ReturnTester.create()) == 1. + assert "Unable to cast Python instance" in str(excinfo.value) + assert m.cpp_ref_any(m.ReturnTester.create()) == 1.0 - assert m.cpp_ref_any(m.get_cm_ref()) == 21. - assert m.cpp_ref_any(m.get_cm_ref()) == 21. + assert m.cpp_ref_any(m.get_cm_ref()) == 21.0 + assert m.cpp_ref_any(m.get_cm_ref()) == 21.0 def test_pass_readonly_array(): @@ -151,7 +165,7 @@ def test_nonunit_stride_from_python(): # Mutator: m.double_threer(second_row) m.double_threec(second_col) - np.testing.assert_array_equal(counting_mat, [[0., 2, 2], [6, 16, 10], [6, 14, 8]]) + np.testing.assert_array_equal(counting_mat, [[0.0, 2, 2], [6, 16, 10], [6, 14, 8]]) def float_to_adscalar(arr, deriv): @@ -176,26 +190,26 @@ def check_array(a, b): def test_eigen_passing_adscalar(): - assert m.equal_to(1., 1.) - assert not m.equal_to(1., 1.1) - assert m.equal_to(m.AutoDiffXd(0, [1.]), m.AutoDiffXd(0, [1.])) - assert not m.equal_to(m.AutoDiffXd(0, [1.]), m.AutoDiffXd(0, [1.1])) + assert m.equal_to(1.0, 1.0) + assert not m.equal_to(1.0, 1.1) + assert m.equal_to(m.AutoDiffXd(0, [1.0]), m.AutoDiffXd(0, [1.0])) + assert not m.equal_to(m.AutoDiffXd(0, [1.0]), m.AutoDiffXd(0, [1.1])) - adscalar_mat = float_to_adscalar(ref, deriv=[1.]) + adscalar_mat = float_to_adscalar(ref, deriv=[1.0]) adscalar_vec_col = adscalar_mat[:, 0] adscalar_vec_row = adscalar_mat[0, :] # Checking if a Python vector is getting doubled, when passed into a dynamic or fixed # row or col vector in Eigen. - double_adscalar_mat = float_to_adscalar(2 * ref, deriv=[2.]) + double_adscalar_mat = float_to_adscalar(2 * ref, deriv=[2.0]) check_array(m.double_adscalar_col(adscalar_vec_col), double_adscalar_mat[:, 0]) check_array(m.double_adscalar_col5(adscalar_vec_col), double_adscalar_mat[:, 0]) check_array(m.double_adscalar_row(adscalar_vec_row), double_adscalar_mat[0, :]) check_array(m.double_adscalar_row6(adscalar_vec_row), double_adscalar_mat[0, :]) # Adding 7 to the a dynamic matrix using reference. - incr_adscalar_mat = float_to_adscalar(ref + 7, deriv=[1.]) - check_array(m.incr_adscalar_matrix(adscalar_mat, 7.), incr_adscalar_mat) + incr_adscalar_mat = float_to_adscalar(ref + 7, deriv=[1.0]) + check_array(m.incr_adscalar_matrix(adscalar_mat, 7.0), incr_adscalar_mat) # The original adscalar_mat remains unchanged in spite of passing by reference, since # `Eigen::Ref` permits copying, and copying is the only valid operation for # `dtype=object`. @@ -272,26 +286,36 @@ def test_negative_stride_from_python(msg): # Mutator: with pytest.raises(TypeError) as excinfo: m.double_threer(second_row) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ double_threer(): incompatible function arguments. The following argument types are supported: 1. (arg0: numpy.ndarray[numpy.float32[1, 3], flags.writeable]) -> None - Invoked with: """ + repr(np.array([ 5., 4., 3.], dtype='float32')) # noqa: E501 line too long + Invoked with: """ # noqa: E501 line too long + + repr(np.array([5.0, 4.0, 3.0], dtype="float32")) + ) with pytest.raises(TypeError) as excinfo: m.double_threec(second_col) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ double_threec(): incompatible function arguments. The following argument types are supported: 1. (arg0: numpy.ndarray[numpy.float32[3, 1], flags.writeable]) -> None - Invoked with: """ + repr(np.array([ 7., 4., 1.], dtype='float32')) # noqa: E501 line too long + Invoked with: """ # noqa: E501 line too long + + repr(np.array([7.0, 4.0, 1.0], dtype="float32")) + ) def test_nonunit_stride_to_python(): assert np.all(m.diagonal(ref) == ref.diagonal()) assert np.all(m.diagonal_1(ref) == ref.diagonal(1)) for i in range(-5, 7): - assert np.all(m.diagonal_n(ref, i) == ref.diagonal(i)), "m.diagonal_n({})".format(i) + assert np.all( + m.diagonal_n(ref, i) == ref.diagonal(i) + ), "m.diagonal_n({})".format(i) assert np.all(m.block(ref, 2, 1, 3, 3) == ref[2:5, 1:4]) assert np.all(m.block(ref, 1, 4, 4, 2) == ref[1:, 4:]) @@ -301,8 +325,10 @@ def test_nonunit_stride_to_python(): def test_eigen_ref_to_python(): chols = [m.cholesky1, m.cholesky2, m.cholesky3, m.cholesky4] for i, chol in enumerate(chols, start=1): - mymat = chol(np.array([[1., 2, 4], [2, 13, 23], [4, 23, 77]])) - assert np.all(mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]])), "cholesky{}".format(i) + mymat = chol(np.array([[1.0, 2, 4], [2, 13, 23], [4, 23, 77]])) + assert np.all( + mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]]) + ), "cholesky{}".format(i) def assign_both(a1, a2, r, c, v): @@ -419,8 +445,12 @@ def test_eigen_return_references(): np.testing.assert_array_equal(a_block1, master[3:5, 3:5]) np.testing.assert_array_equal(a_block2, master[2:5, 2:4]) np.testing.assert_array_equal(a_block3, master[6:10, 7:10]) - np.testing.assert_array_equal(a_corn1, master[0::master.shape[0] - 1, 0::master.shape[1] - 1]) - np.testing.assert_array_equal(a_corn2, master[0::master.shape[0] - 1, 0::master.shape[1] - 1]) + np.testing.assert_array_equal( + a_corn1, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1] + ) + np.testing.assert_array_equal( + a_corn2, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1] + ) np.testing.assert_array_equal(a_copy1, c1want) np.testing.assert_array_equal(a_copy2, c2want) @@ -449,16 +479,28 @@ def test_eigen_keepalive(): cstats = ConstructorStats.get(m.ReturnTester) assert cstats.alive() == 1 unsafe = [a.ref(), a.ref_const(), a.block(1, 2, 3, 4)] - copies = [a.copy_get(), a.copy_view(), a.copy_ref(), a.copy_ref_const(), - a.copy_block(4, 3, 2, 1)] + copies = [ + a.copy_get(), + a.copy_view(), + a.copy_ref(), + a.copy_ref_const(), + a.copy_block(4, 3, 2, 1), + ] del a assert cstats.alive() == 0 del unsafe del copies - for meth in [m.ReturnTester.get, m.ReturnTester.get_ptr, m.ReturnTester.view, - m.ReturnTester.view_ptr, m.ReturnTester.ref_safe, m.ReturnTester.ref_const_safe, - m.ReturnTester.corners, m.ReturnTester.corners_const]: + for meth in [ + m.ReturnTester.get, + m.ReturnTester.get_ptr, + m.ReturnTester.view, + m.ReturnTester.view_ptr, + m.ReturnTester.ref_safe, + m.ReturnTester.ref_const_safe, + m.ReturnTester.corners, + m.ReturnTester.corners_const, + ]: assert_keeps_alive(m.ReturnTester, meth) for meth in [m.ReturnTester.block_safe, m.ReturnTester.block_const]: @@ -468,18 +510,18 @@ def test_eigen_keepalive(): def test_eigen_ref_mutators(): """Tests Eigen's ability to mutate numpy values""" - orig = np.array([[1., 2, 3], [4, 5, 6], [7, 8, 9]]) + orig = np.array([[1.0, 2, 3], [4, 5, 6], [7, 8, 9]]) zr = np.array(orig) - zc = np.array(orig, order='F') + zc = np.array(orig, order="F") m.add_rm(zr, 1, 0, 100) - assert np.all(zr == np.array([[1., 2, 3], [104, 5, 6], [7, 8, 9]])) + assert np.all(zr == np.array([[1.0, 2, 3], [104, 5, 6], [7, 8, 9]])) m.add_cm(zc, 1, 0, 200) - assert np.all(zc == np.array([[1., 2, 3], [204, 5, 6], [7, 8, 9]])) + assert np.all(zc == np.array([[1.0, 2, 3], [204, 5, 6], [7, 8, 9]])) m.add_any(zr, 1, 0, 20) - assert np.all(zr == np.array([[1., 2, 3], [124, 5, 6], [7, 8, 9]])) + assert np.all(zr == np.array([[1.0, 2, 3], [124, 5, 6], [7, 8, 9]])) m.add_any(zc, 1, 0, 10) - assert np.all(zc == np.array([[1., 2, 3], [214, 5, 6], [7, 8, 9]])) + assert np.all(zc == np.array([[1.0, 2, 3], [214, 5, 6], [7, 8, 9]])) # Can't reference a col-major array with a row-major Ref, and vice versa: with pytest.raises(TypeError): @@ -500,8 +542,8 @@ def test_eigen_ref_mutators(): cornersr = zr[0::2, 0::2] cornersc = zc[0::2, 0::2] - assert np.all(cornersr == np.array([[1., 3], [7, 9]])) - assert np.all(cornersc == np.array([[1., 3], [7, 9]])) + assert np.all(cornersr == np.array([[1.0, 3], [7, 9]])) + assert np.all(cornersc == np.array([[1.0, 3], [7, 9]])) with pytest.raises(TypeError): m.add_rm(cornersr, 0, 1, 25) @@ -513,8 +555,8 @@ def test_eigen_ref_mutators(): m.add_cm(cornersc, 0, 1, 25) m.add_any(cornersr, 0, 1, 25) m.add_any(cornersc, 0, 1, 44) - assert np.all(zr == np.array([[1., 2, 28], [4, 5, 6], [7, 8, 9]])) - assert np.all(zc == np.array([[1., 2, 47], [4, 5, 6], [7, 8, 9]])) + assert np.all(zr == np.array([[1.0, 2, 28], [4, 5, 6], [7, 8, 9]])) + assert np.all(zc == np.array([[1.0, 2, 47], [4, 5, 6], [7, 8, 9]])) # You shouldn't be allowed to pass a non-writeable array to a mutating Eigen method: zro = zr[0:4, 0:4] @@ -552,7 +594,7 @@ def test_numpy_ref_mutators(): assert not zrro.flags.owndata and not zrro.flags.writeable zc[1, 2] = 99 - expect = np.array([[11., 12, 13], [21, 22, 99], [31, 32, 33]]) + expect = np.array([[11.0, 12, 13], [21, 22, 99], [31, 32, 33]]) # We should have just changed zc, of course, but also zcro and the original eigen matrix assert np.all(zc == expect) assert np.all(zcro == expect) @@ -600,18 +642,20 @@ def test_both_ref_mutators(): assert np.all(z == z3) assert np.all(z == z4) assert np.all(z == z5) - expect = np.array([[0., 22, 20], [31, 37, 33], [41, 42, 38]]) + expect = np.array([[0.0, 22, 20], [31, 37, 33], [41, 42, 38]]) assert np.all(z == expect) - y = np.array(range(100), dtype='float64').reshape(10, 10) + y = np.array(range(100), dtype="float64").reshape(10, 10) y2 = m.incr_matrix_any(y, 10) # np -> eigen -> np - y3 = m.incr_matrix_any(y2[0::2, 0::2], -33) # np -> eigen -> np slice -> np -> eigen -> np + y3 = m.incr_matrix_any( + y2[0::2, 0::2], -33 + ) # np -> eigen -> np slice -> np -> eigen -> np y4 = m.even_rows(y3) # numpy -> eigen slice -> (... y3) y5 = m.even_cols(y4) # numpy -> eigen slice -> (... y4) y6 = m.incr_matrix_any(y5, 1000) # numpy -> eigen -> (... y5) # Apply same mutations using just numpy: - yexpect = np.array(range(100), dtype='float64').reshape(10, 10) + yexpect = np.array(range(100), dtype="float64").reshape(10, 10) yexpect += 10 yexpect[0::2, 0::2] -= 33 yexpect[0::4, 0::4] += 1000 @@ -626,10 +670,14 @@ def test_both_ref_mutators(): def test_nocopy_wrapper(): # get_elem requires a column-contiguous matrix reference, but should be # callable with other types of matrix (via copying): - int_matrix_colmajor = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], order='F') - dbl_matrix_colmajor = np.array(int_matrix_colmajor, dtype='double', order='F', copy=True) - int_matrix_rowmajor = np.array(int_matrix_colmajor, order='C', copy=True) - dbl_matrix_rowmajor = np.array(int_matrix_rowmajor, dtype='double', order='C', copy=True) + int_matrix_colmajor = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], order="F") + dbl_matrix_colmajor = np.array( + int_matrix_colmajor, dtype="double", order="F", copy=True + ) + int_matrix_rowmajor = np.array(int_matrix_colmajor, order="C", copy=True) + dbl_matrix_rowmajor = np.array( + int_matrix_rowmajor, dtype="double", order="C", copy=True + ) # All should be callable via get_elem: assert m.get_elem(int_matrix_colmajor) == 8 @@ -640,32 +688,38 @@ def test_nocopy_wrapper(): # All but the second should fail with m.get_elem_nocopy: with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_colmajor) - assert ('get_elem_nocopy(): incompatible function arguments.' in str(excinfo.value) and - ', flags.f_contiguous' in str(excinfo.value)) + assert "get_elem_nocopy(): incompatible function arguments." in str( + excinfo.value + ) and ", flags.f_contiguous" in str(excinfo.value) assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_rowmajor) - assert ('get_elem_nocopy(): incompatible function arguments.' in str(excinfo.value) and - ', flags.f_contiguous' in str(excinfo.value)) + assert "get_elem_nocopy(): incompatible function arguments." in str( + excinfo.value + ) and ", flags.f_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(dbl_matrix_rowmajor) - assert ('get_elem_nocopy(): incompatible function arguments.' in str(excinfo.value) and - ', flags.f_contiguous' in str(excinfo.value)) + assert "get_elem_nocopy(): incompatible function arguments." in str( + excinfo.value + ) and ", flags.f_contiguous" in str(excinfo.value) # For the row-major test, we take a long matrix in row-major, so only the third is allowed: with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(int_matrix_colmajor) - assert ('get_elem_rm_nocopy(): incompatible function arguments.' in str(excinfo.value) and - ', flags.c_contiguous' in str(excinfo.value)) + assert "get_elem_rm_nocopy(): incompatible function arguments." in str( + excinfo.value + ) and ", flags.c_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_colmajor) - assert ('get_elem_rm_nocopy(): incompatible function arguments.' in str(excinfo.value) and - ', flags.c_contiguous' in str(excinfo.value)) + assert "get_elem_rm_nocopy(): incompatible function arguments." in str( + excinfo.value + ) and ", flags.c_contiguous" in str(excinfo.value) assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_rowmajor) - assert ('get_elem_rm_nocopy(): incompatible function arguments.' in str(excinfo.value) and - ', flags.c_contiguous' in str(excinfo.value)) + assert "get_elem_rm_nocopy(): incompatible function arguments." in str( + excinfo.value + ) and ", flags.c_contiguous" in str(excinfo.value) def test_eigen_ref_life_support(): @@ -683,12 +737,9 @@ def test_eigen_ref_life_support(): def test_special_matrix_objects(): - assert np.all(m.incr_diag(7) == np.diag([1., 2, 3, 4, 5, 6, 7])) + assert np.all(m.incr_diag(7) == np.diag([1.0, 2, 3, 4, 5, 6, 7])) - asymm = np.array([[ 1., 2, 3, 4], - [ 5, 6, 7, 8], - [ 9, 10, 11, 12], - [13, 14, 15, 16]]) + asymm = np.array([[1.0, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]) symm_lower = np.array(asymm) symm_upper = np.array(asymm) for i in range(4): @@ -701,45 +752,55 @@ def test_special_matrix_objects(): def test_dense_signature(doc): - assert doc(m.double_col) == """ + assert ( + doc(m.double_col) + == """ double_col(arg0: numpy.ndarray[numpy.float32[m, 1]]) -> numpy.ndarray[numpy.float32[m, 1]] """ - assert doc(m.double_row) == """ + ) + assert ( + doc(m.double_row) + == """ double_row(arg0: numpy.ndarray[numpy.float32[1, n]]) -> numpy.ndarray[numpy.float32[1, n]] """ - assert doc(m.double_complex) == (""" + ) + assert doc(m.double_complex) == ( + """ double_complex(arg0: numpy.ndarray[numpy.complex64[m, 1]])""" - """ -> numpy.ndarray[numpy.complex64[m, 1]] - """) - assert doc(m.double_mat_rm) == (""" + """ -> numpy.ndarray[numpy.complex64[m, 1]] + """ + ) + assert doc(m.double_mat_rm) == ( + """ double_mat_rm(arg0: numpy.ndarray[numpy.float32[m, n]])""" - """ -> numpy.ndarray[numpy.float32[m, n]] - """) + """ -> numpy.ndarray[numpy.float32[m, n]] + """ + ) def test_named_arguments(): a = np.array([[1.0, 2], [3, 4], [5, 6]]) b = np.ones((2, 1)) - assert np.all(m.matrix_multiply(a, b) == np.array([[3.], [7], [11]])) - assert np.all(m.matrix_multiply(A=a, B=b) == np.array([[3.], [7], [11]])) - assert np.all(m.matrix_multiply(B=b, A=a) == np.array([[3.], [7], [11]])) + assert np.all(m.matrix_multiply(a, b) == np.array([[3.0], [7], [11]])) + assert np.all(m.matrix_multiply(A=a, B=b) == np.array([[3.0], [7], [11]])) + assert np.all(m.matrix_multiply(B=b, A=a) == np.array([[3.0], [7], [11]])) with pytest.raises(ValueError) as excinfo: m.matrix_multiply(b, a) - assert str(excinfo.value) == 'Nonconformable matrices!' + assert str(excinfo.value) == "Nonconformable matrices!" with pytest.raises(ValueError) as excinfo: m.matrix_multiply(A=b, B=a) - assert str(excinfo.value) == 'Nonconformable matrices!' + assert str(excinfo.value) == "Nonconformable matrices!" with pytest.raises(ValueError) as excinfo: m.matrix_multiply(B=a, A=b) - assert str(excinfo.value) == 'Nonconformable matrices!' + assert str(excinfo.value) == "Nonconformable matrices!" -@pytest.requires_eigen_and_scipy def test_sparse(): + pytest.importorskip("scipy") assert_sparse_equal_ref(m.sparse_r()) assert_sparse_equal_ref(m.sparse_c()) assert_sparse_equal_ref(m.sparse_copy_r(m.sparse_r())) @@ -748,24 +809,34 @@ def test_sparse(): assert_sparse_equal_ref(m.sparse_copy_c(m.sparse_r())) -@pytest.requires_eigen_and_scipy def test_sparse_signature(doc): - assert doc(m.sparse_copy_r) == """ + pytest.importorskip("scipy") + assert ( + doc(m.sparse_copy_r) + == """ sparse_copy_r(arg0: scipy.sparse.csr_matrix[numpy.float32]) -> scipy.sparse.csr_matrix[numpy.float32] """ # noqa: E501 line too long - assert doc(m.sparse_copy_c) == """ + ) + assert ( + doc(m.sparse_copy_c) + == """ sparse_copy_c(arg0: scipy.sparse.csc_matrix[numpy.float32]) -> scipy.sparse.csc_matrix[numpy.float32] """ # noqa: E501 line too long + ) def test_issue738_issue2038(): """Ignore strides on a length-0 or -1 dimension (even if they would be incompatible length > 1)""" - assert np.all(m.iss738_f1(np.array([[1., 2, 3]])) == np.array([[1., 102, 203]])) - assert np.all(m.iss738_f1(np.array([[1.], [2], [3]])) == np.array([[1.], [12], [23]])) - - assert np.all(m.iss738_f2(np.array([[1., 2, 3]])) == np.array([[1., 102, 203]])) - assert np.all(m.iss738_f2(np.array([[1.], [2], [3]])) == np.array([[1.], [12], [23]])) + assert np.all(m.iss738_f1(np.array([[1.0, 2, 3]])) == np.array([[1.0, 102, 203]])) + assert np.all( + m.iss738_f1(np.array([[1.0], [2], [3]])) == np.array([[1.0], [12], [23]]) + ) + + assert np.all(m.iss738_f2(np.array([[1.0, 2, 3]])) == np.array([[1.0, 102, 203]])) + assert np.all( + m.iss738_f2(np.array([[1.0], [2], [3]])) == np.array([[1.0], [12], [23]]) + ) assert np.all(m.iss738_f1(np.zeros((0, 2))) == np.zeros((0, 2))) assert np.all(m.iss738_f2(np.zeros((2, 0))) == np.zeros((2, 0))) diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt index 25972701fc..c960c877a5 100644 --- a/tests/test_embed/CMakeLists.txt +++ b/tests/test_embed/CMakeLists.txt @@ -1,10 +1,14 @@ -if(${PYTHON_MODULE_EXTENSION} MATCHES "pypy") +possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) + +if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy") + message(STATUS "Skipping embed test on PyPy") add_custom_target(cpptest) # Dummy target on PyPy. Embedding is not supported. set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}") return() endif() -find_package(Catch 2.13.0) +find_package(Catch 2.13.2) + if(CATCH_FOUND) message(STATUS "Building interpreter tests using Catch v${CATCH_VERSION}") else() @@ -13,27 +17,29 @@ else() return() endif() +find_package(Threads REQUIRED) + add_executable(test_embed catch.cpp test_interpreter.cpp) -target_include_directories(test_embed PRIVATE "${CATCH_INCLUDE_DIR}") pybind11_enable_warnings(test_embed) -target_link_libraries(test_embed PRIVATE pybind11::embed) +target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads) -find_package(Threads REQUIRED) -target_link_libraries(test_embed PUBLIC Threads::Threads) +if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + file(COPY test_interpreter.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") +endif() add_custom_target( cpptest COMMAND "$" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") pybind11_add_module(external_module THIN_LTO external_module.cpp) set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}") + "${CMAKE_CURRENT_BINARY_DIR}") foreach(config ${CMAKE_CONFIGURATION_TYPES}) string(TOUPPER ${config} config) set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} - "${CMAKE_CURRENT_SOURCE_DIR}") + "${CMAKE_CURRENT_BINARY_DIR}") endforeach() add_dependencies(cpptest external_module) diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 222bd565fb..944334ce66 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -30,7 +30,7 @@ class Widget { class PyWidget final : public Widget { using Widget::Widget; - int the_answer() const override { PYBIND11_OVERLOAD_PURE(int, Widget, the_answer); } + int the_answer() const override { PYBIND11_OVERRIDE_PURE(int, Widget, the_answer); } }; PYBIND11_EMBEDDED_MODULE(widget_module, m) { @@ -51,17 +51,17 @@ PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) { } TEST_CASE("Pass classes and data between modules defined in C++ and Python") { - auto module = py::module::import("test_interpreter"); - REQUIRE(py::hasattr(module, "DerivedWidget")); + auto module_ = py::module_::import("test_interpreter"); + REQUIRE(py::hasattr(module_, "DerivedWidget")); - auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module.attr("__dict__")); + auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module_.attr("__dict__")); py::exec(R"( widget = DerivedWidget("{} - {}".format(hello, x)) message = widget.the_message )", py::globals(), locals); REQUIRE(locals["message"].cast() == "Hello, World! - 5"); - auto py_widget = module.attr("DerivedWidget")("The question"); + auto py_widget = module_.attr("DerivedWidget")("The question"); auto message = py_widget.attr("the_message"); REQUIRE(message.cast() == "The question"); @@ -70,10 +70,10 @@ TEST_CASE("Pass classes and data between modules defined in C++ and Python") { } TEST_CASE("Import error handling") { - REQUIRE_NOTHROW(py::module::import("widget_module")); - REQUIRE_THROWS_WITH(py::module::import("throw_exception"), + REQUIRE_NOTHROW(py::module_::import("widget_module")); + REQUIRE_THROWS_WITH(py::module_::import("throw_exception"), "ImportError: C++ Error"); - REQUIRE_THROWS_WITH(py::module::import("throw_error_already_set"), + REQUIRE_THROWS_WITH(py::module_::import("throw_error_already_set"), Catch::Contains("ImportError: KeyError")); } @@ -107,14 +107,14 @@ bool has_pybind11_internals_static() { TEST_CASE("Restart the interpreter") { // Verify pre-restart state. - REQUIRE(py::module::import("widget_module").attr("add")(1, 2).cast() == 3); + REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast() == 3); REQUIRE(has_pybind11_internals_builtin()); REQUIRE(has_pybind11_internals_static()); - REQUIRE(py::module::import("external_module").attr("A")(123).attr("value").cast() == 123); + REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast() == 123); // local and foreign module internals should point to the same internals: REQUIRE(reinterpret_cast(*py::detail::get_internals_pp()) == - py::module::import("external_module").attr("internals_at")().cast()); + py::module_::import("external_module").attr("internals_at")().cast()); // Restart the interpreter. py::finalize_interpreter(); @@ -130,14 +130,14 @@ TEST_CASE("Restart the interpreter") { REQUIRE(has_pybind11_internals_builtin()); REQUIRE(has_pybind11_internals_static()); REQUIRE(reinterpret_cast(*py::detail::get_internals_pp()) == - py::module::import("external_module").attr("internals_at")().cast()); + py::module_::import("external_module").attr("internals_at")().cast()); // Make sure that an interpreter with no get_internals() created until finalize still gets the // internals destroyed py::finalize_interpreter(); py::initialize_interpreter(); bool ran = false; - py::module::import("__main__").attr("internals_destroy_test") = + py::module_::import("__main__").attr("internals_destroy_test") = py::capsule(&ran, [](void *ran) { py::detail::get_internals(); *static_cast(ran) = true; }); REQUIRE_FALSE(has_pybind11_internals_builtin()); REQUIRE_FALSE(has_pybind11_internals_static()); @@ -149,20 +149,20 @@ TEST_CASE("Restart the interpreter") { REQUIRE_FALSE(has_pybind11_internals_static()); // C++ modules can be reloaded. - auto cpp_module = py::module::import("widget_module"); + auto cpp_module = py::module_::import("widget_module"); REQUIRE(cpp_module.attr("add")(1, 2).cast() == 3); // C++ type information is reloaded and can be used in python modules. - auto py_module = py::module::import("test_interpreter"); + auto py_module = py::module_::import("test_interpreter"); auto py_widget = py_module.attr("DerivedWidget")("Hello after restart"); REQUIRE(py_widget.attr("the_message").cast() == "Hello after restart"); } TEST_CASE("Subinterpreter") { // Add tags to the modules in the main interpreter and test the basics. - py::module::import("__main__").attr("main_tag") = "main interpreter"; + py::module_::import("__main__").attr("main_tag") = "main interpreter"; { - auto m = py::module::import("widget_module"); + auto m = py::module_::import("widget_module"); m.attr("extension_module_tag") = "added to module in main interpreter"; REQUIRE(m.attr("add")(1, 2).cast() == 3); @@ -181,9 +181,9 @@ TEST_CASE("Subinterpreter") { REQUIRE(has_pybind11_internals_static()); // Modules tags should be gone. - REQUIRE_FALSE(py::hasattr(py::module::import("__main__"), "tag")); + REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag")); { - auto m = py::module::import("widget_module"); + auto m = py::module_::import("widget_module"); REQUIRE_FALSE(py::hasattr(m, "extension_module_tag")); // Function bindings should still work. @@ -194,8 +194,8 @@ TEST_CASE("Subinterpreter") { Py_EndInterpreter(sub_tstate); PyThreadState_Swap(main_tstate); - REQUIRE(py::hasattr(py::module::import("__main__"), "main_tag")); - REQUIRE(py::hasattr(py::module::import("widget_module"), "extension_module_tag")); + REQUIRE(py::hasattr(py::module_::import("__main__"), "main_tag")); + REQUIRE(py::hasattr(py::module_::import("widget_module"), "extension_module_tag")); } TEST_CASE("Execution frame") { @@ -245,7 +245,7 @@ TEST_CASE("Reload module from file") { // Disable generation of cached bytecode (.pyc files) for this test, otherwise // Python might pick up an old version from the cache instead of the new versions // of the .py files generated below - auto sys = py::module::import("sys"); + auto sys = py::module_::import("sys"); bool dont_write_bytecode = sys.attr("dont_write_bytecode").cast(); sys.attr("dont_write_bytecode") = true; // Reset the value at scope exit @@ -267,8 +267,8 @@ TEST_CASE("Reload module from file") { }); // Import the module from file - auto module = py::module::import(module_name.c_str()); - int result = module.attr("test")().cast(); + auto module_ = py::module_::import(module_name.c_str()); + int result = module_.attr("test")().cast(); REQUIRE(result == 1); // Update the module .py file with a small change @@ -278,7 +278,7 @@ TEST_CASE("Reload module from file") { test_module.close(); // Reload the module - module.reload(); - result = module.attr("test")().cast(); + module_.reload(); + result = module_.attr("test")().cast(); REQUIRE(result == 2); } diff --git a/tests/test_enum.py b/tests/test_enum.py index bfaa193e9b..e9732fa74f 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -7,32 +7,50 @@ def test_unscoped_enum(): assert str(m.UnscopedEnum.EOne) == "UnscopedEnum.EOne" assert str(m.UnscopedEnum.ETwo) == "UnscopedEnum.ETwo" assert str(m.EOne) == "UnscopedEnum.EOne" + assert repr(m.UnscopedEnum.EOne) == "" + assert repr(m.UnscopedEnum.ETwo) == "" + assert repr(m.EOne) == "" # name property assert m.UnscopedEnum.EOne.name == "EOne" + assert m.UnscopedEnum.EOne.value == 1 assert m.UnscopedEnum.ETwo.name == "ETwo" - assert m.EOne.name == "EOne" - # name readonly + assert m.UnscopedEnum.ETwo.value == 2 + assert m.EOne is m.UnscopedEnum.EOne + # name, value readonly with pytest.raises(AttributeError): m.UnscopedEnum.EOne.name = "" - # name returns a copy - foo = m.UnscopedEnum.EOne.name - foo = "bar" + with pytest.raises(AttributeError): + m.UnscopedEnum.EOne.value = 10 + # name, value returns a copy + # TODO: Neither the name nor value tests actually check against aliasing. + # Use a mutable type that has reference semantics. + nonaliased_name = m.UnscopedEnum.EOne.name + nonaliased_name = "bar" # noqa: F841 assert m.UnscopedEnum.EOne.name == "EOne" + nonaliased_value = m.UnscopedEnum.EOne.value + nonaliased_value = 10 # noqa: F841 + assert m.UnscopedEnum.EOne.value == 1 # __members__ property - assert m.UnscopedEnum.__members__ == \ - {"EOne": m.UnscopedEnum.EOne, "ETwo": m.UnscopedEnum.ETwo, "EThree": m.UnscopedEnum.EThree} + assert m.UnscopedEnum.__members__ == { + "EOne": m.UnscopedEnum.EOne, + "ETwo": m.UnscopedEnum.ETwo, + "EThree": m.UnscopedEnum.EThree, + } # __members__ readonly with pytest.raises(AttributeError): m.UnscopedEnum.__members__ = {} # __members__ returns a copy - foo = m.UnscopedEnum.__members__ - foo["bar"] = "baz" - assert m.UnscopedEnum.__members__ == \ - {"EOne": m.UnscopedEnum.EOne, "ETwo": m.UnscopedEnum.ETwo, "EThree": m.UnscopedEnum.EThree} + nonaliased_members = m.UnscopedEnum.__members__ + nonaliased_members["bar"] = "baz" + assert m.UnscopedEnum.__members__ == { + "EOne": m.UnscopedEnum.EOne, + "ETwo": m.UnscopedEnum.ETwo, + "EThree": m.UnscopedEnum.EThree, + } - for docstring_line in '''An unscoped enumeration + for docstring_line in """An unscoped enumeration Members: @@ -40,7 +58,9 @@ def test_unscoped_enum(): ETwo : Docstring for ETwo - EThree : Docstring for EThree'''.split('\n'): + EThree : Docstring for EThree""".split( + "\n" + ): assert docstring_line in m.UnscopedEnum.__doc__ # Unscoped enums will accept ==/!= int comparisons @@ -50,10 +70,10 @@ def test_unscoped_enum(): assert y != 3 assert 3 != y # Compare with None - assert (y != None) # noqa: E711 + assert y != None # noqa: E711 assert not (y == None) # noqa: E711 # Compare with an object - assert (y != object()) + assert y != object() assert not (y == object()) # Compare with string assert y != "2" @@ -62,25 +82,25 @@ def test_unscoped_enum(): assert not (y == "2") with pytest.raises(TypeError): - y < object() + y < object() # noqa: B015 with pytest.raises(TypeError): - y <= object() + y <= object() # noqa: B015 with pytest.raises(TypeError): - y > object() + y > object() # noqa: B015 with pytest.raises(TypeError): - y >= object() + y >= object() # noqa: B015 with pytest.raises(TypeError): - y | object() + y | object() # noqa: B015 with pytest.raises(TypeError): - y & object() + y & object() # noqa: B015 with pytest.raises(TypeError): - y ^ object() + y ^ object() # noqa: B015 assert int(m.UnscopedEnum.ETwo) == 2 assert str(m.UnscopedEnum(2)) == "UnscopedEnum.ETwo" @@ -116,20 +136,20 @@ def test_scoped_enum(): assert z != 3 assert 3 != z # Compare with None - assert (z != None) # noqa: E711 + assert z != None # noqa: E711 assert not (z == None) # noqa: E711 # Compare with an object - assert (z != object()) + assert z != object() assert not (z == object()) # Scoped enums will *NOT* accept >, <, >= and <= int comparisons (Will throw exceptions) with pytest.raises(TypeError): - z > 3 + z > 3 # noqa: B015 with pytest.raises(TypeError): - z < 3 + z < 3 # noqa: B015 with pytest.raises(TypeError): - z >= 3 + z >= 3 # noqa: B015 with pytest.raises(TypeError): - z <= 3 + z <= 3 # noqa: B015 # order assert m.ScopedEnum.Two < m.ScopedEnum.Three @@ -143,6 +163,8 @@ def test_scoped_enum(): def test_implicit_conversion(): assert str(m.ClassWithUnscopedEnum.EMode.EFirstMode) == "EMode.EFirstMode" assert str(m.ClassWithUnscopedEnum.EFirstMode) == "EMode.EFirstMode" + assert repr(m.ClassWithUnscopedEnum.EMode.EFirstMode) == "" + assert repr(m.ClassWithUnscopedEnum.EFirstMode) == "" f = m.ClassWithUnscopedEnum.test_function first = m.ClassWithUnscopedEnum.EFirstMode @@ -167,7 +189,7 @@ def test_implicit_conversion(): x[f(first)] = 3 x[f(second)] = 4 # Hashing test - assert str(x) == "{EMode.EFirstMode: 3, EMode.ESecondMode: 4}" + assert repr(x) == "{: 3, : 4}" def test_binary_operators(): @@ -205,3 +227,10 @@ def test_duplicate_enum_name(): with pytest.raises(ValueError) as excinfo: m.register_bad_enum() assert str(excinfo.value) == 'SimpleEnum: element "ONE" already exists!' + + +def test_docstring_signatures(): + for enum_type in [m.ScopedEnum, m.UnscopedEnum]: + for attr in enum_type.__dict__.values(): + # Issue #2623/PR #2637: Add argument names to enum_ methods + assert "arg0" not in (attr.__doc__ or "") diff --git a/tests/test_eval.cpp b/tests/test_eval.cpp index e094821911..5416c2ec4d 100644 --- a/tests/test_eval.cpp +++ b/tests/test_eval.cpp @@ -14,7 +14,7 @@ TEST_SUBMODULE(eval_, m) { // test_evals - auto global = py::dict(py::module::import("__main__").attr("__dict__")); + auto global = py::dict(py::module_::import("__main__").attr("__dict__")); m.def("test_eval_statements", [global]() { auto local = py::dict(); @@ -88,4 +88,12 @@ TEST_SUBMODULE(eval_, m) { } return false; }); + + // test_eval_empty_globals + m.def("eval_empty_globals", [](py::object global) { + if (global.is_none()) + global = py::dict(); + auto int_class = py::eval("isinstance(42, int)", global); + return global; + }); } diff --git a/tests/test_eval.py b/tests/test_eval.py index 66bec55f8b..1bb05af05e 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- import os + import pytest + +import env # noqa: F401 + from pybind11_tests import eval_ as m @@ -15,9 +19,17 @@ def test_evals(capture): assert m.test_eval_failure() -@pytest.unsupported_on_pypy3 +@pytest.mark.xfail("env.PYPY and not env.PY2", raises=RuntimeError) def test_eval_file(): filename = os.path.join(os.path.dirname(__file__), "test_eval_call.py") assert m.test_eval_file(filename) assert m.test_eval_file_failure() + + +def test_eval_empty_globals(): + assert "__builtins__" in m.eval_empty_globals(None) + + g = {} + assert "__builtins__" in m.eval_empty_globals(g) + assert "__builtins__" in g diff --git a/tests/test_eval_call.py b/tests/test_eval_call.py index d42a0a6d30..373b67bac8 100644 --- a/tests/test_eval_call.py +++ b/tests/test_eval_call.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- # This file is called from 'test_eval.py' -if 'call_test2' in locals(): +if "call_test2" in locals(): call_test2(y) # noqa: F821 undefined name diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 56cd9bc48f..e27c16dfef 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -13,7 +13,7 @@ class MyException : public std::exception { public: explicit MyException(const char * m) : message{m} {} - virtual const char * what() const noexcept override {return message.c_str();} + const char * what() const noexcept override {return message.c_str();} private: std::string message = ""; }; @@ -22,7 +22,7 @@ class MyException : public std::exception { class MyException2 : public std::exception { public: explicit MyException2(const char * m) : message{m} {} - virtual const char * what() const noexcept override {return message.c_str();} + const char * what() const noexcept override {return message.c_str();} private: std::string message = ""; }; @@ -32,6 +32,13 @@ class MyException3 { public: explicit MyException3(const char * m) : message{m} {} virtual const char * what() const noexcept {return message.c_str();} + // Rule of 5 BEGIN: to preempt compiler warnings. + MyException3(const MyException3&) = default; + MyException3(MyException3&&) = default; + MyException3& operator=(const MyException3&) = default; + MyException3& operator=(MyException3&&) = default; + virtual ~MyException3() = default; + // Rule of 5 END. private: std::string message = ""; }; @@ -41,7 +48,7 @@ class MyException3 { class MyException4 : public std::exception { public: explicit MyException4(const char * m) : message{m} {} - virtual const char * what() const noexcept override {return message.c_str();} + const char * what() const noexcept override {return message.c_str();} private: std::string message = ""; }; @@ -65,6 +72,25 @@ struct PythonCallInDestructor { py::dict d; }; + + +struct PythonAlreadySetInDestructor { + PythonAlreadySetInDestructor(const py::str &s) : s(s) {} + ~PythonAlreadySetInDestructor() { + py::dict foo; + try { + // Assign to a py::object to force read access of nonexistent dict entry + py::object o = foo["bar"]; + } + catch (py::error_already_set& ex) { + ex.discard_as_unraisable(s); + } + } + + py::str s; +}; + + TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); @@ -144,7 +170,7 @@ TEST_SUBMODULE(exceptions, m) { m.def("modulenotfound_exception_matches_base", []() { try { // On Python >= 3.6, this raises a ModuleNotFoundError, a subclass of ImportError - py::module::import("nonexistent"); + py::module_::import("nonexistent"); } catch (py::error_already_set &ex) { if (!ex.matches(PyExc_ImportError)) throw; @@ -183,6 +209,11 @@ TEST_SUBMODULE(exceptions, m) { return false; }); + m.def("python_alreadyset_in_destructor", [](py::str s) { + PythonAlreadySetInDestructor alreadyset_in_destructor(s); + return true; + }); + // test_nested_throws m.def("try_catch", [m](py::object exc_type, py::function f, py::args args) { try { f(*args); } @@ -194,4 +225,7 @@ TEST_SUBMODULE(exceptions, m) { } }); + // Test repr that cannot be displayed + m.def("simple_bool_passthrough", [](bool x) {return x;}); + } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 053e7d4a28..c6cb65299e 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys + import pytest from pybind11_tests import exceptions as m @@ -48,6 +50,44 @@ def test_python_call_in_catch(): assert d["good"] is True +def ignore_pytest_unraisable_warning(f): + unraisable = "PytestUnraisableExceptionWarning" + if hasattr(pytest, unraisable): # Python >= 3.8 and pytest >= 6 + dec = pytest.mark.filterwarnings("ignore::pytest.{}".format(unraisable)) + return dec(f) + else: + return f + + +@ignore_pytest_unraisable_warning +def test_python_alreadyset_in_destructor(monkeypatch, capsys): + hooked = False + triggered = [False] # mutable, so Python 2.7 closure can modify it + + if hasattr(sys, "unraisablehook"): # Python 3.8+ + hooked = True + # Don't take `sys.unraisablehook`, as that's overwritten by pytest + default_hook = sys.__unraisablehook__ + + def hook(unraisable_hook_args): + exc_type, exc_value, exc_tb, err_msg, obj = unraisable_hook_args + if obj == "already_set demo": + triggered[0] = True + default_hook(unraisable_hook_args) + return + + # Use monkeypatch so pytest can apply and remove the patch as appropriate + monkeypatch.setattr(sys, "unraisablehook", hook) + + assert m.python_alreadyset_in_destructor("already_set demo") is True + if hooked: + assert triggered[0] is True + + _, captured_stderr = capsys.readouterr() + # Error message is different in Python 2 and 3, check for words that appear in both + assert "ignored" in captured_stderr and "already_set demo" in captured_stderr + + def test_exception_matches(): assert m.exception_matches() assert m.exception_matches_base() @@ -78,7 +118,9 @@ def test_custom(msg): # Can we fall-through to the default handler? with pytest.raises(RuntimeError) as excinfo: m.throws_logic_error() - assert msg(excinfo.value) == "this error should fall through to the standard handler" + assert ( + msg(excinfo.value) == "this error should fall through to the standard handler" + ) # OverFlow error translation. with pytest.raises(OverflowError) as excinfo: @@ -137,7 +179,13 @@ def pycatch(exctype, f, *args): # C++ -> Python -> C++ -> Python with capture: m.try_catch( - m.MyException5, pycatch, m.MyException, m.try_catch, m.MyException, throw_myex5) + m.MyException5, + pycatch, + m.MyException, + m.try_catch, + m.MyException, + throw_myex5, + ) assert str(capture).startswith("MyException5: nested error 5") # C++ -> Python -> C++ @@ -149,3 +197,13 @@ def pycatch(exctype, f, *args): with pytest.raises(m.MyException5) as excinfo: m.try_catch(m.MyException, pycatch, m.MyException, m.throws5) assert str(excinfo.value) == "this is a helper-defined translated exception" + + +# This can often happen if you wrap a pybind11 class in a Python wrapper +def test_invalid_repr(): + class MyRepr(object): + def __repr__(self): + raise AttributeError("Example error") + + with pytest.raises(TypeError): + m.simple_bool_passthrough(MyRepr()) diff --git a/tests/test_factory_constructors.cpp b/tests/test_factory_constructors.cpp index 5cfbfdc3f8..7ff7e7b52c 100644 --- a/tests/test_factory_constructors.cpp +++ b/tests/test_factory_constructors.cpp @@ -11,6 +11,7 @@ #include "pybind11_tests.h" #include "constructor_stats.h" #include +#include // Classes for testing python construction via C++ factory function: // Not publicly constructible, copyable, or movable: @@ -57,13 +58,13 @@ class TestFactory4 : public TestFactory3 { public: TestFactory4() : TestFactory3() { print_default_created(this); } TestFactory4(int v) : TestFactory3(v) { print_created(this, v); } - virtual ~TestFactory4() { print_destroyed(this); } + ~TestFactory4() override { print_destroyed(this); } }; // Another class for an invalid downcast test class TestFactory5 : public TestFactory3 { public: TestFactory5(int i) : TestFactory3(i) { print_created(this, i); } - virtual ~TestFactory5() { print_destroyed(this); } + ~TestFactory5() override { print_destroyed(this); } }; class TestFactory6 { @@ -87,8 +88,8 @@ class PyTF6 : public TestFactory6 { PyTF6(PyTF6 &&f) : TestFactory6(std::move(f)) { print_move_created(this); } PyTF6(const PyTF6 &f) : TestFactory6(f) { print_copy_created(this); } PyTF6(std::string s) : TestFactory6((int) s.size()) { alias = true; print_created(this, s); } - virtual ~PyTF6() { print_destroyed(this); } - int get() override { PYBIND11_OVERLOAD(int, TestFactory6, get, /*no args*/); } + ~PyTF6() override { print_destroyed(this); } + int get() override { PYBIND11_OVERRIDE(int, TestFactory6, get, /*no args*/); } }; class TestFactory7 { @@ -108,8 +109,8 @@ class PyTF7 : public TestFactory7 { PyTF7(int i) : TestFactory7(i) { alias = true; print_created(this, i); } PyTF7(PyTF7 &&f) : TestFactory7(std::move(f)) { print_move_created(this); } PyTF7(const PyTF7 &f) : TestFactory7(f) { print_copy_created(this); } - virtual ~PyTF7() { print_destroyed(this); } - int get() override { PYBIND11_OVERLOAD(int, TestFactory7, get, /*no args*/); } + ~PyTF7() override { print_destroyed(this); } + int get() override { PYBIND11_OVERRIDE(int, TestFactory7, get, /*no args*/); } }; @@ -141,7 +142,7 @@ class TestFactoryHelper { TEST_SUBMODULE(factory_constructors, m) { // Define various trivial types to allow simpler overload resolution: - py::module m_tag = m.def_submodule("tag"); + py::module_ m_tag = m.def_submodule("tag"); #define MAKE_TAG_TYPE(Name) \ struct Name##_tag {}; \ py::class_(m_tag, #Name "_tag").def(py::init<>()); \ @@ -154,6 +155,8 @@ TEST_SUBMODULE(factory_constructors, m) { MAKE_TAG_TYPE(TF4); MAKE_TAG_TYPE(TF5); MAKE_TAG_TYPE(null_ptr); + MAKE_TAG_TYPE(null_unique_ptr); + MAKE_TAG_TYPE(null_shared_ptr); MAKE_TAG_TYPE(base); MAKE_TAG_TYPE(invalid_base); MAKE_TAG_TYPE(alias); @@ -180,11 +183,14 @@ TEST_SUBMODULE(factory_constructors, m) { auto c4a = [c](pointer_tag, TF4_tag, int a) { (void) c; return new TestFactory4(a);}; // test_init_factory_basic, test_init_factory_casting - py::class_>(m, "TestFactory3") + py::class_> pyTestFactory3(m, "TestFactory3"); + pyTestFactory3 .def(py::init([](pointer_tag, int v) { return TestFactoryHelper::construct3(v); })) - .def(py::init([](shared_ptr_tag) { return TestFactoryHelper::construct3(); })) - .def("__init__", [](TestFactory3 &self, std::string v) { new (&self) TestFactory3(v); }) // placement-new ctor - + .def(py::init([](shared_ptr_tag) { return TestFactoryHelper::construct3(); })); + ignoreOldStyleInitWarnings([&pyTestFactory3]() { + pyTestFactory3.def("__init__", [](TestFactory3 &self, std::string v) { new (&self) TestFactory3(v); }); // placement-new ctor + }); + pyTestFactory3 // factories returning a derived type: .def(py::init(c4a)) // derived ptr .def(py::init([](pointer_tag, TF5_tag, int a) { return new TestFactory5(a); })) @@ -194,6 +200,8 @@ TEST_SUBMODULE(factory_constructors, m) { // Returns nullptr: .def(py::init([](null_ptr_tag) { return (TestFactory3 *) nullptr; })) + .def(py::init([](null_unique_ptr_tag) { return std::unique_ptr(); })) + .def(py::init([](null_shared_ptr_tag) { return std::shared_ptr(); })) .def_readwrite("value", &TestFactory3::value) ; @@ -299,24 +307,32 @@ TEST_SUBMODULE(factory_constructors, m) { static void operator delete(void *p) { py::print("noisy delete"); ::operator delete(p); } #endif }; - py::class_(m, "NoisyAlloc") + + + py::class_ pyNoisyAlloc(m, "NoisyAlloc"); // Since these overloads have the same number of arguments, the dispatcher will try each of // them until the arguments convert. Thus we can get a pre-allocation here when passing a // single non-integer: - .def("__init__", [](NoisyAlloc *a, int i) { new (a) NoisyAlloc(i); }) // Regular constructor, runs first, requires preallocation - .def(py::init([](double d) { return new NoisyAlloc(d); })) - - // The two-argument version: first the factory pointer overload. - .def(py::init([](int i, int) { return new NoisyAlloc(i); })) - // Return-by-value: - .def(py::init([](double d, int) { return NoisyAlloc(d); })) - // Old-style placement new init; requires preallocation - .def("__init__", [](NoisyAlloc &a, double d, double) { new (&a) NoisyAlloc(d); }) - // Requires deallocation of previous overload preallocated value: - .def(py::init([](int i, double) { return new NoisyAlloc(i); })) - // Regular again: requires yet another preallocation - .def("__init__", [](NoisyAlloc &a, int i, std::string) { new (&a) NoisyAlloc(i); }) - ; + ignoreOldStyleInitWarnings([&pyNoisyAlloc]() { + pyNoisyAlloc.def("__init__", [](NoisyAlloc *a, int i) { new (a) NoisyAlloc(i); }); // Regular constructor, runs first, requires preallocation + }); + + pyNoisyAlloc.def(py::init([](double d) { return new NoisyAlloc(d); })); + + // The two-argument version: first the factory pointer overload. + pyNoisyAlloc.def(py::init([](int i, int) { return new NoisyAlloc(i); })); + // Return-by-value: + pyNoisyAlloc.def(py::init([](double d, int) { return NoisyAlloc(d); })); + // Old-style placement new init; requires preallocation + ignoreOldStyleInitWarnings([&pyNoisyAlloc]() { + pyNoisyAlloc.def("__init__", [](NoisyAlloc &a, double d, double) { new (&a) NoisyAlloc(d); }); + }); + // Requires deallocation of previous overload preallocated value: + pyNoisyAlloc.def(py::init([](int i, double) { return new NoisyAlloc(i); })); + // Regular again: requires yet another preallocation + ignoreOldStyleInitWarnings([&pyNoisyAlloc]() { + pyNoisyAlloc.def("__init__", [](NoisyAlloc &a, int i, std::string) { new (&a) NoisyAlloc(i); }); + }); diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index 49e6f4f331..ffcce6fd4d 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -2,6 +2,8 @@ import pytest import re +import env # noqa: F401 + from pybind11_tests import factory_constructors as m from pybind11_tests.factory_constructors import tag from pybind11_tests import ConstructorStats @@ -10,7 +12,10 @@ def test_init_factory_basic(): """Tests py::init_factory() wrapper around various ways of returning the object""" - cstats = [ConstructorStats.get(c) for c in [m.TestFactory1, m.TestFactory2, m.TestFactory3]] + cstats = [ + ConstructorStats.get(c) + for c in [m.TestFactory1, m.TestFactory2, m.TestFactory3] + ] cstats[0].alive() # force gc n_inst = ConstructorStats.detail_reg_inst() @@ -39,9 +44,12 @@ def test_init_factory_basic(): z3 = m.TestFactory3("bye") assert z3.value == "bye" - with pytest.raises(TypeError) as excinfo: - m.TestFactory3(tag.null_ptr) - assert str(excinfo.value) == "pybind11::init(): factory function returned nullptr" + for null_ptr_kind in [tag.null_ptr, tag.null_unique_ptr, tag.null_shared_ptr]: + with pytest.raises(TypeError) as excinfo: + m.TestFactory3(null_ptr_kind) + assert ( + str(excinfo.value) == "pybind11::init(): factory function returned nullptr" + ) assert [i.alive() for i in cstats] == [3, 3, 3] assert ConstructorStats.detail_reg_inst() == n_inst + 9 @@ -56,7 +64,7 @@ def test_init_factory_basic(): assert [i.values() for i in cstats] == [ ["3", "hi!"], ["7", "hi again"], - ["42", "bye"] + ["42", "bye"], ] assert [i.default_constructions for i in cstats] == [1, 1, 1] @@ -64,7 +72,9 @@ def test_init_factory_basic(): def test_init_factory_signature(msg): with pytest.raises(TypeError) as excinfo: m.TestFactory1("invalid", "constructor", "arguments") - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ __init__(): incompatible constructor arguments. The following argument types are supported: 1. m.factory_constructors.TestFactory1(arg0: m.factory_constructors.tag.unique_ptr_tag, arg1: int) 2. m.factory_constructors.TestFactory1(arg0: str) @@ -73,8 +83,11 @@ def test_init_factory_signature(msg): Invoked with: 'invalid', 'constructor', 'arguments' """ # noqa: E501 line too long + ) - assert msg(m.TestFactory1.__init__.__doc__) == """ + assert ( + msg(m.TestFactory1.__init__.__doc__) + == """ __init__(*args, **kwargs) Overloaded function. @@ -86,12 +99,16 @@ def test_init_factory_signature(msg): 4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None """ # noqa: E501 line too long + ) def test_init_factory_casting(): """Tests py::init_factory() wrapper with various upcasting and downcasting returns""" - cstats = [ConstructorStats.get(c) for c in [m.TestFactory3, m.TestFactory4, m.TestFactory5]] + cstats = [ + ConstructorStats.get(c) + for c in [m.TestFactory3, m.TestFactory4, m.TestFactory5] + ] cstats[0].alive() # force gc n_inst = ConstructorStats.detail_reg_inst() @@ -129,7 +146,7 @@ def test_init_factory_casting(): assert [i.values() for i in cstats] == [ ["4", "5", "6", "7", "8"], ["4", "5", "8"], - ["6", "7"] + ["6", "7"], ] @@ -199,7 +216,7 @@ def get(self): assert [i.values() for i in cstats] == [ ["1", "8", "3", "4", "5", "6", "123", "10", "47"], - ["hi there", "3", "4", "6", "move", "123", "why hello!", "move", "47"] + ["hi there", "3", "4", "6", "move", "123", "why hello!", "move", "47"], ] @@ -263,9 +280,11 @@ def get(self): assert not g1.has_alias() with pytest.raises(TypeError) as excinfo: PythFactory7(tag.shared_ptr, tag.invalid_base, 14) - assert (str(excinfo.value) == - "pybind11::init(): construction failed: returned holder-wrapped instance is not an " - "alias instance") + assert ( + str(excinfo.value) + == "pybind11::init(): construction failed: returned holder-wrapped instance is not an " + "alias instance" + ) assert [i.alive() for i in cstats] == [13, 7] assert ConstructorStats.detail_reg_inst() == n_inst + 13 @@ -279,7 +298,7 @@ def get(self): assert [i.values() for i in cstats] == [ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "100", "11", "12", "13", "14"], - ["2", "4", "6", "8", "9", "100", "12"] + ["2", "4", "6", "8", "9", "100", "12"], ] @@ -289,7 +308,7 @@ def test_no_placement_new(capture): with capture: a = m.NoPlacementNew(123) - found = re.search(r'^operator new called, returning (\d+)\n$', str(capture)) + found = re.search(r"^operator new called, returning (\d+)\n$", str(capture)) assert found assert a.i == 123 with capture: @@ -300,7 +319,7 @@ def test_no_placement_new(capture): with capture: b = m.NoPlacementNew() - found = re.search(r'^operator new called, returning (\d+)\n$', str(capture)) + found = re.search(r"^operator new called, returning (\d+)\n$", str(capture)) assert found assert b.i == 100 with capture: @@ -328,19 +347,21 @@ def create_and_destroy(*args): def strip_comments(s): - return re.sub(r'\s+#.*', '', s) + return re.sub(r"\s+#.*", "", s) -def test_reallocations(capture, msg): +def test_reallocation_a(capture, msg): """When the constructor is overloaded, previous overloads can require a preallocated value. This test makes sure that such preallocated values only happen when they might be necessary, - and that they are deallocated properly""" + and that they are deallocated properly.""" pytest.gc_collect() with capture: create_and_destroy(1) - assert msg(capture) == """ + assert ( + msg(capture) + == """ noisy new noisy placement new NoisyAlloc(int 1) @@ -348,9 +369,14 @@ def test_reallocations(capture, msg): ~NoisyAlloc() noisy delete """ + ) + + +def test_reallocation_b(capture, msg): with capture: create_and_destroy(1.5) - assert msg(capture) == strip_comments(""" + assert msg(capture) == strip_comments( + """ noisy new # allocation required to attempt first overload noisy delete # have to dealloc before considering factory init overload noisy new # pointer factory calling "new", part 1: allocation @@ -358,43 +384,59 @@ def test_reallocations(capture, msg): --- ~NoisyAlloc() # Destructor noisy delete # operator delete - """) + """ + ) + +def test_reallocation_c(capture, msg): with capture: create_and_destroy(2, 3) - assert msg(capture) == strip_comments(""" + assert msg(capture) == strip_comments( + """ noisy new # pointer factory calling "new", allocation NoisyAlloc(int 2) # constructor --- ~NoisyAlloc() # Destructor noisy delete # operator delete - """) + """ + ) + +def test_reallocation_d(capture, msg): with capture: create_and_destroy(2.5, 3) - assert msg(capture) == strip_comments(""" + assert msg(capture) == strip_comments( + """ NoisyAlloc(double 2.5) # construction (local func variable: operator_new not called) noisy new # return-by-value "new" part 1: allocation ~NoisyAlloc() # moved-away local func variable destruction --- ~NoisyAlloc() # Destructor noisy delete # operator delete - """) + """ + ) + +def test_reallocation_e(capture, msg): with capture: create_and_destroy(3.5, 4.5) - assert msg(capture) == strip_comments(""" + assert msg(capture) == strip_comments( + """ noisy new # preallocation needed before invoking placement-new overload noisy placement new # Placement new NoisyAlloc(double 3.5) # construction --- ~NoisyAlloc() # Destructor noisy delete # operator delete - """) + """ + ) + +def test_reallocation_f(capture, msg): with capture: create_and_destroy(4, 0.5) - assert msg(capture) == strip_comments(""" + assert msg(capture) == strip_comments( + """ noisy new # preallocation needed before invoking placement-new overload noisy delete # deallocation of preallocated storage noisy new # Factory pointer allocation @@ -402,11 +444,15 @@ def test_reallocations(capture, msg): --- ~NoisyAlloc() # Destructor noisy delete # operator delete - """) + """ + ) + +def test_reallocation_g(capture, msg): with capture: create_and_destroy(5, "hi") - assert msg(capture) == strip_comments(""" + assert msg(capture) == strip_comments( + """ noisy new # preallocation needed before invoking first placement new noisy delete # delete before considering new-style constructor noisy new # preallocation for second placement new @@ -415,13 +461,15 @@ def test_reallocations(capture, msg): --- ~NoisyAlloc() # Destructor noisy delete # operator delete - """) + """ + ) -@pytest.unsupported_on_py2 +@pytest.mark.skipif("env.PY2") def test_invalid_self(): """Tests invocation of the pybind-registered base class with an invalid `self` argument. You can only actually do this on Python 3: Python 2 raises an exception itself if you try.""" + class NotPybindDerived(object): pass @@ -445,16 +493,26 @@ def __init__(self, bad): a = m.TestFactory2(tag.pointer, 1) m.TestFactory6.__init__(a, tag.alias, 1) elif bad == 3: - m.TestFactory6.__init__(NotPybindDerived.__new__(NotPybindDerived), tag.base, 1) + m.TestFactory6.__init__( + NotPybindDerived.__new__(NotPybindDerived), tag.base, 1 + ) elif bad == 4: - m.TestFactory6.__init__(NotPybindDerived.__new__(NotPybindDerived), tag.alias, 1) + m.TestFactory6.__init__( + NotPybindDerived.__new__(NotPybindDerived), tag.alias, 1 + ) for arg in (1, 2): with pytest.raises(TypeError) as excinfo: BrokenTF1(arg) - assert str(excinfo.value) == "__init__(self, ...) called with invalid `self` argument" + assert ( + str(excinfo.value) + == "__init__(self, ...) called with invalid `self` argument" + ) for arg in (1, 2, 3, 4): with pytest.raises(TypeError) as excinfo: BrokenTF6(arg) - assert str(excinfo.value) == "__init__(self, ...) called with invalid `self` argument" + assert ( + str(excinfo.value) + == "__init__(self, ...) called with invalid `self` argument" + ) diff --git a/tests/test_gil_scoped.cpp b/tests/test_gil_scoped.cpp index dc9b7ed224..b6a45a5f03 100644 --- a/tests/test_gil_scoped.cpp +++ b/tests/test_gil_scoped.cpp @@ -22,10 +22,10 @@ class VirtClass { class PyVirtClass : public VirtClass { void virtual_func() override { - PYBIND11_OVERLOAD(void, VirtClass, virtual_func,); + PYBIND11_OVERRIDE(void, VirtClass, virtual_func,); } void pure_virtual_func() override { - PYBIND11_OVERLOAD_PURE(void, VirtClass, pure_virtual_func,); + PYBIND11_OVERRIDE_PURE(void, VirtClass, pure_virtual_func,); } }; @@ -45,7 +45,7 @@ TEST_SUBMODULE(gil_scoped, m) { [](VirtClass &virt) { virt.pure_virtual_func(); }); m.def("test_cross_module_gil", []() { - auto cm = py::module::import("cross_module_gil_utils"); + auto cm = py::module_::import("cross_module_gil_utils"); auto gil_acquire = reinterpret_cast( PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); py::gil_scoped_release gil_release; diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 1307712ad3..0a1d62747d 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import multiprocessing import threading + from pybind11_tests import gil_scoped as m @@ -20,6 +21,7 @@ def _run_in_process(target, *args, **kwargs): def _python_to_cpp_to_python(): """Calls different C++ functions that come back to Python.""" + class ExtendedVirtClass(m.VirtClass): def virtual_func(self): pass @@ -49,6 +51,7 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): thread.join() +# TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 def test_python_to_cpp_to_python_from_thread(): """Makes sure there is no GIL deadlock when running in a thread. @@ -57,6 +60,7 @@ def test_python_to_cpp_to_python_from_thread(): assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 +# TODO: FIXME on macOS Python 3.9 def test_python_to_cpp_to_python_from_thread_multiple_parallel(): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. @@ -65,14 +69,18 @@ def test_python_to_cpp_to_python_from_thread_multiple_parallel(): assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 +# TODO: FIXME on macOS Python 3.9 def test_python_to_cpp_to_python_from_thread_multiple_sequential(): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 + assert ( + _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 + ) +# TODO: FIXME on macOS Python 3.9 def test_python_to_cpp_to_python_from_process(): """Makes sure there is no GIL deadlock when using processes. diff --git a/tests/test_iostream.cpp b/tests/test_iostream.cpp index e67f88af5f..1be0655dfb 100644 --- a/tests/test_iostream.cpp +++ b/tests/test_iostream.cpp @@ -7,10 +7,15 @@ BSD-style license that can be found in the LICENSE file. */ +#if defined(_MSC_VER) && _MSC_VER < 1910 // VS 2015's MSVC +# pragma warning(disable: 4702) // unreachable code in system header (xatomic.h(382)) +#endif #include #include "pybind11_tests.h" +#include #include +#include void noisy_function(std::string msg, bool flush) { @@ -25,6 +30,40 @@ void noisy_funct_dual(std::string msg, std::string emsg) { std::cerr << emsg; } +// object to manage C++ thread +// simply repeatedly write to std::cerr until stopped +// redirect is called at some point to test the safety of scoped_estream_redirect +struct TestThread { + TestThread() : t_{nullptr}, stop_{false} { + auto thread_f = [this] { + while (!stop_) { + std::cout << "x" << std::flush; + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } }; + t_ = new std::thread(std::move(thread_f)); + } + + ~TestThread() { + delete t_; + } + + void stop() { stop_ = true; } + + void join() { + py::gil_scoped_release gil_lock; + t_->join(); + } + + void sleep() { + py::gil_scoped_release gil_lock; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + std::thread * t_; + std::atomic stop_; +}; + + TEST_SUBMODULE(iostream, m) { add_ostream_redirect(m); @@ -37,7 +76,7 @@ TEST_SUBMODULE(iostream, m) { }); m.def("captured_output", [](std::string msg) { - py::scoped_ostream_redirect redir(std::cout, py::module::import("sys").attr("stdout")); + py::scoped_ostream_redirect redir(std::cout, py::module_::import("sys").attr("stdout")); std::cout << msg << std::flush; }); @@ -46,7 +85,7 @@ TEST_SUBMODULE(iostream, m) { py::arg("msg"), py::arg("flush")=true); m.def("captured_err", [](std::string msg) { - py::scoped_ostream_redirect redir(std::cerr, py::module::import("sys").attr("stderr")); + py::scoped_ostream_redirect redir(std::cerr, py::module_::import("sys").attr("stderr")); std::cerr << msg << std::flush; }); @@ -65,9 +104,15 @@ TEST_SUBMODULE(iostream, m) { }); m.def("captured_dual", [](std::string msg, std::string emsg) { - py::scoped_ostream_redirect redirout(std::cout, py::module::import("sys").attr("stdout")); - py::scoped_ostream_redirect redirerr(std::cerr, py::module::import("sys").attr("stderr")); + py::scoped_ostream_redirect redirout(std::cout, py::module_::import("sys").attr("stdout")); + py::scoped_ostream_redirect redirerr(std::cerr, py::module_::import("sys").attr("stderr")); std::cout << msg << std::flush; std::cerr << emsg << std::flush; }); + + py::class_(m, "TestThread") + .def(py::init<>()) + .def("stop", &TestThread::stop) + .def("join", &TestThread::join) + .def("sleep", &TestThread::sleep); } diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 7ac4fcece0..6d493beda3 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -18,6 +18,7 @@ # Python 3.4 from contextlib import redirect_stdout except ImportError: + @contextmanager def redirect_stdout(target): original = sys.stdout @@ -25,10 +26,12 @@ def redirect_stdout(target): yield sys.stdout = original + try: # Python 3.5 from contextlib import redirect_stderr except ImportError: + @contextmanager def redirect_stderr(target): original = sys.stderr @@ -42,16 +45,16 @@ def test_captured(capsys): m.captured_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == '' + assert stderr == "" m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == '' + assert stderr == "" m.captured_err(msg) stdout, stderr = capsys.readouterr() - assert stdout == '' + assert stdout == "" assert stderr == msg @@ -63,7 +66,7 @@ def test_captured_large_string(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == '' + assert stderr == "" def test_guard_capture(capsys): @@ -71,7 +74,7 @@ def test_guard_capture(capsys): m.guard_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == '' + assert stderr == "" def test_series_captured(capture): @@ -88,7 +91,7 @@ def test_flush(capfd): with m.ostream_redirect(): m.noisy_function(msg, flush=False) stdout, stderr = capfd.readouterr() - assert stdout == '' + assert stdout == "" m.noisy_function(msg2, flush=True) stdout, stderr = capfd.readouterr() @@ -107,15 +110,15 @@ def test_not_captured(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == '' - assert stream.getvalue() == '' + assert stderr == "" + assert stream.getvalue() == "" stream = StringIO() with redirect_stdout(stream): m.captured_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == '' - assert stderr == '' + assert stdout == "" + assert stderr == "" assert stream.getvalue() == msg @@ -125,16 +128,16 @@ def test_err(capfd): with redirect_stderr(stream): m.raw_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == '' + assert stdout == "" assert stderr == msg - assert stream.getvalue() == '' + assert stream.getvalue() == "" stream = StringIO() with redirect_stderr(stream): m.captured_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == '' - assert stderr == '' + assert stdout == "" + assert stderr == "" assert stream.getvalue() == msg @@ -146,8 +149,8 @@ def test_multi_captured(capfd): m.captured_output("c") m.raw_output("d") stdout, stderr = capfd.readouterr() - assert stdout == 'bd' - assert stream.getvalue() == 'ac' + assert stdout == "bd" + assert stream.getvalue() == "ac" def test_dual(capsys): @@ -164,14 +167,14 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == '' + assert stream.getvalue() == "" stream = StringIO() with redirect_stdout(stream): with m.ostream_redirect(): m.raw_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == '' + assert stdout == "" assert stream.getvalue() == msg stream = StringIO() @@ -179,7 +182,7 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == '' + assert stream.getvalue() == "" def test_redirect_err(capfd): @@ -193,7 +196,7 @@ def test_redirect_err(capfd): m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == '' + assert stderr == "" assert stream.getvalue() == msg2 @@ -209,7 +212,30 @@ def test_redirect_both(capfd): m.raw_output(msg) m.raw_err(msg2) stdout, stderr = capfd.readouterr() - assert stdout == '' - assert stderr == '' + assert stdout == "" + assert stderr == "" assert stream.getvalue() == msg assert stream2.getvalue() == msg2 + + +def test_threading(): + with m.ostream_redirect(stdout=True, stderr=False): + # start some threads + threads = [] + + # start some threads + for _j in range(20): + threads.append(m.TestThread()) + + # give the threads some time to fail + threads[0].sleep() + + # stop all the threads + for t in threads: + t.stop() + + for t in threads: + t.join() + + # if a thread segfaults, we don't get here + assert True diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 64bc2377b2..627a79690e 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -71,7 +71,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { py::tuple t(a.size()); for (size_t i = 0; i < a.size(); i++) // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); return t; }); m.def("mixed_args_refcount", [](py::object o, py::args a) { @@ -80,7 +80,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { t[0] = o.ref_count(); for (size_t i = 0; i < a.size(); i++) // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i + 1] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i + 1] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); return t; }); @@ -95,28 +95,39 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // m.def("bad_args7", [](py::kwargs, py::kwargs) {}); // test_keyword_only_args - m.def("kwonly_all", [](int i, int j) { return py::make_tuple(i, j); }, - py::kwonly(), py::arg("i"), py::arg("j")); - m.def("kwonly_some", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, - py::arg(), py::kwonly(), py::arg("j"), py::arg("k")); - m.def("kwonly_with_defaults", [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, - py::arg() = 3, "j"_a = 4, py::kwonly(), "k"_a = 5, "z"_a); - m.def("kwonly_mixed", [](int i, int j) { return py::make_tuple(i, j); }, - "i"_a, py::kwonly(), "j"_a); - m.def("kwonly_plus_more", [](int i, int j, int k, py::kwargs kwargs) { + m.def("kw_only_all", [](int i, int j) { return py::make_tuple(i, j); }, + py::kw_only(), py::arg("i"), py::arg("j")); + m.def("kw_only_some", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg(), py::kw_only(), py::arg("j"), py::arg("k")); + m.def("kw_only_with_defaults", [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, + py::arg() = 3, "j"_a = 4, py::kw_only(), "k"_a = 5, "z"_a); + m.def("kw_only_mixed", [](int i, int j) { return py::make_tuple(i, j); }, + "i"_a, py::kw_only(), "j"_a); + m.def("kw_only_plus_more", [](int i, int j, int k, py::kwargs kwargs) { return py::make_tuple(i, j, k, kwargs); }, - py::arg() /* positional */, py::arg("j") = -1 /* both */, py::kwonly(), py::arg("k") /* kw-only */); + py::arg() /* positional */, py::arg("j") = -1 /* both */, py::kw_only(), py::arg("k") /* kw-only */); - m.def("register_invalid_kwonly", [](py::module m) { - m.def("bad_kwonly", [](int i, int j) { return py::make_tuple(i, j); }, - py::kwonly(), py::arg() /* invalid unnamed argument */, "j"_a); + m.def("register_invalid_kw_only", [](py::module_ m) { + m.def("bad_kw_only", [](int i, int j) { return py::make_tuple(i, j); }, + py::kw_only(), py::arg() /* invalid unnamed argument */, "j"_a); }); + // test_positional_only_args + m.def("pos_only_all", [](int i, int j) { return py::make_tuple(i, j); }, + py::arg("i"), py::arg("j"), py::pos_only()); + m.def("pos_only_mix", [](int i, int j) { return py::make_tuple(i, j); }, + py::arg("i"), py::pos_only(), py::arg("j")); + m.def("pos_kw_only_mix", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg("i"), py::pos_only(), py::arg("j"), py::kw_only(), py::arg("k")); + m.def("pos_only_def_mix", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg("i"), py::arg("j") = 2, py::pos_only(), py::arg("k") = 3); + + // These should fail to compile: - // argument annotations are required when using kwonly -// m.def("bad_kwonly1", [](int) {}, py::kwonly()); - // can't specify both `py::kwonly` and a `py::args` argument -// m.def("bad_kwonly2", [](int i, py::args) {}, py::kwonly(), "i"_a); + // argument annotations are required when using kw_only +// m.def("bad_kw_only1", [](int) {}, py::kw_only()); + // can't specify both `py::kw_only` and a `py::args` argument +// m.def("bad_kw_only2", [](int i, py::args) {}, py::kw_only(), "i"_a); // test_function_signatures (along with most of the above) struct KWClass { void foo(int, float) {} }; @@ -127,5 +138,5 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // Make sure a class (not an instance) can be used as a default argument. // The return value doesn't matter, only that the module is importable. m.def("class_default_argument", [](py::object a) { return py::repr(a); }, - "a"_a = py::module::import("decimal").attr("Decimal")); + "a"_a = py::module_::import("decimal").attr("Decimal")); } diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index dad40dbebf..12fe705b4d 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- import pytest -from pybind11_tests import kwargs_and_defaults as m -import platform -import sys +import env # noqa: F401 -pypy = platform.python_implementation() == "PyPy" +from pybind11_tests import kwargs_and_defaults as m def test_function_signatures(doc): @@ -17,11 +15,17 @@ def test_function_signatures(doc): assert doc(m.kw_func_udl) == "kw_func_udl(x: int, y: int = 300) -> str" assert doc(m.kw_func_udl_z) == "kw_func_udl_z(x: int, y: int = 0) -> str" assert doc(m.args_function) == "args_function(*args) -> tuple" - assert doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple" - assert doc(m.KWClass.foo0) == \ - "foo0(self: m.kwargs_and_defaults.KWClass, arg0: int, arg1: float) -> None" - assert doc(m.KWClass.foo1) == \ - "foo1(self: m.kwargs_and_defaults.KWClass, x: int, y: float) -> None" + assert ( + doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple" + ) + assert ( + doc(m.KWClass.foo0) + == "foo0(self: m.kwargs_and_defaults.KWClass, arg0: int, arg1: float) -> None" + ) + assert ( + doc(m.KWClass.foo1) + == "foo1(self: m.kwargs_and_defaults.KWClass, x: int, y: float) -> None" + ) def test_named_arguments(msg): @@ -42,7 +46,9 @@ def test_named_arguments(msg): # noinspection PyArgumentList m.kw_func2(x=5, y=10, z=12) assert excinfo.match( - r'(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))' + '{3}$') + r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))" + + "{3}$" + ) assert m.kw_func4() == "{13 17}" assert m.kw_func4(myList=[1, 2, 3]) == "{1 2 3}" @@ -52,11 +58,11 @@ def test_named_arguments(msg): def test_arg_and_kwargs(): - args = 'arg1_value', 'arg2_value', 3 + args = "arg1_value", "arg2_value", 3 assert m.args_function(*args) == args - args = 'a1', 'a2' - kwargs = dict(arg3='a3', arg4=4) + args = "a1", "a2" + kwargs = dict(arg3="a3", arg4=4) assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) @@ -70,89 +76,166 @@ def test_mixed_args_and_kwargs(msg): assert mpa(1, 2.5) == (1, 2.5, ()) with pytest.raises(TypeError) as excinfo: assert mpa(1) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ mixed_plus_args(): incompatible function arguments. The following argument types are supported: 1. (arg0: int, arg1: float, *args) -> tuple Invoked with: 1 """ # noqa: E501 line too long + ) with pytest.raises(TypeError) as excinfo: assert mpa() - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ mixed_plus_args(): incompatible function arguments. The following argument types are supported: 1. (arg0: int, arg1: float, *args) -> tuple Invoked with: """ # noqa: E501 line too long + ) - assert mpk(-2, 3.5, pi=3.14159, e=2.71828) == (-2, 3.5, {'e': 2.71828, 'pi': 3.14159}) + assert mpk(-2, 3.5, pi=3.14159, e=2.71828) == ( + -2, + 3.5, + {"e": 2.71828, "pi": 3.14159}, + ) assert mpak(7, 7.7, 7.77, 7.777, 7.7777, minusseven=-7) == ( - 7, 7.7, (7.77, 7.777, 7.7777), {'minusseven': -7}) + 7, + 7.7, + (7.77, 7.777, 7.7777), + {"minusseven": -7}, + ) assert mpakd() == (1, 3.14159, (), {}) assert mpakd(3) == (3, 3.14159, (), {}) assert mpakd(j=2.71828) == (1, 2.71828, (), {}) - assert mpakd(k=42) == (1, 3.14159, (), {'k': 42}) + assert mpakd(k=42) == (1, 3.14159, (), {"k": 42}) assert mpakd(1, 1, 2, 3, 5, 8, then=13, followedby=21) == ( - 1, 1, (2, 3, 5, 8), {'then': 13, 'followedby': 21}) + 1, + 1, + (2, 3, 5, 8), + {"then": 13, "followedby": 21}, + ) # Arguments specified both positionally and via kwargs should fail: with pytest.raises(TypeError) as excinfo: assert mpakd(1, i=1) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported: 1. (i: int = 1, j: float = 3.14159, *args, **kwargs) -> tuple Invoked with: 1; kwargs: i=1 """ # noqa: E501 line too long + ) with pytest.raises(TypeError) as excinfo: assert mpakd(1, 2, j=1) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported: 1. (i: int = 1, j: float = 3.14159, *args, **kwargs) -> tuple Invoked with: 1, 2; kwargs: j=1 """ # noqa: E501 line too long + ) def test_keyword_only_args(msg): - assert m.kwonly_all(i=1, j=2) == (1, 2) - assert m.kwonly_all(j=1, i=2) == (2, 1) + assert m.kw_only_all(i=1, j=2) == (1, 2) + assert m.kw_only_all(j=1, i=2) == (2, 1) with pytest.raises(TypeError) as excinfo: - assert m.kwonly_all(i=1) == (1,) + assert m.kw_only_all(i=1) == (1,) assert "incompatible function arguments" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: - assert m.kwonly_all(1, 2) == (1, 2) + assert m.kw_only_all(1, 2) == (1, 2) assert "incompatible function arguments" in str(excinfo.value) - assert m.kwonly_some(1, k=3, j=2) == (1, 2, 3) + assert m.kw_only_some(1, k=3, j=2) == (1, 2, 3) - assert m.kwonly_with_defaults(z=8) == (3, 4, 5, 8) - assert m.kwonly_with_defaults(2, z=8) == (2, 4, 5, 8) - assert m.kwonly_with_defaults(2, j=7, k=8, z=9) == (2, 7, 8, 9) - assert m.kwonly_with_defaults(2, 7, z=9, k=8) == (2, 7, 8, 9) + assert m.kw_only_with_defaults(z=8) == (3, 4, 5, 8) + assert m.kw_only_with_defaults(2, z=8) == (2, 4, 5, 8) + assert m.kw_only_with_defaults(2, j=7, k=8, z=9) == (2, 7, 8, 9) + assert m.kw_only_with_defaults(2, 7, z=9, k=8) == (2, 7, 8, 9) - assert m.kwonly_mixed(1, j=2) == (1, 2) - assert m.kwonly_mixed(j=2, i=3) == (3, 2) - assert m.kwonly_mixed(i=2, j=3) == (2, 3) + assert m.kw_only_mixed(1, j=2) == (1, 2) + assert m.kw_only_mixed(j=2, i=3) == (3, 2) + assert m.kw_only_mixed(i=2, j=3) == (2, 3) - assert m.kwonly_plus_more(4, 5, k=6, extra=7) == (4, 5, 6, {'extra': 7}) - assert m.kwonly_plus_more(3, k=5, j=4, extra=6) == (3, 4, 5, {'extra': 6}) - assert m.kwonly_plus_more(2, k=3, extra=4) == (2, -1, 3, {'extra': 4}) + assert m.kw_only_plus_more(4, 5, k=6, extra=7) == (4, 5, 6, {"extra": 7}) + assert m.kw_only_plus_more(3, k=5, j=4, extra=6) == (3, 4, 5, {"extra": 6}) + assert m.kw_only_plus_more(2, k=3, extra=4) == (2, -1, 3, {"extra": 4}) with pytest.raises(TypeError) as excinfo: - assert m.kwonly_mixed(i=1) == (1,) + assert m.kw_only_mixed(i=1) == (1,) assert "incompatible function arguments" in str(excinfo.value) with pytest.raises(RuntimeError) as excinfo: - m.register_invalid_kwonly(m) - assert msg(excinfo.value) == """ - arg(): cannot specify an unnamed argument after an kwonly() annotation + m.register_invalid_kw_only(m) + assert ( + msg(excinfo.value) + == """ + arg(): cannot specify an unnamed argument after an kw_only() annotation """ + ) + + +def test_positional_only_args(msg): + assert m.pos_only_all(1, 2) == (1, 2) + assert m.pos_only_all(2, 1) == (2, 1) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_all(i=1, j=2) + assert "incompatible function arguments" in str(excinfo.value) + + assert m.pos_only_mix(1, 2) == (1, 2) + assert m.pos_only_mix(2, j=1) == (2, 1) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_mix(i=1, j=2) + assert "incompatible function arguments" in str(excinfo.value) + + assert m.pos_kw_only_mix(1, 2, k=3) == (1, 2, 3) + assert m.pos_kw_only_mix(1, j=2, k=3) == (1, 2, 3) + + with pytest.raises(TypeError) as excinfo: + m.pos_kw_only_mix(i=1, j=2, k=3) + assert "incompatible function arguments" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + m.pos_kw_only_mix(1, 2, 3) + assert "incompatible function arguments" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_def_mix() + assert "incompatible function arguments" in str(excinfo.value) + + assert m.pos_only_def_mix(1) == (1, 2, 3) + assert m.pos_only_def_mix(1, 4) == (1, 4, 3) + assert m.pos_only_def_mix(1, 4, 7) == (1, 4, 7) + assert m.pos_only_def_mix(1, 4, k=7) == (1, 4, 7) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_def_mix(1, j=4) + assert "incompatible function arguments" in str(excinfo.value) + + +def test_signatures(): + assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__ + assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__ + assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__ + assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__ + assert ( + "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" + == m.pos_kw_only_mix.__doc__ + ) -@pytest.mark.xfail(pypy and sys.version_info < (3, 0), - reason="PyPy2 doesn't seem to double count") +@pytest.mark.xfail("env.PYPY and env.PY2", reason="PyPy2 doesn't double count") def test_args_refcount(): """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular arguments""" @@ -174,11 +257,18 @@ def test_args_refcount(): assert m.args_function(-1, myval) == (-1, myval) assert refcount(myval) == expected - assert m.mixed_plus_args_kwargs(5, 6.0, myval, a=myval) == (5, 6.0, (myval,), {"a": myval}) + assert m.mixed_plus_args_kwargs(5, 6.0, myval, a=myval) == ( + 5, + 6.0, + (myval,), + {"a": myval}, + ) assert refcount(myval) == expected - assert m.args_kwargs_function(7, 8, myval, a=1, b=myval) == \ - ((7, 8, myval), {"a": 1, "b": myval}) + assert m.args_kwargs_function(7, 8, myval, a=1, b=myval) == ( + (7, 8, myval), + {"a": 1, "b": myval}, + ) assert refcount(myval) == expected exp3 = refcount(myval, myval, myval) diff --git a/tests/test_local_bindings.cpp b/tests/test_local_bindings.cpp index 97c02dbeb5..c61e388833 100644 --- a/tests/test_local_bindings.cpp +++ b/tests/test_local_bindings.cpp @@ -41,7 +41,7 @@ TEST_SUBMODULE(local_bindings, m) { // should raise a runtime error from the duplicate definition attempt. If test_class isn't // available it *also* throws a runtime error (with "test_class not enabled" as value). m.def("register_local_external", [m]() { - auto main = py::module::import("pybind11_tests"); + auto main = py::module_::import("pybind11_tests"); if (py::hasattr(main, "class_")) { bind_local(m, "LocalExternal", py::module_local()); } diff --git a/tests/test_local_bindings.py b/tests/test_local_bindings.py index 913cf0ee5b..a38564b6e1 100644 --- a/tests/test_local_bindings.py +++ b/tests/test_local_bindings.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import pytest +import env # noqa: F401 + from pybind11_tests import local_bindings as m @@ -34,8 +36,8 @@ def test_local_bindings(): assert i2.get() == 11 assert i2.get2() == 12 - assert not hasattr(i1, 'get2') - assert not hasattr(i2, 'get3') + assert not hasattr(i1, "get2") + assert not hasattr(i2, "get3") # Loading within the local module assert m.local_value(i1) == 5 @@ -53,7 +55,9 @@ def test_nonlocal_failure(): with pytest.raises(RuntimeError) as excinfo: cm.register_nonlocal() - assert str(excinfo.value) == 'generic_type: type "NonLocalType" is already registered!' + assert ( + str(excinfo.value) == 'generic_type: type "NonLocalType" is already registered!' + ) def test_duplicate_local(): @@ -61,9 +65,12 @@ def test_duplicate_local(): with pytest.raises(RuntimeError) as excinfo: m.register_local_external() import pybind11_tests + assert str(excinfo.value) == ( 'generic_type: type "LocalExternal" is already registered!' - if hasattr(pybind11_tests, 'class_') else 'test_class not enabled') + if hasattr(pybind11_tests, "class_") + else "test_class not enabled" + ) def test_stl_bind_local(): @@ -96,8 +103,8 @@ def test_stl_bind_local(): d1["b"] = v1[1] d2["c"] = v2[0] d2["d"] = v2[1] - assert {i: d1[i].get() for i in d1} == {'a': 0, 'b': 1} - assert {i: d2[i].get() for i in d2} == {'c': 2, 'd': 3} + assert {i: d1[i].get() for i in d1} == {"a": 0, "b": 1} + assert {i: d2[i].get() for i in d2} == {"c": 2, "d": 3} def test_stl_bind_global(): @@ -105,15 +112,21 @@ def test_stl_bind_global(): with pytest.raises(RuntimeError) as excinfo: cm.register_nonlocal_map() - assert str(excinfo.value) == 'generic_type: type "NonLocalMap" is already registered!' + assert ( + str(excinfo.value) == 'generic_type: type "NonLocalMap" is already registered!' + ) with pytest.raises(RuntimeError) as excinfo: cm.register_nonlocal_vec() - assert str(excinfo.value) == 'generic_type: type "NonLocalVec" is already registered!' + assert ( + str(excinfo.value) == 'generic_type: type "NonLocalVec" is already registered!' + ) with pytest.raises(RuntimeError) as excinfo: cm.register_nonlocal_map2() - assert str(excinfo.value) == 'generic_type: type "NonLocalMap2" is already registered!' + assert ( + str(excinfo.value) == 'generic_type: type "NonLocalMap2" is already registered!' + ) def test_mixed_local_global(): @@ -121,6 +134,7 @@ def test_mixed_local_global(): type can be registered even if the type is already registered globally. With the module, casting will go to the local type; outside the module casting goes to the global type.""" import pybind11_cross_module_tests as cm + m.register_mixed_global() m.register_mixed_local() @@ -143,17 +157,30 @@ def test_mixed_local_global(): a.append(cm.get_mixed_gl(11)) a.append(cm.get_mixed_lg(12)) - assert [x.get() for x in a] == \ - [101, 1002, 103, 1004, 105, 1006, 207, 2008, 109, 1010, 211, 2012] + assert [x.get() for x in a] == [ + 101, + 1002, + 103, + 1004, + 105, + 1006, + 207, + 2008, + 109, + 1010, + 211, + 2012, + ] def test_internal_locals_differ(): """Makes sure the internal local type map differs across the two modules""" import pybind11_cross_module_tests as cm + assert m.local_cpp_types_addr() != cm.local_cpp_types_addr() -@pytest.bug_in_pypy +@pytest.mark.xfail("env.PYPY and sys.pypy_version_info < (7, 3, 2)") def test_stl_caster_vs_stl_bind(msg): """One module uses a generic vector caster from `` while the other exports `std::vector` via `py:bind_vector` and `py::module_local`""" @@ -166,13 +193,16 @@ def test_stl_caster_vs_stl_bind(msg): v2 = [1, 2, 3] assert m.load_vector_via_caster(v2) == 6 with pytest.raises(TypeError) as excinfo: - cm.load_vector_via_binding(v2) == 6 - assert msg(excinfo.value) == """ + cm.load_vector_via_binding(v2) + assert ( + msg(excinfo.value) + == """ load_vector_via_binding(): incompatible function arguments. The following argument types are supported: 1. (arg0: pybind11_cross_module_tests.VectorInt) -> int Invoked with: [1, 2, 3] """ # noqa: E501 line too long + ) def test_cross_module_calls(): diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 11d4e7b350..f99909bda7 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -207,12 +207,12 @@ TEST_SUBMODULE(methods_and_attributes, m) { // test_no_mixed_overloads // Raise error if trying to mix static/non-static overloads on the same name: .def_static("add_mixed_overloads1", []() { - auto emna = py::reinterpret_borrow>(py::module::import("pybind11_tests.methods_and_attributes").attr("ExampleMandA")); + auto emna = py::reinterpret_borrow>(py::module_::import("pybind11_tests.methods_and_attributes").attr("ExampleMandA")); emna.def ("overload_mixed1", static_cast(&ExampleMandA::overloaded)) .def_static("overload_mixed1", static_cast(&ExampleMandA::overloaded)); }) .def_static("add_mixed_overloads2", []() { - auto emna = py::reinterpret_borrow>(py::module::import("pybind11_tests.methods_and_attributes").attr("ExampleMandA")); + auto emna = py::reinterpret_borrow>(py::module_::import("pybind11_tests.methods_and_attributes").attr("ExampleMandA")); emna.def_static("overload_mixed2", static_cast(&ExampleMandA::overloaded)) .def ("overload_mixed2", static_cast(&ExampleMandA::overloaded)); }) @@ -284,6 +284,12 @@ TEST_SUBMODULE(methods_and_attributes, m) { py::class_(m, "MetaclassOverride", py::metaclass((PyObject *) &PyType_Type)) .def_property_readonly_static("readonly", [](py::object) { return 1; }); + // test_overload_ordering + m.def("overload_order", [](std::string) { return 1; }); + m.def("overload_order", [](std::string) { return 2; }); + m.def("overload_order", [](int) { return 3; }); + m.def("overload_order", [](int) { return 4; }, py::prepend{}); + #if !defined(PYPY_VERSION) // test_dynamic_attributes class DynamicClass { @@ -308,28 +314,33 @@ TEST_SUBMODULE(methods_and_attributes, m) { m.attr("debug_enabled") = false; #endif m.def("bad_arg_def_named", []{ - auto m = py::module::import("pybind11_tests"); + auto m = py::module_::import("pybind11_tests"); m.def("should_fail", [](int, UnregisteredType) {}, py::arg(), py::arg("a") = UnregisteredType()); }); m.def("bad_arg_def_unnamed", []{ - auto m = py::module::import("pybind11_tests"); + auto m = py::module_::import("pybind11_tests"); m.def("should_fail", [](int, UnregisteredType) {}, py::arg(), py::arg() = UnregisteredType()); }); + // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. + // test_accepts_none py::class_>(m, "NoneTester") .def(py::init<>()); - m.def("no_none1", &none1, py::arg().none(false)); - m.def("no_none2", &none2, py::arg().none(false)); - m.def("no_none3", &none3, py::arg().none(false)); - m.def("no_none4", &none4, py::arg().none(false)); - m.def("no_none5", &none5, py::arg().none(false)); + m.def("no_none1", &none1, py::arg{}.none(false)); + m.def("no_none2", &none2, py::arg{}.none(false)); + m.def("no_none3", &none3, py::arg{}.none(false)); + m.def("no_none4", &none4, py::arg{}.none(false)); + m.def("no_none5", &none5, py::arg{}.none(false)); m.def("ok_none1", &none1); - m.def("ok_none2", &none2, py::arg().none(true)); + m.def("ok_none2", &none2, py::arg{}.none(true)); m.def("ok_none3", &none3); - m.def("ok_none4", &none4, py::arg().none(true)); + m.def("ok_none4", &none4, py::arg{}.none(true)); m.def("ok_none5", &none5); + m.def("no_none_kwarg", &none2, "a"_a.none(false)); + m.def("no_none_kwarg_kw_only", &none2, py::kw_only(), "a"_a.none(false)); + // test_str_issue // Issue #283: __str__ called on uninitialized instance when constructor arguments invalid py::class_(m, "StrIssue") diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 6515cbf594..9c6bc22903 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- import pytest + +import env # noqa: F401 + from pybind11_tests import methods_and_attributes as m from pybind11_tests import ConstructorStats @@ -37,17 +40,17 @@ def test_methods_and_attributes(): assert instance1.overloaded(0) == "(int)" assert instance1.overloaded(1, 1.0) == "(int, float)" assert instance1.overloaded(2.0, 2) == "(float, int)" - assert instance1.overloaded(3, 3) == "(int, int)" - assert instance1.overloaded(4., 4.) == "(float, float)" + assert instance1.overloaded(3, 3) == "(int, int)" + assert instance1.overloaded(4.0, 4.0) == "(float, float)" assert instance1.overloaded_const(-3) == "(int) const" assert instance1.overloaded_const(5, 5.0) == "(int, float) const" assert instance1.overloaded_const(6.0, 6) == "(float, int) const" - assert instance1.overloaded_const(7, 7) == "(int, int) const" - assert instance1.overloaded_const(8., 8.) == "(float, float) const" + assert instance1.overloaded_const(7, 7) == "(int, int) const" + assert instance1.overloaded_const(8.0, 8.0) == "(float, float) const" assert instance1.overloaded_float(1, 1) == "(float, float)" - assert instance1.overloaded_float(1, 1.) == "(float, float)" - assert instance1.overloaded_float(1., 1) == "(float, float)" - assert instance1.overloaded_float(1., 1.) == "(float, float)" + assert instance1.overloaded_float(1, 1.0) == "(float, float)" + assert instance1.overloaded_float(1.0, 1) == "(float, float)" + assert instance1.overloaded_float(1.0, 1.0) == "(float, float)" assert instance1.value == 320 instance1.value = 100 @@ -168,6 +171,19 @@ def test_static_properties(): assert m.TestPropertiesOverride().def_readonly == 99 assert m.TestPropertiesOverride.def_readonly_static == 99 + # Only static attributes can be deleted + del m.TestPropertiesOverride.def_readonly_static + assert ( + hasattr(m.TestPropertiesOverride, "def_readonly_static") + and m.TestPropertiesOverride.def_readonly_static + is m.TestProperties.def_readonly_static + ) + assert "def_readonly_static" not in m.TestPropertiesOverride.__dict__ + properties_override = m.TestPropertiesOverride() + with pytest.raises(AttributeError) as excinfo: + del properties_override.def_readonly + assert "can't delete attribute" in str(excinfo.value) + def test_static_cls(): """Static property getter and setters expect the type object as the their only argument""" @@ -190,7 +206,10 @@ def test_metaclass_override(): assert type(m.MetaclassOverride).__name__ == "type" assert m.MetaclassOverride.readonly == 1 - assert type(m.MetaclassOverride.__dict__["readonly"]).__name__ == "pybind11_static_property" + assert ( + type(m.MetaclassOverride.__dict__["readonly"]).__name__ + == "pybind11_static_property" + ) # Regular `type` replaces the property instead of calling `__set__()` m.MetaclassOverride.readonly = 2 @@ -203,22 +222,26 @@ def test_no_mixed_overloads(): with pytest.raises(RuntimeError) as excinfo: m.ExampleMandA.add_mixed_overloads1() - assert (str(excinfo.value) == - "overloading a method with both static and instance methods is not supported; " + - ("compile in debug mode for more details" if not debug_enabled else - "error while attempting to bind static method ExampleMandA.overload_mixed1" - "(arg0: float) -> str") - ) + assert str( + excinfo.value + ) == "overloading a method with both static and instance methods is not supported; " + ( + "compile in debug mode for more details" + if not debug_enabled + else "error while attempting to bind static method ExampleMandA.overload_mixed1" + "(arg0: float) -> str" + ) with pytest.raises(RuntimeError) as excinfo: m.ExampleMandA.add_mixed_overloads2() - assert (str(excinfo.value) == - "overloading a method with both static and instance methods is not supported; " + - ("compile in debug mode for more details" if not debug_enabled else - "error while attempting to bind instance method ExampleMandA.overload_mixed2" - "(self: pybind11_tests.methods_and_attributes.ExampleMandA, arg0: int, arg1: int)" - " -> str") - ) + assert str( + excinfo.value + ) == "overloading a method with both static and instance methods is not supported; " + ( + "compile in debug mode for more details" + if not debug_enabled + else "error while attempting to bind instance method ExampleMandA.overload_mixed2" + "(self: pybind11_tests.methods_and_attributes.ExampleMandA, arg0: int, arg1: int)" + " -> str" + ) @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) @@ -257,8 +280,8 @@ def test_property_rvalue_policy(): assert os.value == 1 -# https://bitbucket.org/pypy/pypy/issues/2447 -@pytest.unsupported_on_pypy +# https://foss.heptapod.net/pypy/pypy/-/issues/2447 +@pytest.mark.xfail("env.PYPY") def test_dynamic_attributes(): instance = m.DynamicClass() assert not hasattr(instance, "foo") @@ -299,8 +322,8 @@ class PythonDerivedDynamicClass(m.DynamicClass): assert cstats.alive() == 0 -# https://bitbucket.org/pypy/pypy/issues/2447 -@pytest.unsupported_on_pypy +# https://foss.heptapod.net/pypy/pypy/-/issues/2447 +@pytest.mark.xfail("env.PYPY") def test_cyclic_gc(): # One object references itself instance = m.DynamicClass() @@ -330,8 +353,8 @@ def test_bad_arg_default(msg): assert msg(excinfo.value) == ( "arg(): could not convert default argument 'a: UnregisteredType' in function " "'should_fail' into a Python object (type not registered yet?)" - if debug_enabled else - "arg(): could not convert default argument into a Python object (type not registered " + if debug_enabled + else "arg(): could not convert default argument into a Python object (type not registered " "yet?). Compile in debug mode for more information." ) @@ -340,8 +363,8 @@ def test_bad_arg_default(msg): assert msg(excinfo.value) == ( "arg(): could not convert default argument 'UnregisteredType' in function " "'should_fail' into a Python object (type not registered yet?)" - if debug_enabled else - "arg(): could not convert default argument into a Python object (type not registered " + if debug_enabled + else "arg(): could not convert default argument into a Python object (type not registered " "yet?). Compile in debug mode for more information." ) @@ -378,12 +401,15 @@ def test_accepts_none(msg): # The first one still raises because you can't pass None as a lvalue reference arg: with pytest.raises(TypeError) as excinfo: assert m.ok_none1(None) == -1 - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ ok_none1(): incompatible function arguments. The following argument types are supported: 1. (arg0: m.methods_and_attributes.NoneTester) -> int Invoked with: None """ + ) # The rest take the argument as pointer or holder, and accept None: assert m.ok_none2(None) == -1 @@ -391,6 +417,19 @@ def test_accepts_none(msg): assert m.ok_none4(None) == -1 assert m.ok_none5(None) == -1 + with pytest.raises(TypeError) as excinfo: + m.no_none_kwarg(None) + assert "incompatible function arguments" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + m.no_none_kwarg(a=None) + assert "incompatible function arguments" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + m.no_none_kwarg_kw_only(None) + assert "incompatible function arguments" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + m.no_none_kwarg_kw_only(a=None) + assert "incompatible function arguments" in str(excinfo.value) + def test_str_issue(msg): """#283: __str__ called on uninitialized instance when constructor arguments invalid""" @@ -399,13 +438,16 @@ def test_str_issue(msg): with pytest.raises(TypeError) as excinfo: str(m.StrIssue("no", "such", "constructor")) - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ __init__(): incompatible constructor arguments. The following argument types are supported: 1. m.methods_and_attributes.StrIssue(arg0: int) 2. m.methods_and_attributes.StrIssue() Invoked with: 'no', 'such', 'constructor' """ + ) def test_unregistered_base_implementations(): @@ -435,3 +477,31 @@ def test_ref_qualified(): r.refQualified(17) assert r.value == 17 assert r.constRefQualified(23) == 40 + + +def test_overload_ordering(): + "Check to see if the normal overload order (first defined) and prepend overload order works" + assert m.overload_order("string") == 1 + assert m.overload_order(0) == 4 + + # Different for Python 2 vs. 3 + uni_name = type(u"").__name__ + + assert "1. overload_order(arg0: int) -> int" in m.overload_order.__doc__ + assert ( + "2. overload_order(arg0: {}) -> int".format(uni_name) + in m.overload_order.__doc__ + ) + assert ( + "3. overload_order(arg0: {}) -> int".format(uni_name) + in m.overload_order.__doc__ + ) + assert "4. overload_order(arg0: int) -> int" in m.overload_order.__doc__ + + with pytest.raises(TypeError) as err: + m.overload_order(1.1) + + assert "1. (arg0: int) -> int" in str(err.value) + assert "2. (arg0: {}) -> int".format(uni_name) in str(err.value) + assert "3. (arg0: {}) -> int".format(uni_name) in str(err.value) + assert "4. (arg0: int) -> int" in str(err.value) diff --git a/tests/test_modules.cpp b/tests/test_modules.cpp index c1475fa623..67387e809b 100644 --- a/tests/test_modules.cpp +++ b/tests/test_modules.cpp @@ -13,6 +13,7 @@ TEST_SUBMODULE(modules, m) { // test_nested_modules + // This is intentionally "py::module" to verify it still can be used in place of "py::module_" py::module m_sub = m.def_submodule("subsubmodule"); m_sub.def("submodule_func", []() { return "submodule_func()"; }); @@ -50,6 +51,7 @@ TEST_SUBMODULE(modules, m) { .def_readwrite("a1", &B::a1) // def_readonly uses an internal reference return policy by default .def_readwrite("a2", &B::a2); + // This is intentionally "py::module" to verify it still can be used in place of "py::module_" m.attr("OD") = py::module::import("collections").attr("OrderedDict"); // test_duplicate_registration @@ -60,7 +62,8 @@ TEST_SUBMODULE(modules, m) { class Dupe3 { }; class DupeException { }; - auto dm = py::module("dummy"); + // Go ahead and leak, until we have a non-leaking py::module_ constructor + auto dm = py::module_::create_extension_module("dummy", nullptr, new py::module_::module_def); auto failures = py::list(); py::class_(dm, "Dupe1"); diff --git a/tests/test_modules.py b/tests/test_modules.py index 7e21005245..3390031aff 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -6,9 +6,13 @@ def test_nested_modules(): import pybind11_tests + assert pybind11_tests.__name__ == "pybind11_tests" assert pybind11_tests.modules.__name__ == "pybind11_tests.modules" - assert pybind11_tests.modules.subsubmodule.__name__ == "pybind11_tests.modules.subsubmodule" + assert ( + pybind11_tests.modules.subsubmodule.__name__ + == "pybind11_tests.modules.subsubmodule" + ) assert m.__name__ == "pybind11_tests.modules" assert ms.__name__ == "pybind11_tests.modules.subsubmodule" @@ -35,7 +39,7 @@ def test_reference_internal(): del b assert astats.alive() == 0 assert bstats.alive() == 0 - assert astats.values() == ['1', '2', '42', '43'] + assert astats.values() == ["1", "2", "42", "43"] assert bstats.values() == [] assert astats.default_constructions == 0 assert bstats.default_constructions == 1 @@ -54,7 +58,7 @@ def test_importing(): from collections import OrderedDict assert OD is OrderedDict - assert str(OD([(1, 'a'), (2, 'b')])) == "OrderedDict([(1, 'a'), (2, 'b')])" + assert str(OD([(1, "a"), (2, "b")])) == "OrderedDict([(1, 'a'), (2, 'b')])" def test_pydoc(): @@ -71,3 +75,16 @@ def test_duplicate_registration(): """Registering two things with the same name""" assert m.duplicate_registration() == [] + + +def test_builtin_key_type(): + """Test that all the keys in the builtin modules have type str. + + Previous versions of pybind11 would add a unicode key in python 2. + """ + if hasattr(__builtins__, "keys"): + keys = __builtins__.keys() + else: # this is to make pypy happy since builtins is different there. + keys = __builtins__.__dict__.keys() + + assert {type(k) for k in keys} == {str} diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index bd5907c635..ec2d829db0 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -214,14 +214,12 @@ TEST_SUBMODULE(multiple_inheritance, m) { .def_readwrite_static("static_value", &VanillaStaticMix2::static_value); -#if !(defined(PYPY_VERSION) && (PYPY_VERSION_NUM < 0x06000000)) struct WithDict { }; struct VanillaDictMix1 : Vanilla, WithDict { }; struct VanillaDictMix2 : WithDict, Vanilla { }; py::class_(m, "WithDict", py::dynamic_attr()).def(py::init<>()); py::class_(m, "VanillaDictMix1").def(py::init<>()); py::class_(m, "VanillaDictMix2").def(py::init<>()); -#endif // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance diff --git a/tests/test_multiple_inheritance.py b/tests/test_multiple_inheritance.py index 146aaebcef..1d4ecd95f0 100644 --- a/tests/test_multiple_inheritance.py +++ b/tests/test_multiple_inheritance.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- import pytest + +import env # noqa: F401 + from pybind11_tests import ConstructorStats from pybind11_tests import multiple_inheritance as m @@ -11,7 +14,8 @@ def test_multiple_inheritance_cpp(): assert mt.bar() == 4 -@pytest.bug_in_pypy +@pytest.mark.skipif("env.PYPY and env.PY2") +@pytest.mark.xfail("env.PYPY and not env.PY2") def test_multiple_inheritance_mix1(): class Base1: def __init__(self, i): @@ -32,7 +36,6 @@ def __init__(self, i, j): def test_multiple_inheritance_mix2(): - class Base2: def __init__(self, i): self.i = i @@ -51,9 +54,9 @@ def __init__(self, i, j): assert mt.bar() == 4 -@pytest.bug_in_pypy +@pytest.mark.skipif("env.PYPY and env.PY2") +@pytest.mark.xfail("env.PYPY and not env.PY2") def test_multiple_inheritance_python(): - class MI1(m.Base1, m.Base2): def __init__(self, i, j): m.Base1.__init__(self, i) @@ -159,7 +162,6 @@ def __init__(self, i): def test_multiple_inheritance_python_many_bases(): - class MIMany14(m.BaseN1, m.BaseN2, m.BaseN3, m.BaseN4): def __init__(self): m.BaseN1.__init__(self, 1) @@ -174,8 +176,16 @@ def __init__(self): m.BaseN7.__init__(self, 7) m.BaseN8.__init__(self, 8) - class MIMany916(m.BaseN9, m.BaseN10, m.BaseN11, m.BaseN12, m.BaseN13, m.BaseN14, m.BaseN15, - m.BaseN16): + class MIMany916( + m.BaseN9, + m.BaseN10, + m.BaseN11, + m.BaseN12, + m.BaseN13, + m.BaseN14, + m.BaseN15, + m.BaseN16, + ): def __init__(self): m.BaseN9.__init__(self, 9) m.BaseN10.__init__(self, 10) @@ -221,7 +231,6 @@ def __init__(self): def test_multiple_inheritance_virtbase(): - class MITypePy(m.Base12a): def __init__(self, i, j): m.Base12a.__init__(self, i, j) @@ -234,7 +243,7 @@ def __init__(self, i, j): def test_mi_static_properties(): """Mixing bases with and without static properties should be possible - and the result should be independent of base definition order""" + and the result should be independent of base definition order""" for d in (m.VanillaStaticMix1(), m.VanillaStaticMix2()): assert d.vanilla() == "Vanilla" @@ -256,7 +265,7 @@ def test_mi_static_properties(): assert d.static_value == 0 -@pytest.unsupported_on_pypy_lt_6 +# Requires PyPy 6+ def test_mi_dynamic_attributes(): """Mixing bases with and without dynamic attribute support""" @@ -352,7 +361,7 @@ def test_diamond_inheritance(): assert d is d.c0().c1().b().c0().b() -@pytest.unsupported_on_pypy +@pytest.mark.skipif(env.PYPY, reason="Unsupported on PyPy") def test_mi_ownership_constraint(): # See `test_ownership_transfer` for positive tests. diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 156a3bfa8e..dca7145f9d 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -22,7 +22,7 @@ struct DtypeCheck { template DtypeCheck get_dtype_check(const char* name) { - py::module np = py::module::import("numpy"); + py::module_ np = py::module_::import("numpy"); DtypeCheck check{}; check.numpy = np.attr("dtype")(np.attr(name)); check.pybind11 = py::dtype::of(); @@ -89,23 +89,23 @@ template arr data_t(const arr_t& a, Ix... index) { template arr& mutate_data(arr& a, Ix... index) { auto ptr = (uint8_t *) a.mutable_data(index...); - for (ssize_t i = 0; i < a.nbytes() - a.offset_at(index...); i++) + for (py::ssize_t i = 0; i < a.nbytes() - a.offset_at(index...); i++) ptr[i] = (uint8_t) (ptr[i] * 2); return a; } template arr_t& mutate_data_t(arr_t& a, Ix... index) { auto ptr = a.mutable_data(index...); - for (ssize_t i = 0; i < a.size() - a.index_at(index...); i++) + for (py::ssize_t i = 0; i < a.size() - a.index_at(index...); i++) ptr[i]++; return a; } -template ssize_t index_at(const arr& a, Ix... idx) { return a.index_at(idx...); } -template ssize_t index_at_t(const arr_t& a, Ix... idx) { return a.index_at(idx...); } -template ssize_t offset_at(const arr& a, Ix... idx) { return a.offset_at(idx...); } -template ssize_t offset_at_t(const arr_t& a, Ix... idx) { return a.offset_at(idx...); } -template ssize_t at_t(const arr_t& a, Ix... idx) { return a.at(idx...); } +template py::ssize_t index_at(const arr& a, Ix... idx) { return a.index_at(idx...); } +template py::ssize_t index_at_t(const arr_t& a, Ix... idx) { return a.index_at(idx...); } +template py::ssize_t offset_at(const arr& a, Ix... idx) { return a.offset_at(idx...); } +template py::ssize_t offset_at_t(const arr_t& a, Ix... idx) { return a.offset_at(idx...); } +template py::ssize_t at_t(const arr_t& a, Ix... idx) { return a.at(idx...); } template arr_t& mutate_at_t(arr_t& a, Ix... idx) { a.mutable_at(idx...)++; return a; } #define def_index_fn(name, type) \ @@ -133,7 +133,7 @@ template py::handle auxiliaries(T &&r, T2 &&r2) { static int data_i = 42; TEST_SUBMODULE(numpy_array, sm) { - try { py::module::import("numpy"); } + try { py::module_::import("numpy"); } catch (...) { return; } // test_dtypes @@ -159,9 +159,9 @@ TEST_SUBMODULE(numpy_array, sm) { // test_array_attributes sm.def("ndim", [](const arr& a) { return a.ndim(); }); sm.def("shape", [](const arr& a) { return arr(a.ndim(), a.shape()); }); - sm.def("shape", [](const arr& a, ssize_t dim) { return a.shape(dim); }); + sm.def("shape", [](const arr& a, py::ssize_t dim) { return a.shape(dim); }); sm.def("strides", [](const arr& a) { return arr(a.ndim(), a.strides()); }); - sm.def("strides", [](const arr& a, ssize_t dim) { return a.strides(dim); }); + sm.def("strides", [](const arr& a, py::ssize_t dim) { return a.strides(dim); }); sm.def("writeable", [](const arr& a) { return a.writeable(); }); sm.def("size", [](const arr& a) { return a.size(); }); sm.def("itemsize", [](const arr& a) { return a.itemsize(); }); @@ -212,7 +212,7 @@ TEST_SUBMODULE(numpy_array, sm) { .def(py::init<>()) .def("numpy_view", [](py::object &obj) { py::print("ArrayClass::numpy_view()"); - ArrayClass &a = obj.cast(); + auto &a = obj.cast(); return py::array_t({2}, {4}, a.data, obj); } ); @@ -258,9 +258,11 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("overloaded2", [](py::array_t>) { return "float complex"; }); sm.def("overloaded2", [](py::array_t) { return "float"; }); + // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. + // Only accept the exact types: - sm.def("overloaded3", [](py::array_t) { return "int"; }, py::arg().noconvert()); - sm.def("overloaded3", [](py::array_t) { return "double"; }, py::arg().noconvert()); + sm.def("overloaded3", [](py::array_t) { return "int"; }, py::arg{}.noconvert()); + sm.def("overloaded3", [](py::array_t) { return "double"; }, py::arg{}.noconvert()); // Make sure we don't do unsafe coercion (e.g. float to int) when not using forcecast, but // rather that float gets converted via the safe (conversion to double) overload: @@ -281,33 +283,33 @@ TEST_SUBMODULE(numpy_array, sm) { // test_array_unchecked_fixed_dims sm.def("proxy_add2", [](py::array_t a, double v) { auto r = a.mutable_unchecked<2>(); - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) r(i, j) += v; - }, py::arg().noconvert(), py::arg()); + }, py::arg{}.noconvert(), py::arg()); sm.def("proxy_init3", [](double start) { py::array_t a({ 3, 3, 3 }); auto r = a.mutable_unchecked<3>(); - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t k = 0; k < r.shape(2); k++) r(i, j, k) = start++; return a; }); sm.def("proxy_init3F", [](double start) { py::array_t a({ 3, 3, 3 }); auto r = a.mutable_unchecked<3>(); - for (ssize_t k = 0; k < r.shape(2); k++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t i = 0; i < r.shape(0); i++) r(i, j, k) = start++; return a; }); sm.def("proxy_squared_L2_norm", [](py::array_t a) { auto r = a.unchecked<1>(); double sumsq = 0; - for (ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t i = 0; i < r.shape(0); i++) sumsq += r[i] * r(i); // Either notation works for a 1D array return sumsq; }); @@ -318,22 +320,34 @@ TEST_SUBMODULE(numpy_array, sm) { return auxiliaries(r, r2); }); + sm.def("proxy_auxiliaries1_const_ref", [](py::array_t a) { + const auto &r = a.unchecked<1>(); + const auto &r2 = a.mutable_unchecked<1>(); + return r(0) == r2(0) && r[0] == r2[0]; + }); + + sm.def("proxy_auxiliaries2_const_ref", [](py::array_t a) { + const auto &r = a.unchecked<2>(); + const auto &r2 = a.mutable_unchecked<2>(); + return r(0, 0) == r2(0, 0); + }); + // test_array_unchecked_dyn_dims // Same as the above, but without a compile-time dimensions specification: sm.def("proxy_add2_dyn", [](py::array_t a, double v) { auto r = a.mutable_unchecked(); if (r.ndim() != 2) throw std::domain_error("error: ndim != 2"); - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) r(i, j) += v; - }, py::arg().noconvert(), py::arg()); + }, py::arg{}.noconvert(), py::arg()); sm.def("proxy_init3_dyn", [](double start) { py::array_t a({ 3, 3, 3 }); auto r = a.mutable_unchecked(); if (r.ndim() != 3) throw std::domain_error("error: ndim != 3"); - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t k = 0; k < r.shape(2); k++) r(i, j, k) = start++; return a; }); @@ -362,7 +376,7 @@ TEST_SUBMODULE(numpy_array, sm) { // test_array_resize // reshape array to 2D without changing size sm.def("array_reshape2", [](py::array_t a) { - const ssize_t dim_sz = (ssize_t)std::sqrt(a.size()); + const auto dim_sz = (py::ssize_t)std::sqrt(a.size()); if (dim_sz * dim_sz != a.size()) throw std::domain_error("array_reshape2: input array total size is not a squared integer"); a.resize({dim_sz, dim_sz}); @@ -382,9 +396,45 @@ TEST_SUBMODULE(numpy_array, sm) { return a; }); -#if PY_MAJOR_VERSION >= 3 - sm.def("index_using_ellipsis", [](py::array a) { - return a[py::make_tuple(0, py::ellipsis(), 0)]; - }); -#endif + sm.def("index_using_ellipsis", [](py::array a) { + return a[py::make_tuple(0, py::ellipsis(), 0)]; + }); + + // test_argument_conversions + sm.def("accept_double", + [](py::array_t) {}, + py::arg("a")); + sm.def("accept_double_forcecast", + [](py::array_t) {}, + py::arg("a")); + sm.def("accept_double_c_style", + [](py::array_t) {}, + py::arg("a")); + sm.def("accept_double_c_style_forcecast", + [](py::array_t) {}, + py::arg("a")); + sm.def("accept_double_f_style", + [](py::array_t) {}, + py::arg("a")); + sm.def("accept_double_f_style_forcecast", + [](py::array_t) {}, + py::arg("a")); + sm.def("accept_double_noconvert", + [](py::array_t) {}, + "a"_a.noconvert()); + sm.def("accept_double_forcecast_noconvert", + [](py::array_t) {}, + "a"_a.noconvert()); + sm.def("accept_double_c_style_noconvert", + [](py::array_t) {}, + "a"_a.noconvert()); + sm.def("accept_double_c_style_forcecast_noconvert", + [](py::array_t) {}, + "a"_a.noconvert()); + sm.def("accept_double_f_style_noconvert", + [](py::array_t) {}, + "a"_a.noconvert()); + sm.def("accept_double_f_style_forcecast_noconvert", + [](py::array_t) {}, + "a"_a.noconvert()); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 2c977cd62e..02f3ecfc0e 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- import pytest -from pybind11_tests import numpy_array as m -pytestmark = pytest.requires_numpy +import env # noqa: F401 + +from pybind11_tests import numpy_array as m -with pytest.suppress(ImportError): - import numpy as np +np = pytest.importorskip("numpy") def test_dtypes(): @@ -19,33 +19,36 @@ def test_dtypes(): print(check) assert check.numpy == check.pybind11, check if check.numpy.num != check.pybind11.num: - print("NOTE: typenum mismatch for {}: {} != {}".format( - check, check.numpy.num, check.pybind11.num)) + print( + "NOTE: typenum mismatch for {}: {} != {}".format( + check, check.numpy.num, check.pybind11.num + ) + ) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def arr(): - return np.array([[1, 2, 3], [4, 5, 6]], '=u2') + return np.array([[1, 2, 3], [4, 5, 6]], "=u2") def test_array_attributes(): - a = np.array(0, 'f8') + a = np.array(0, "f8") assert m.ndim(a) == 0 assert all(m.shape(a) == []) assert all(m.strides(a) == []) with pytest.raises(IndexError) as excinfo: m.shape(a, 0) - assert str(excinfo.value) == 'invalid axis: 0 (ndim = 0)' + assert str(excinfo.value) == "invalid axis: 0 (ndim = 0)" with pytest.raises(IndexError) as excinfo: m.strides(a, 0) - assert str(excinfo.value) == 'invalid axis: 0 (ndim = 0)' + assert str(excinfo.value) == "invalid axis: 0 (ndim = 0)" assert m.writeable(a) assert m.size(a) == 1 assert m.itemsize(a) == 8 assert m.nbytes(a) == 8 assert m.owndata(a) - a = np.array([[1, 2, 3], [4, 5, 6]], 'u2').view() + a = np.array([[1, 2, 3], [4, 5, 6]], "u2").view() a.flags.writeable = False assert m.ndim(a) == 2 assert all(m.shape(a) == [2, 3]) @@ -56,10 +59,10 @@ def test_array_attributes(): assert m.strides(a, 1) == 2 with pytest.raises(IndexError) as excinfo: m.shape(a, 2) - assert str(excinfo.value) == 'invalid axis: 2 (ndim = 2)' + assert str(excinfo.value) == "invalid axis: 2 (ndim = 2)" with pytest.raises(IndexError) as excinfo: m.strides(a, 2) - assert str(excinfo.value) == 'invalid axis: 2 (ndim = 2)' + assert str(excinfo.value) == "invalid axis: 2 (ndim = 2)" assert not m.writeable(a) assert m.size(a) == 6 assert m.itemsize(a) == 2 @@ -67,7 +70,9 @@ def test_array_attributes(): assert not m.owndata(a) -@pytest.mark.parametrize('args, ret', [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)]) +@pytest.mark.parametrize( + "args, ret", [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] +) def test_index_offset(arr, args, ret): assert m.index_at(arr, *args) == ret assert m.index_at_t(arr, *args) == ret @@ -76,31 +81,46 @@ def test_index_offset(arr, args, ret): def test_dim_check_fail(arr): - for func in (m.index_at, m.index_at_t, m.offset_at, m.offset_at_t, m.data, m.data_t, - m.mutate_data, m.mutate_data_t): + for func in ( + m.index_at, + m.index_at_t, + m.offset_at, + m.offset_at_t, + m.data, + m.data_t, + m.mutate_data, + m.mutate_data_t, + ): with pytest.raises(IndexError) as excinfo: func(arr, 1, 2, 3) - assert str(excinfo.value) == 'too many indices for an array: 3 (ndim = 2)' - - -@pytest.mark.parametrize('args, ret', - [([], [1, 2, 3, 4, 5, 6]), - ([1], [4, 5, 6]), - ([0, 1], [2, 3, 4, 5, 6]), - ([1, 2], [6])]) + assert str(excinfo.value) == "too many indices for an array: 3 (ndim = 2)" + + +@pytest.mark.parametrize( + "args, ret", + [ + ([], [1, 2, 3, 4, 5, 6]), + ([1], [4, 5, 6]), + ([0, 1], [2, 3, 4, 5, 6]), + ([1, 2], [6]), + ], +) def test_data(arr, args, ret): from sys import byteorder + assert all(m.data_t(arr, *args) == ret) - assert all(m.data(arr, *args)[(0 if byteorder == 'little' else 1)::2] == ret) - assert all(m.data(arr, *args)[(1 if byteorder == 'little' else 0)::2] == 0) + assert all(m.data(arr, *args)[(0 if byteorder == "little" else 1) :: 2] == ret) + assert all(m.data(arr, *args)[(1 if byteorder == "little" else 0) :: 2] == 0) -@pytest.mark.parametrize('dim', [0, 1, 3]) +@pytest.mark.parametrize("dim", [0, 1, 3]) def test_at_fail(arr, dim): for func in m.at_t, m.mutate_at_t: with pytest.raises(IndexError) as excinfo: func(arr, *([0] * dim)) - assert str(excinfo.value) == 'index dimension mismatch: {} (ndim = 2)'.format(dim) + assert str(excinfo.value) == "index dimension mismatch: {} (ndim = 2)".format( + dim + ) def test_at(arr): @@ -113,10 +133,14 @@ def test_at(arr): def test_mutate_readonly(arr): arr.flags.writeable = False - for func, args in (m.mutate_data, ()), (m.mutate_data_t, ()), (m.mutate_at_t, (0, 0)): + for func, args in ( + (m.mutate_data, ()), + (m.mutate_data_t, ()), + (m.mutate_at_t, (0, 0)), + ): with pytest.raises(ValueError) as excinfo: func(arr, *args) - assert str(excinfo.value) == 'array is not writeable' + assert str(excinfo.value) == "array is not writeable" def test_mutate_data(arr): @@ -134,14 +158,22 @@ def test_mutate_data(arr): def test_bounds_check(arr): - for func in (m.index_at, m.index_at_t, m.data, m.data_t, - m.mutate_data, m.mutate_data_t, m.at_t, m.mutate_at_t): + for func in ( + m.index_at, + m.index_at_t, + m.data, + m.data_t, + m.mutate_data, + m.mutate_data_t, + m.at_t, + m.mutate_at_t, + ): with pytest.raises(IndexError) as excinfo: func(arr, 2, 0) - assert str(excinfo.value) == 'index 2 is out of bounds for axis 0 with size 2' + assert str(excinfo.value) == "index 2 is out of bounds for axis 0 with size 2" with pytest.raises(IndexError) as excinfo: func(arr, 0, 4) - assert str(excinfo.value) == 'index 4 is out of bounds for axis 1 with size 3' + assert str(excinfo.value) == "index 4 is out of bounds for axis 1 with size 3" def test_make_c_f_array(): @@ -163,10 +195,11 @@ def test_make_empty_shaped_array(): def test_wrap(): def assert_references(a, b, base=None): from distutils.version import LooseVersion + if base is None: base = a assert a is not b - assert a.__array_interface__['data'][0] == b.__array_interface__['data'][0] + assert a.__array_interface__["data"][0] == b.__array_interface__["data"][0] assert a.shape == b.shape assert a.strides == b.strides assert a.flags.c_contiguous == b.flags.c_contiguous @@ -189,12 +222,12 @@ def assert_references(a, b, base=None): a2 = m.wrap(a1) assert_references(a1, a2) - a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order='F') + a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="F") assert a1.flags.owndata and a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) - a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order='C') + a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="C") a1.flags.writeable = False a2 = m.wrap(a1) assert_references(a1, a2) @@ -224,11 +257,14 @@ def test_numpy_view(capture): assert np.all(ac_view_1 == np.array([1, 2], dtype=np.int32)) del ac pytest.gc_collect() - assert capture == """ + assert ( + capture + == """ ArrayClass() ArrayClass::numpy_view() ArrayClass::numpy_view() """ + ) ac_view_1[0] = 4 ac_view_1[1] = 3 assert ac_view_2[0] == 4 @@ -238,12 +274,14 @@ def test_numpy_view(capture): del ac_view_2 pytest.gc_collect() pytest.gc_collect() - assert capture == """ + assert ( + capture + == """ ~ArrayClass() """ + ) -@pytest.unsupported_on_pypy def test_cast_numpy_int64_to_uint64(): m.function_taking_uint64(123) m.function_taking_uint64(np.uint64(123)) @@ -272,20 +310,22 @@ def test_constructors(): def test_overload_resolution(msg): # Exact overload matches: - assert m.overloaded(np.array([1], dtype='float64')) == 'double' - assert m.overloaded(np.array([1], dtype='float32')) == 'float' - assert m.overloaded(np.array([1], dtype='ushort')) == 'unsigned short' - assert m.overloaded(np.array([1], dtype='intc')) == 'int' - assert m.overloaded(np.array([1], dtype='longlong')) == 'long long' - assert m.overloaded(np.array([1], dtype='complex')) == 'double complex' - assert m.overloaded(np.array([1], dtype='csingle')) == 'float complex' + assert m.overloaded(np.array([1], dtype="float64")) == "double" + assert m.overloaded(np.array([1], dtype="float32")) == "float" + assert m.overloaded(np.array([1], dtype="ushort")) == "unsigned short" + assert m.overloaded(np.array([1], dtype="intc")) == "int" + assert m.overloaded(np.array([1], dtype="longlong")) == "long long" + assert m.overloaded(np.array([1], dtype="complex")) == "double complex" + assert m.overloaded(np.array([1], dtype="csingle")) == "float complex" # No exact match, should call first convertible version: - assert m.overloaded(np.array([1], dtype='uint8')) == 'double' + assert m.overloaded(np.array([1], dtype="uint8")) == "double" with pytest.raises(TypeError) as excinfo: m.overloaded("not an array") - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ overloaded(): incompatible function arguments. The following argument types are supported: 1. (arg0: numpy.ndarray[numpy.float64]) -> str 2. (arg0: numpy.ndarray[numpy.float32]) -> str @@ -297,15 +337,16 @@ def test_overload_resolution(msg): Invoked with: 'not an array' """ + ) - assert m.overloaded2(np.array([1], dtype='float64')) == 'double' - assert m.overloaded2(np.array([1], dtype='float32')) == 'float' - assert m.overloaded2(np.array([1], dtype='complex64')) == 'float complex' - assert m.overloaded2(np.array([1], dtype='complex128')) == 'double complex' - assert m.overloaded2(np.array([1], dtype='float32')) == 'float' + assert m.overloaded2(np.array([1], dtype="float64")) == "double" + assert m.overloaded2(np.array([1], dtype="float32")) == "float" + assert m.overloaded2(np.array([1], dtype="complex64")) == "float complex" + assert m.overloaded2(np.array([1], dtype="complex128")) == "double complex" + assert m.overloaded2(np.array([1], dtype="float32")) == "float" - assert m.overloaded3(np.array([1], dtype='float64')) == 'double' - assert m.overloaded3(np.array([1], dtype='intc')) == 'int' + assert m.overloaded3(np.array([1], dtype="float64")) == "double" + assert m.overloaded3(np.array([1], dtype="intc")) == "int" expected_exc = """ overloaded3(): incompatible function arguments. The following argument types are supported: 1. (arg0: numpy.ndarray[numpy.int32]) -> str @@ -314,47 +355,49 @@ def test_overload_resolution(msg): Invoked with: """ with pytest.raises(TypeError) as excinfo: - m.overloaded3(np.array([1], dtype='uintc')) - assert msg(excinfo.value) == expected_exc + repr(np.array([1], dtype='uint32')) + m.overloaded3(np.array([1], dtype="uintc")) + assert msg(excinfo.value) == expected_exc + repr(np.array([1], dtype="uint32")) with pytest.raises(TypeError) as excinfo: - m.overloaded3(np.array([1], dtype='float32')) - assert msg(excinfo.value) == expected_exc + repr(np.array([1.], dtype='float32')) + m.overloaded3(np.array([1], dtype="float32")) + assert msg(excinfo.value) == expected_exc + repr(np.array([1.0], dtype="float32")) with pytest.raises(TypeError) as excinfo: - m.overloaded3(np.array([1], dtype='complex')) - assert msg(excinfo.value) == expected_exc + repr(np.array([1. + 0.j])) + m.overloaded3(np.array([1], dtype="complex")) + assert msg(excinfo.value) == expected_exc + repr(np.array([1.0 + 0.0j])) # Exact matches: - assert m.overloaded4(np.array([1], dtype='double')) == 'double' - assert m.overloaded4(np.array([1], dtype='longlong')) == 'long long' + assert m.overloaded4(np.array([1], dtype="double")) == "double" + assert m.overloaded4(np.array([1], dtype="longlong")) == "long long" # Non-exact matches requiring conversion. Since float to integer isn't a # save conversion, it should go to the double overload, but short can go to # either (and so should end up on the first-registered, the long long). - assert m.overloaded4(np.array([1], dtype='float32')) == 'double' - assert m.overloaded4(np.array([1], dtype='short')) == 'long long' + assert m.overloaded4(np.array([1], dtype="float32")) == "double" + assert m.overloaded4(np.array([1], dtype="short")) == "long long" - assert m.overloaded5(np.array([1], dtype='double')) == 'double' - assert m.overloaded5(np.array([1], dtype='uintc')) == 'unsigned int' - assert m.overloaded5(np.array([1], dtype='float32')) == 'unsigned int' + assert m.overloaded5(np.array([1], dtype="double")) == "double" + assert m.overloaded5(np.array([1], dtype="uintc")) == "unsigned int" + assert m.overloaded5(np.array([1], dtype="float32")) == "unsigned int" def test_greedy_string_overload(): """Tests fix for #685 - ndarray shouldn't go to std::string overload""" assert m.issue685("abc") == "string" - assert m.issue685(np.array([97, 98, 99], dtype='b')) == "array" + assert m.issue685(np.array([97, 98, 99], dtype="b")) == "array" assert m.issue685(123) == "other" def test_array_unchecked_fixed_dims(msg): - z1 = np.array([[1, 2], [3, 4]], dtype='float64') + z1 = np.array([[1, 2], [3, 4]], dtype="float64") m.proxy_add2(z1, 10) assert np.all(z1 == [[11, 12], [13, 14]]) with pytest.raises(ValueError) as excinfo: - m.proxy_add2(np.array([1., 2, 3]), 5.0) - assert msg(excinfo.value) == "array has incorrect number of dimensions: 1; expected 2" + m.proxy_add2(np.array([1.0, 2, 3]), 5.0) + assert ( + msg(excinfo.value) == "array has incorrect number of dimensions: 1; expected 2" + ) - expect_c = np.ndarray(shape=(3, 3, 3), buffer=np.array(range(3, 30)), dtype='int') + expect_c = np.ndarray(shape=(3, 3, 3), buffer=np.array(range(3, 30)), dtype="int") assert np.all(m.proxy_init3(3.0) == expect_c) expect_f = np.transpose(expect_c) assert np.all(m.proxy_init3F(3.0) == expect_f) @@ -365,13 +408,16 @@ def test_array_unchecked_fixed_dims(msg): assert m.proxy_auxiliaries2(z1) == [11, 11, True, 2, 8, 2, 2, 4, 32] assert m.proxy_auxiliaries2(z1) == m.array_auxiliaries2(z1) + assert m.proxy_auxiliaries1_const_ref(z1[0, :]) + assert m.proxy_auxiliaries2_const_ref(z1) + def test_array_unchecked_dyn_dims(msg): - z1 = np.array([[1, 2], [3, 4]], dtype='float64') + z1 = np.array([[1, 2], [3, 4]], dtype="float64") m.proxy_add2_dyn(z1, 10) assert np.all(z1 == [[11, 12], [13, 14]]) - expect_c = np.ndarray(shape=(3, 3, 3), buffer=np.array(range(3, 30)), dtype='int') + expect_c = np.ndarray(shape=(3, 3, 3), buffer=np.array(range(3, 30)), dtype="int") assert np.all(m.proxy_init3_dyn(3.0) == expect_c) assert m.proxy_auxiliaries2_dyn(z1) == [11, 11, True, 2, 8, 2, 2, 4, 32] @@ -381,15 +427,15 @@ def test_array_unchecked_dyn_dims(msg): def test_array_failure(): with pytest.raises(ValueError) as excinfo: m.array_fail_test() - assert str(excinfo.value) == 'cannot create a pybind11::array from a nullptr' + assert str(excinfo.value) == "cannot create a pybind11::array from a nullptr" with pytest.raises(ValueError) as excinfo: m.array_t_fail_test() - assert str(excinfo.value) == 'cannot create a pybind11::array_t from a nullptr' + assert str(excinfo.value) == "cannot create a pybind11::array_t from a nullptr" with pytest.raises(ValueError) as excinfo: m.array_fail_test_negative_size() - assert str(excinfo.value) == 'negative dimensions are not allowed' + assert str(excinfo.value) == "negative dimensions are not allowed" def test_initializer_list(): @@ -400,46 +446,93 @@ def test_initializer_list(): def test_array_resize(msg): - a = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype='float64') + a = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="float64") m.array_reshape2(a) - assert(a.size == 9) - assert(np.all(a == [[1, 2, 3], [4, 5, 6], [7, 8, 9]])) + assert a.size == 9 + assert np.all(a == [[1, 2, 3], [4, 5, 6], [7, 8, 9]]) # total size change should succced with refcheck off m.array_resize3(a, 4, False) - assert(a.size == 64) + assert a.size == 64 # ... and fail with refcheck on try: m.array_resize3(a, 3, True) except ValueError as e: - assert(str(e).startswith("cannot resize an array")) + assert str(e).startswith("cannot resize an array") # transposed array doesn't own data b = a.transpose() try: m.array_resize3(b, 3, False) except ValueError as e: - assert(str(e).startswith("cannot resize this array: it does not own its data")) + assert str(e).startswith("cannot resize this array: it does not own its data") # ... but reshape should be fine m.array_reshape2(b) - assert(b.shape == (8, 8)) + assert b.shape == (8, 8) -@pytest.unsupported_on_pypy +@pytest.mark.xfail("env.PYPY") def test_array_create_and_resize(msg): a = m.create_and_resize(2) - assert(a.size == 4) - assert(np.all(a == 42.)) + assert a.size == 4 + assert np.all(a == 42.0) -@pytest.unsupported_on_py2 def test_index_using_ellipsis(): a = m.index_using_ellipsis(np.zeros((5, 6, 7))) assert a.shape == (6,) -@pytest.unsupported_on_pypy +@pytest.mark.parametrize("forcecast", [False, True]) +@pytest.mark.parametrize("contiguity", [None, "C", "F"]) +@pytest.mark.parametrize("noconvert", [False, True]) +@pytest.mark.filterwarnings( + "ignore:Casting complex values to real discards the imaginary part:numpy.ComplexWarning" +) +def test_argument_conversions(forcecast, contiguity, noconvert): + function_name = "accept_double" + if contiguity == "C": + function_name += "_c_style" + elif contiguity == "F": + function_name += "_f_style" + if forcecast: + function_name += "_forcecast" + if noconvert: + function_name += "_noconvert" + function = getattr(m, function_name) + + for dtype in [np.dtype("float32"), np.dtype("float64"), np.dtype("complex128")]: + for order in ["C", "F"]: + for shape in [(2, 2), (1, 3, 1, 1), (1, 1, 1), (0,)]: + if not noconvert: + # If noconvert is not passed, only complex128 needs to be truncated and + # "cannot be safely obtained". So without `forcecast`, the argument shouldn't + # be accepted. + should_raise = dtype.name == "complex128" and not forcecast + else: + # If noconvert is passed, only float64 and the matching order is accepted. + # If at most one dimension has a size greater than 1, the array is also + # trivially contiguous. + trivially_contiguous = sum(1 for d in shape if d > 1) <= 1 + should_raise = dtype.name != "float64" or ( + contiguity is not None + and contiguity != order + and not trivially_contiguous + ) + + array = np.zeros(shape, dtype=dtype, order=order) + if not should_raise: + function(array) + else: + with pytest.raises( + TypeError, match="incompatible function arguments" + ): + function(array) + + +@pytest.mark.xfail("env.PYPY") def test_dtype_refcount_leak(): from sys import getrefcount + dtype = np.dtype(np.float_) a = np.array([1], dtype=dtype) before = getrefcount(dtype) diff --git a/tests/test_numpy_dtypes.cpp b/tests/test_numpy_dtypes.cpp index 467e0253f7..9dece73ec3 100644 --- a/tests/test_numpy_dtypes.cpp +++ b/tests/test_numpy_dtypes.cpp @@ -168,7 +168,7 @@ py::list print_recarray(py::array_t arr) { const auto req = arr.request(); const auto ptr = static_cast(req.ptr); auto l = py::list(); - for (ssize_t i = 0; i < req.size; i++) { + for (py::ssize_t i = 0; i < req.size; i++) { std::stringstream ss; ss << ptr[i]; l.append(py::str(ss.str())); @@ -180,8 +180,8 @@ py::array_t test_array_ctors(int i) { using arr_t = py::array_t; std::vector data { 1, 2, 3, 4, 5, 6 }; - std::vector shape { 3, 2 }; - std::vector strides { 8, 4 }; + std::vector shape { 3, 2 }; + std::vector strides { 8, 4 }; auto ptr = data.data(); auto vptr = (void *) ptr; @@ -255,11 +255,30 @@ struct A {}; struct B {}; TEST_SUBMODULE(numpy_dtypes, m) { - try { py::module::import("numpy"); } + try { py::module_::import("numpy"); } catch (...) { return; } // typeinfo may be registered before the dtype descriptor for scalar casts to work... - py::class_(m, "SimpleStruct"); + py::class_(m, "SimpleStruct") + // Explicit construct to ensure zero-valued initialization. + .def(py::init([]() { return SimpleStruct(); })) + .def_readwrite("bool_", &SimpleStruct::bool_) + .def_readwrite("uint_", &SimpleStruct::uint_) + .def_readwrite("float_", &SimpleStruct::float_) + .def_readwrite("ldbl_", &SimpleStruct::ldbl_) + .def("astuple", [](const SimpleStruct& self) { + return py::make_tuple(self.bool_, self.uint_, self.float_, self.ldbl_); + }) + .def_static("fromtuple", [](const py::tuple tup) { + if (py::len(tup) != 4) { + throw py::cast_error("Invalid size"); + } + return SimpleStruct{ + tup[0].cast(), + tup[1].cast(), + tup[2].cast(), + tup[3].cast()}; + }); PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(SimpleStructReordered, bool_, uint_, float_, ldbl_); @@ -339,6 +358,14 @@ TEST_SUBMODULE(numpy_dtypes, m) { }); // test_dtype + std::vector dtype_names{ + "byte", "short", "intc", "int_", "longlong", + "ubyte", "ushort", "uintc", "uint", "ulonglong", + "half", "single", "double", "longdouble", + "csingle", "cdouble", "clongdouble", + "bool_", "datetime64", "timedelta64", "object_" + }; + m.def("print_dtypes", []() { py::list l; for (const py::handle &d : { @@ -357,6 +384,18 @@ TEST_SUBMODULE(numpy_dtypes, m) { return l; }); m.def("test_dtype_ctors", &test_dtype_ctors); + m.def("test_dtype_kind", [dtype_names]() { + py::list list; + for (auto& dt_name : dtype_names) + list.append(py::dtype(dt_name).kind()); + return list; + }); + m.def("test_dtype_char_", [dtype_names]() { + py::list list; + for (auto& dt_name : dtype_names) + list.append(py::dtype(dt_name).char_()); + return list; + }); m.def("test_dtype_methods", []() { py::list list; auto dt1 = py::dtype::of(); @@ -379,7 +418,7 @@ TEST_SUBMODULE(numpy_dtypes, m) { if (non_empty) { auto req = arr.request(); auto ptr = static_cast(req.ptr); - for (ssize_t i = 0; i < req.size * req.itemsize; i++) + for (py::ssize_t i = 0; i < req.size * req.itemsize; i++) static_cast(req.ptr)[i] = 0; ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a'; ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a'; @@ -462,10 +501,16 @@ TEST_SUBMODULE(numpy_dtypes, m) { m.def("buffer_to_dtype", [](py::buffer& buf) { return py::dtype(buf.request()); }); // test_scalar_conversion - m.def("f_simple", [](SimpleStruct s) { return s.uint_ * 10; }); + auto f_simple = [](SimpleStruct s) { return s.uint_ * 10; }; + m.def("f_simple", f_simple); m.def("f_packed", [](PackedStruct s) { return s.uint_ * 10; }); m.def("f_nested", [](NestedStruct s) { return s.a.uint_ * 10; }); + // test_vectorize + m.def("f_simple_vectorized", py::vectorize(f_simple)); + auto f_simple_pass_thru = [](SimpleStruct s) { return s; }; + m.def("f_simple_pass_thru_vectorized", py::vectorize(f_simple_pass_thru)); + // test_register_dtype m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); }); diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index d173435fe6..0a5881e49d 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -1,65 +1,80 @@ # -*- coding: utf-8 -*- import re + import pytest -from pybind11_tests import numpy_dtypes as m -pytestmark = pytest.requires_numpy +import env # noqa: F401 + +from pybind11_tests import numpy_dtypes as m -with pytest.suppress(ImportError): - import numpy as np +np = pytest.importorskip("numpy") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def simple_dtype(): - ld = np.dtype('longdouble') - return np.dtype({'names': ['bool_', 'uint_', 'float_', 'ldbl_'], - 'formats': ['?', 'u4', 'f4', 'f{}'.format(ld.itemsize)], - 'offsets': [0, 4, 8, (16 if ld.alignment > 4 else 12)]}) + ld = np.dtype("longdouble") + return np.dtype( + { + "names": ["bool_", "uint_", "float_", "ldbl_"], + "formats": ["?", "u4", "f4", "f{}".format(ld.itemsize)], + "offsets": [0, 4, 8, (16 if ld.alignment > 4 else 12)], + } + ) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def packed_dtype(): - return np.dtype([('bool_', '?'), ('uint_', 'u4'), ('float_', 'f4'), ('ldbl_', 'g')]) + return np.dtype([("bool_", "?"), ("uint_", "u4"), ("float_", "f4"), ("ldbl_", "g")]) def dt_fmt(): from sys import byteorder - e = '<' if byteorder == 'little' else '>' - return ("{{'names':['bool_','uint_','float_','ldbl_']," - " 'formats':['?','" + e + "u4','" + e + "f4','" + e + "f{}']," - " 'offsets':[0,4,8,{}], 'itemsize':{}}}") + + e = "<" if byteorder == "little" else ">" + return ( + "{{'names':['bool_','uint_','float_','ldbl_']," + " 'formats':['?','" + e + "u4','" + e + "f4','" + e + "f{}']," + " 'offsets':[0,4,8,{}], 'itemsize':{}}}" + ) def simple_dtype_fmt(): - ld = np.dtype('longdouble') + ld = np.dtype("longdouble") simple_ld_off = 12 + 4 * (ld.alignment > 4) return dt_fmt().format(ld.itemsize, simple_ld_off, simple_ld_off + ld.itemsize) def packed_dtype_fmt(): from sys import byteorder + return "[('bool_', '?'), ('uint_', '{e}u4'), ('float_', '{e}f4'), ('ldbl_', '{e}f{}')]".format( - np.dtype('longdouble').itemsize, e='<' if byteorder == 'little' else '>') + np.dtype("longdouble").itemsize, e="<" if byteorder == "little" else ">" + ) def partial_ld_offset(): - return 12 + 4 * (np.dtype('uint64').alignment > 4) + 8 + 8 * ( - np.dtype('longdouble').alignment > 8) + return ( + 12 + + 4 * (np.dtype("uint64").alignment > 4) + + 8 + + 8 * (np.dtype("longdouble").alignment > 8) + ) def partial_dtype_fmt(): - ld = np.dtype('longdouble') + ld = np.dtype("longdouble") partial_ld_off = partial_ld_offset() return dt_fmt().format(ld.itemsize, partial_ld_off, partial_ld_off + ld.itemsize) def partial_nested_fmt(): - ld = np.dtype('longdouble') + ld = np.dtype("longdouble") partial_nested_off = 8 + 8 * (ld.alignment > 8) partial_ld_off = partial_ld_offset() partial_nested_size = partial_nested_off * 2 + partial_ld_off + ld.itemsize return "{{'names':['a'], 'formats':[{}], 'offsets':[{}], 'itemsize':{}}}".format( - partial_dtype_fmt(), partial_nested_off, partial_nested_size) + partial_dtype_fmt(), partial_nested_off, partial_nested_size + ) def assert_equal(actual, expected_data, expected_dtype): @@ -69,15 +84,19 @@ def assert_equal(actual, expected_data, expected_dtype): def test_format_descriptors(): with pytest.raises(RuntimeError) as excinfo: m.get_format_unbound() - assert re.match('^NumPy type info missing for .*UnboundStruct.*$', str(excinfo.value)) + assert re.match( + "^NumPy type info missing for .*UnboundStruct.*$", str(excinfo.value) + ) - ld = np.dtype('longdouble') - ldbl_fmt = ('4x' if ld.alignment > 4 else '') + ld.char + ld = np.dtype("longdouble") + ldbl_fmt = ("4x" if ld.alignment > 4 else "") + ld.char ss_fmt = "^T{?:bool_:3xI:uint_:f:float_:" + ldbl_fmt + ":ldbl_:}" - dbl = np.dtype('double') - partial_fmt = ("^T{?:bool_:3xI:uint_:f:float_:" + - str(4 * (dbl.alignment > 4) + dbl.itemsize + 8 * (ld.alignment > 8)) + - "xg:ldbl_:}") + dbl = np.dtype("double") + partial_fmt = ( + "^T{?:bool_:3xI:uint_:f:float_:" + + str(4 * (dbl.alignment > 4) + dbl.itemsize + 8 * (ld.alignment > 8)) + + "xg:ldbl_:}" + ) nested_extra = str(max(8, ld.alignment)) assert m.print_format_descriptors() == [ ss_fmt, @@ -87,14 +106,15 @@ def test_format_descriptors(): "^T{" + nested_extra + "x" + partial_fmt + ":a:" + nested_extra + "x}", "^T{3s:a:3s:b:}", "^T{(3)4s:a:(2)i:b:(3)B:c:1x(4, 2)f:d:}", - '^T{q:e1:B:e2:}', - '^T{Zf:cflt:Zd:cdbl:}' + "^T{q:e1:B:e2:}", + "^T{Zf:cflt:Zd:cdbl:}", ] def test_dtype(simple_dtype): from sys import byteorder - e = '<' if byteorder == 'little' else '>' + + e = "<" if byteorder == "little" else ">" assert m.print_dtypes() == [ simple_dtype_fmt(), @@ -103,30 +123,63 @@ def test_dtype(simple_dtype): partial_dtype_fmt(), partial_nested_fmt(), "[('a', 'S3'), ('b', 'S3')]", - ("{{'names':['a','b','c','d'], " + - "'formats':[('S4', (3,)),('" + e + "i4', (2,)),('u1', (3,)),('" + e + "f4', (4, 2))], " + - "'offsets':[0,12,20,24], 'itemsize':56}}").format(e=e), + ( + "{{'names':['a','b','c','d'], " + + "'formats':[('S4', (3,)),('" + + e + + "i4', (2,)),('u1', (3,)),('" + + e + + "f4', (4, 2))], " + + "'offsets':[0,12,20,24], 'itemsize':56}}" + ).format(e=e), "[('e1', '" + e + "i8'), ('e2', 'u1')]", "[('x', 'i1'), ('y', '" + e + "u8')]", - "[('cflt', '" + e + "c8'), ('cdbl', '" + e + "c16')]" + "[('cflt', '" + e + "c8'), ('cdbl', '" + e + "c16')]", ] - d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'], - 'offsets': [1, 10], 'itemsize': 20}) - d2 = np.dtype([('a', 'i4'), ('b', 'f4')]) - assert m.test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'), - np.dtype('bool'), d1, d1, np.dtype('uint32'), d2] + d1 = np.dtype( + { + "names": ["a", "b"], + "formats": ["int32", "float64"], + "offsets": [1, 10], + "itemsize": 20, + } + ) + d2 = np.dtype([("a", "i4"), ("b", "f4")]) + assert m.test_dtype_ctors() == [ + np.dtype("int32"), + np.dtype("float64"), + np.dtype("bool"), + d1, + d1, + np.dtype("uint32"), + d2, + ] - assert m.test_dtype_methods() == [np.dtype('int32'), simple_dtype, False, True, - np.dtype('int32').itemsize, simple_dtype.itemsize] + assert m.test_dtype_methods() == [ + np.dtype("int32"), + simple_dtype, + False, + True, + np.dtype("int32").itemsize, + simple_dtype.itemsize, + ] - assert m.trailing_padding_dtype() == m.buffer_to_dtype(np.zeros(1, m.trailing_padding_dtype())) + assert m.trailing_padding_dtype() == m.buffer_to_dtype( + np.zeros(1, m.trailing_padding_dtype()) + ) + + assert m.test_dtype_kind() == list("iiiiiuuuuuffffcccbMmO") + assert m.test_dtype_char_() == list("bhilqBHILQefdgFDG?MmO") def test_recarray(simple_dtype, packed_dtype): elements = [(False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)] - for func, dtype in [(m.create_rec_simple, simple_dtype), (m.create_rec_packed, packed_dtype)]: + for func, dtype in [ + (m.create_rec_simple, simple_dtype), + (m.create_rec_packed, packed_dtype), + ]: arr = func(0) assert arr.dtype == dtype assert_equal(arr, [], simple_dtype) @@ -137,20 +190,24 @@ def test_recarray(simple_dtype, packed_dtype): assert_equal(arr, elements, simple_dtype) assert_equal(arr, elements, packed_dtype) + # Show what recarray's look like in NumPy. + assert type(arr[0]) == np.void + assert type(arr[0].item()) == tuple + if dtype == simple_dtype: assert m.print_rec_simple(arr) == [ "s:0,0,0,-0", "s:1,1,1.5,-2.5", - "s:0,2,3,-5" + "s:0,2,3,-5", ] else: assert m.print_rec_packed(arr) == [ "p:0,0,0,-0", "p:1,1,1.5,-2.5", - "p:0,2,3,-5" + "p:0,2,3,-5", ] - nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) + nested_dtype = np.dtype([("a", simple_dtype), ("b", packed_dtype)]) arr = m.create_rec_nested(0) assert arr.dtype == nested_dtype @@ -158,33 +215,39 @@ def test_recarray(simple_dtype, packed_dtype): arr = m.create_rec_nested(3) assert arr.dtype == nested_dtype - assert_equal(arr, [((False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5)), - ((True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)), - ((False, 2, 3.0, -5.0), (True, 3, 4.5, -7.5))], nested_dtype) + assert_equal( + arr, + [ + ((False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5)), + ((True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)), + ((False, 2, 3.0, -5.0), (True, 3, 4.5, -7.5)), + ], + nested_dtype, + ) assert m.print_rec_nested(arr) == [ "n:a=s:0,0,0,-0;b=p:1,1,1.5,-2.5", "n:a=s:1,1,1.5,-2.5;b=p:0,2,3,-5", - "n:a=s:0,2,3,-5;b=p:1,3,4.5,-7.5" + "n:a=s:0,2,3,-5;b=p:1,3,4.5,-7.5", ] arr = m.create_rec_partial(3) assert str(arr.dtype) == partial_dtype_fmt() partial_dtype = arr.dtype - assert '' not in arr.dtype.fields + assert "" not in arr.dtype.fields assert partial_dtype.itemsize > simple_dtype.itemsize assert_equal(arr, elements, simple_dtype) assert_equal(arr, elements, packed_dtype) arr = m.create_rec_partial_nested(3) assert str(arr.dtype) == partial_nested_fmt() - assert '' not in arr.dtype.fields - assert '' not in arr.dtype.fields['a'][0].fields + assert "" not in arr.dtype.fields + assert "" not in arr.dtype.fields["a"][0].fields assert arr.dtype.itemsize > partial_dtype.itemsize - np.testing.assert_equal(arr['a'], m.create_rec_partial(3)) + np.testing.assert_equal(arr["a"], m.create_rec_partial(3)) def test_array_constructors(): - data = np.arange(1, 7, dtype='int32') + data = np.arange(1, 7, dtype="int32") for i in range(8): np.testing.assert_array_equal(m.test_array_ctors(10 + i), data.reshape((3, 2))) np.testing.assert_array_equal(m.test_array_ctors(20 + i), data.reshape((3, 2))) @@ -200,82 +263,92 @@ def test_string_array(): "a='',b=''", "a='a',b='a'", "a='ab',b='ab'", - "a='abc',b='abc'" + "a='abc',b='abc'", ] dtype = arr.dtype - assert arr['a'].tolist() == [b'', b'a', b'ab', b'abc'] - assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc'] + assert arr["a"].tolist() == [b"", b"a", b"ab", b"abc"] + assert arr["b"].tolist() == [b"", b"a", b"ab", b"abc"] arr = m.create_string_array(False) assert dtype == arr.dtype def test_array_array(): from sys import byteorder - e = '<' if byteorder == 'little' else '>' + + e = "<" if byteorder == "little" else ">" arr = m.create_array_array(3) assert str(arr.dtype) == ( - "{{'names':['a','b','c','d'], " + - "'formats':[('S4', (3,)),('" + e + "i4', (2,)),('u1', (3,)),('{e}f4', (4, 2))], " + - "'offsets':[0,12,20,24], 'itemsize':56}}").format(e=e) + "{{'names':['a','b','c','d'], " + + "'formats':[('S4', (3,)),('" + + e + + "i4', (2,)),('u1', (3,)),('{e}f4', (4, 2))], " + + "'offsets':[0,12,20,24], 'itemsize':56}}" + ).format(e=e) assert m.print_array_array(arr) == [ - "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1}," + - "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", - "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001}," + - "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", - "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001}," + - "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", + "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1}," + + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", + "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001}," + + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", + "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001}," + + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", + ] + assert arr["a"].tolist() == [ + [b"ABCD", b"KLMN", b"UVWX"], + [b"WXYZ", b"GHIJ", b"QRST"], + [b"STUV", b"CDEF", b"MNOP"], ] - assert arr['a'].tolist() == [[b'ABCD', b'KLMN', b'UVWX'], - [b'WXYZ', b'GHIJ', b'QRST'], - [b'STUV', b'CDEF', b'MNOP']] - assert arr['b'].tolist() == [[0, 1], [1000, 1001], [2000, 2001]] + assert arr["b"].tolist() == [[0, 1], [1000, 1001], [2000, 2001]] assert m.create_array_array(0).dtype == arr.dtype def test_enum_array(): from sys import byteorder - e = '<' if byteorder == 'little' else '>' + + e = "<" if byteorder == "little" else ">" arr = m.create_enum_array(3) dtype = arr.dtype - assert dtype == np.dtype([('e1', e + 'i8'), ('e2', 'u1')]) - assert m.print_enum_array(arr) == [ - "e1=A,e2=X", - "e1=B,e2=Y", - "e1=A,e2=X" - ] - assert arr['e1'].tolist() == [-1, 1, -1] - assert arr['e2'].tolist() == [1, 2, 1] + assert dtype == np.dtype([("e1", e + "i8"), ("e2", "u1")]) + assert m.print_enum_array(arr) == ["e1=A,e2=X", "e1=B,e2=Y", "e1=A,e2=X"] + assert arr["e1"].tolist() == [-1, 1, -1] + assert arr["e2"].tolist() == [1, 2, 1] assert m.create_enum_array(0).dtype == dtype def test_complex_array(): from sys import byteorder - e = '<' if byteorder == 'little' else '>' + + e = "<" if byteorder == "little" else ">" arr = m.create_complex_array(3) dtype = arr.dtype - assert dtype == np.dtype([('cflt', e + 'c8'), ('cdbl', e + 'c16')]) + assert dtype == np.dtype([("cflt", e + "c8"), ("cdbl", e + "c16")]) assert m.print_complex_array(arr) == [ "c:(0,0.25),(0.5,0.75)", "c:(1,1.25),(1.5,1.75)", - "c:(2,2.25),(2.5,2.75)" + "c:(2,2.25),(2.5,2.75)", ] - assert arr['cflt'].tolist() == [0.0 + 0.25j, 1.0 + 1.25j, 2.0 + 2.25j] - assert arr['cdbl'].tolist() == [0.5 + 0.75j, 1.5 + 1.75j, 2.5 + 2.75j] + assert arr["cflt"].tolist() == [0.0 + 0.25j, 1.0 + 1.25j, 2.0 + 2.25j] + assert arr["cdbl"].tolist() == [0.5 + 0.75j, 1.5 + 1.75j, 2.5 + 2.75j] assert m.create_complex_array(0).dtype == dtype def test_signature(doc): - assert doc(m.create_rec_nested) == \ - "create_rec_nested(arg0: int) -> numpy.ndarray[NestedStruct]" + assert ( + doc(m.create_rec_nested) + == "create_rec_nested(arg0: int) -> numpy.ndarray[NestedStruct]" + ) def test_scalar_conversion(): n = 3 - arrays = [m.create_rec_simple(n), m.create_rec_packed(n), - m.create_rec_nested(n), m.create_enum_array(n)] + arrays = [ + m.create_rec_simple(n), + m.create_rec_packed(n), + m.create_rec_nested(n), + m.create_enum_array(n), + ] funcs = [m.f_simple, m.f_packed, m.f_nested] for i, func in enumerate(funcs): @@ -285,18 +358,68 @@ def test_scalar_conversion(): else: with pytest.raises(TypeError) as excinfo: func(arr[0]) - assert 'incompatible function arguments' in str(excinfo.value) + assert "incompatible function arguments" in str(excinfo.value) + + +def test_vectorize(): + n = 3 + array = m.create_rec_simple(n) + values = m.f_simple_vectorized(array) + np.testing.assert_array_equal(values, [0, 10, 20]) + array_2 = m.f_simple_pass_thru_vectorized(array) + np.testing.assert_array_equal(array, array_2) + + +def test_cls_and_dtype_conversion(simple_dtype): + s = m.SimpleStruct() + assert s.astuple() == (False, 0, 0.0, 0.0) + assert m.SimpleStruct.fromtuple(s.astuple()).astuple() == s.astuple() + + s.uint_ = 2 + assert m.f_simple(s) == 20 + + # Try as recarray of shape==(1,). + s_recarray = np.array([(False, 2, 0.0, 0.0)], dtype=simple_dtype) + # Show that this will work for vectorized case. + np.testing.assert_array_equal(m.f_simple_vectorized(s_recarray), [20]) + + # Show as a scalar that inherits from np.generic. + s_scalar = s_recarray[0] + assert isinstance(s_scalar, np.void) + assert m.f_simple(s_scalar) == 20 + + # Show that an *array* scalar (np.ndarray.shape == ()) does not convert. + # More specifically, conversion to SimpleStruct is not implicit. + s_recarray_scalar = s_recarray.reshape(()) + assert isinstance(s_recarray_scalar, np.ndarray) + assert s_recarray_scalar.dtype == simple_dtype + with pytest.raises(TypeError) as excinfo: + m.f_simple(s_recarray_scalar) + assert "incompatible function arguments" in str(excinfo.value) + # Explicitly convert to m.SimpleStruct. + assert m.f_simple(m.SimpleStruct.fromtuple(s_recarray_scalar.item())) == 20 + + # Show that an array of dtype=object does *not* convert. + s_array_object = np.array([s]) + assert s_array_object.dtype == object + with pytest.raises(TypeError) as excinfo: + m.f_simple_vectorized(s_array_object) + assert "incompatible function arguments" in str(excinfo.value) + # Explicitly convert to `np.array(..., dtype=simple_dtype)` + s_array = np.array([s.astuple()], dtype=simple_dtype) + np.testing.assert_array_equal(m.f_simple_vectorized(s_array), [20]) def test_register_dtype(): with pytest.raises(RuntimeError) as excinfo: m.register_dtype() - assert 'dtype is already registered' in str(excinfo.value) + assert "dtype is already registered" in str(excinfo.value) -@pytest.unsupported_on_pypy +@pytest.mark.xfail("env.PYPY") def test_str_leak(): from sys import getrefcount + fmt = "f4" pytest.gc_collect() start = getrefcount(fmt) diff --git a/tests/test_numpy_vectorize.cpp b/tests/test_numpy_vectorize.cpp index a875a74b99..274b7558a9 100644 --- a/tests/test_numpy_vectorize.cpp +++ b/tests/test_numpy_vectorize.cpp @@ -17,7 +17,7 @@ double my_func(int x, float y, double z) { } TEST_SUBMODULE(numpy_vectorize, m) { - try { py::module::import("numpy"); } + try { py::module_::import("numpy"); } catch (...) { return; } // test_vectorize, test_docs, test_array_collapse @@ -37,7 +37,7 @@ TEST_SUBMODULE(numpy_vectorize, m) { )); // test_type_selection - // Numpy function which only accepts specific data types + // NumPy function which only accepts specific data types m.def("selective_func", [](py::array_t) { return "Int branch taken."; }); m.def("selective_func", [](py::array_t) { return "Float branch taken."; }); m.def("selective_func", [](py::array_t, py::array::c_style>) { return "Complex float branch taken."; }); @@ -50,7 +50,9 @@ TEST_SUBMODULE(numpy_vectorize, m) { NonPODClass(int v) : value{v} {} int value; }; - py::class_(m, "NonPODClass").def(py::init()); + py::class_(m, "NonPODClass") + .def(py::init()) + .def_readwrite("value", &NonPODClass::value); m.def("vec_passthrough", py::vectorize( [](double *a, double b, py::array_t c, const int &d, int &e, NonPODClass f, const double g) { return *a + b + c.at(0) + d + e + f.value + g; @@ -81,9 +83,11 @@ TEST_SUBMODULE(numpy_vectorize, m) { py::array_t arg2, py::array_t arg3 ) { - ssize_t ndim; - std::vector shape; + py::ssize_t ndim; + std::vector shape; std::array buffers {{ arg1.request(), arg2.request(), arg3.request() }}; return py::detail::broadcast(buffers, ndim, shape); }); + + m.def("add_to", py::vectorize([](NonPODClass& x, int a) { x.value += a; })); } diff --git a/tests/test_numpy_vectorize.py b/tests/test_numpy_vectorize.py index bd3c01347c..4e6b2d1982 100644 --- a/tests/test_numpy_vectorize.py +++ b/tests/test_numpy_vectorize.py @@ -2,10 +2,7 @@ import pytest from pybind11_tests import numpy_vectorize as m -pytestmark = pytest.requires_numpy - -with pytest.suppress(ImportError): - import numpy as np +np = pytest.importorskip("numpy") def test_vectorize(capture): @@ -20,28 +17,40 @@ def test_vectorize(capture): assert capture == "my_func(x:int=1, y:float=2, z:float=3)" with capture: assert np.allclose(f(np.array([1, 3]), np.array([2, 4]), 3), [6, 36]) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=3) my_func(x:int=3, y:float=4, z:float=3) """ + ) with capture: - a = np.array([[1, 2], [3, 4]], order='F') - b = np.array([[10, 20], [30, 40]], order='F') + a = np.array([[1, 2], [3, 4]], order="F") + b = np.array([[10, 20], [30, 40]], order="F") c = 3 result = f(a, b, c) assert np.allclose(result, a * b * c) assert result.flags.f_contiguous # All inputs are F order and full or singletons, so we the result is in col-major order: - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=10, z:float=3) my_func(x:int=3, y:float=30, z:float=3) my_func(x:int=2, y:float=20, z:float=3) my_func(x:int=4, y:float=40, z:float=3) """ + ) with capture: - a, b, c = np.array([[1, 3, 5], [7, 9, 11]]), np.array([[2, 4, 6], [8, 10, 12]]), 3 + a, b, c = ( + np.array([[1, 3, 5], [7, 9, 11]]), + np.array([[2, 4, 6], [8, 10, 12]]), + 3, + ) assert np.allclose(f(a, b, c), a * b * c) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=3) my_func(x:int=3, y:float=4, z:float=3) my_func(x:int=5, y:float=6, z:float=3) @@ -49,10 +58,13 @@ def test_vectorize(capture): my_func(x:int=9, y:float=10, z:float=3) my_func(x:int=11, y:float=12, z:float=3) """ + ) with capture: a, b, c = np.array([[1, 2, 3], [4, 5, 6]]), np.array([2, 3, 4]), 2 assert np.allclose(f(a, b, c), a * b * c) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=2) my_func(x:int=2, y:float=3, z:float=2) my_func(x:int=3, y:float=4, z:float=2) @@ -60,10 +72,13 @@ def test_vectorize(capture): my_func(x:int=5, y:float=3, z:float=2) my_func(x:int=6, y:float=4, z:float=2) """ + ) with capture: a, b, c = np.array([[1, 2, 3], [4, 5, 6]]), np.array([[2], [3]]), 2 assert np.allclose(f(a, b, c), a * b * c) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=2) my_func(x:int=2, y:float=2, z:float=2) my_func(x:int=3, y:float=2, z:float=2) @@ -71,10 +86,17 @@ def test_vectorize(capture): my_func(x:int=5, y:float=3, z:float=2) my_func(x:int=6, y:float=3, z:float=2) """ + ) with capture: - a, b, c = np.array([[1, 2, 3], [4, 5, 6]], order='F'), np.array([[2], [3]]), 2 + a, b, c = ( + np.array([[1, 2, 3], [4, 5, 6]], order="F"), + np.array([[2], [3]]), + 2, + ) assert np.allclose(f(a, b, c), a * b * c) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=2) my_func(x:int=2, y:float=2, z:float=2) my_func(x:int=3, y:float=2, z:float=2) @@ -82,36 +104,53 @@ def test_vectorize(capture): my_func(x:int=5, y:float=3, z:float=2) my_func(x:int=6, y:float=3, z:float=2) """ + ) with capture: a, b, c = np.array([[1, 2, 3], [4, 5, 6]])[::, ::2], np.array([[2], [3]]), 2 assert np.allclose(f(a, b, c), a * b * c) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=2) my_func(x:int=3, y:float=2, z:float=2) my_func(x:int=4, y:float=3, z:float=2) my_func(x:int=6, y:float=3, z:float=2) """ + ) with capture: - a, b, c = np.array([[1, 2, 3], [4, 5, 6]], order='F')[::, ::2], np.array([[2], [3]]), 2 + a, b, c = ( + np.array([[1, 2, 3], [4, 5, 6]], order="F")[::, ::2], + np.array([[2], [3]]), + 2, + ) assert np.allclose(f(a, b, c), a * b * c) - assert capture == """ + assert ( + capture + == """ my_func(x:int=1, y:float=2, z:float=2) my_func(x:int=3, y:float=2, z:float=2) my_func(x:int=4, y:float=3, z:float=2) my_func(x:int=6, y:float=3, z:float=2) """ + ) def test_type_selection(): assert m.selective_func(np.array([1], dtype=np.int32)) == "Int branch taken." assert m.selective_func(np.array([1.0], dtype=np.float32)) == "Float branch taken." - assert m.selective_func(np.array([1.0j], dtype=np.complex64)) == "Complex float branch taken." + assert ( + m.selective_func(np.array([1.0j], dtype=np.complex64)) + == "Complex float branch taken." + ) def test_docs(doc): - assert doc(m.vectorized_func) == """ + assert ( + doc(m.vectorized_func) + == """ vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object """ # noqa: E501 line too long + ) def test_trivial_broadcasting(): @@ -119,16 +158,24 @@ def test_trivial_broadcasting(): assert vectorized_is_trivial(1, 2, 3) == trivial.c_trivial assert vectorized_is_trivial(np.array(1), np.array(2), 3) == trivial.c_trivial - assert vectorized_is_trivial(np.array([1, 3]), np.array([2, 4]), 3) == trivial.c_trivial + assert ( + vectorized_is_trivial(np.array([1, 3]), np.array([2, 4]), 3) + == trivial.c_trivial + ) assert trivial.c_trivial == vectorized_is_trivial( - np.array([[1, 3, 5], [7, 9, 11]]), np.array([[2, 4, 6], [8, 10, 12]]), 3) - assert vectorized_is_trivial( - np.array([[1, 2, 3], [4, 5, 6]]), np.array([2, 3, 4]), 2) == trivial.non_trivial - assert vectorized_is_trivial( - np.array([[1, 2, 3], [4, 5, 6]]), np.array([[2], [3]]), 2) == trivial.non_trivial - z1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]], dtype='int32') - z2 = np.array(z1, dtype='float32') - z3 = np.array(z1, dtype='float64') + np.array([[1, 3, 5], [7, 9, 11]]), np.array([[2, 4, 6], [8, 10, 12]]), 3 + ) + assert ( + vectorized_is_trivial(np.array([[1, 2, 3], [4, 5, 6]]), np.array([2, 3, 4]), 2) + == trivial.non_trivial + ) + assert ( + vectorized_is_trivial(np.array([[1, 2, 3], [4, 5, 6]]), np.array([[2], [3]]), 2) + == trivial.non_trivial + ) + z1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]], dtype="int32") + z2 = np.array(z1, dtype="float32") + z3 = np.array(z1, dtype="float64") assert vectorized_is_trivial(z1, z2, z3) == trivial.c_trivial assert vectorized_is_trivial(1, z2, z3) == trivial.c_trivial assert vectorized_is_trivial(z1, 1, z3) == trivial.c_trivial @@ -138,7 +185,7 @@ def test_trivial_broadcasting(): assert vectorized_is_trivial(1, 1, z3[::2, ::2]) == trivial.non_trivial assert vectorized_is_trivial(z1, 1, z3[1::4, 1::4]) == trivial.c_trivial - y1 = np.array(z1, order='F') + y1 = np.array(z1, order="F") y2 = np.array(y1) y3 = np.array(y1) assert vectorized_is_trivial(y1, y2, y3) == trivial.f_trivial @@ -159,30 +206,41 @@ def test_trivial_broadcasting(): def test_passthrough_arguments(doc): assert doc(m.vec_passthrough) == ( - "vec_passthrough(" + ", ".join([ - "arg0: float", - "arg1: numpy.ndarray[numpy.float64]", - "arg2: numpy.ndarray[numpy.float64]", - "arg3: numpy.ndarray[numpy.int32]", - "arg4: int", - "arg5: m.numpy_vectorize.NonPODClass", - "arg6: numpy.ndarray[numpy.float64]"]) + ") -> object") - - b = np.array([[10, 20, 30]], dtype='float64') + "vec_passthrough(" + + ", ".join( + [ + "arg0: float", + "arg1: numpy.ndarray[numpy.float64]", + "arg2: numpy.ndarray[numpy.float64]", + "arg3: numpy.ndarray[numpy.int32]", + "arg4: int", + "arg5: m.numpy_vectorize.NonPODClass", + "arg6: numpy.ndarray[numpy.float64]", + ] + ) + + ") -> object" + ) + + b = np.array([[10, 20, 30]], dtype="float64") c = np.array([100, 200]) # NOT a vectorized argument - d = np.array([[1000], [2000], [3000]], dtype='int') - g = np.array([[1000000, 2000000, 3000000]], dtype='int') # requires casting + d = np.array([[1000], [2000], [3000]], dtype="int") + g = np.array([[1000000, 2000000, 3000000]], dtype="int") # requires casting assert np.all( - m.vec_passthrough(1, b, c, d, 10000, m.NonPODClass(100000), g) == - np.array([[1111111, 2111121, 3111131], - [1112111, 2112121, 3112131], - [1113111, 2113121, 3113131]])) + m.vec_passthrough(1, b, c, d, 10000, m.NonPODClass(100000), g) + == np.array( + [ + [1111111, 2111121, 3111131], + [1112111, 2112121, 3112131], + [1113111, 2113121, 3113131], + ] + ) + ) def test_method_vectorization(): o = m.VectorizeTestClass(3) - x = np.array([1, 2], dtype='int') - y = np.array([[10], [20]], dtype='float32') + x = np.array([1, 2], dtype="int") + y = np.array([[10], [20]], dtype="float32") assert np.all(o.method(x, y) == [[14, 15], [24, 25]]) @@ -191,7 +249,18 @@ def test_array_collapse(): assert not isinstance(m.vectorized_func(np.array(1), 2, 3), np.ndarray) z = m.vectorized_func([1], 2, 3) assert isinstance(z, np.ndarray) - assert z.shape == (1, ) + assert z.shape == (1,) z = m.vectorized_func(1, [[[2]]], 3) assert isinstance(z, np.ndarray) assert z.shape == (1, 1, 1) + + +def test_vectorized_noreturn(): + x = m.NonPODClass(0) + assert x.value == 0 + m.add_to(x, [1, 2, 3, 4]) + assert x.value == 10 + m.add_to(x, 1) + assert x.value == 11 + m.add_to(x, [[1, 1], [2, 3]]) + assert x.value == 18 diff --git a/tests/test_opaque_types.cpp b/tests/test_opaque_types.cpp index 0d20d9a01c..5a23431635 100644 --- a/tests/test_opaque_types.cpp +++ b/tests/test_opaque_types.cpp @@ -60,8 +60,14 @@ TEST_SUBMODULE(opaque_types, m) { m.def("get_null_str_value", [](char *ptr) { return reinterpret_cast(ptr); }); m.def("return_unique_ptr", []() -> std::unique_ptr { - StringList *result = new StringList(); + auto *result = new StringList(); result->push_back("some value"); return std::unique_ptr(result); }); + + // test unions + py::class_(m, "IntFloat") + .def(py::init<>()) + .def_readwrite("i", &IntFloat::i) + .def_readwrite("f", &IntFloat::f); } diff --git a/tests/test_opaque_types.py b/tests/test_opaque_types.py index 3f2392775d..773794634c 100644 --- a/tests/test_opaque_types.py +++ b/tests/test_opaque_types.py @@ -32,12 +32,15 @@ def test_pointers(msg): with pytest.raises(TypeError) as excinfo: m.get_void_ptr_value([1, 2, 3]) # This should not work - assert msg(excinfo.value) == """ + assert ( + msg(excinfo.value) + == """ get_void_ptr_value(): incompatible function arguments. The following argument types are supported: 1. (arg0: capsule) -> int Invoked with: [1, 2, 3] """ # noqa: E501 line too long + ) assert m.return_null_str() is None assert m.get_null_str_value(m.return_null_str()) is not None @@ -45,3 +48,11 @@ def test_pointers(msg): ptr = m.return_unique_ptr() assert "StringList" in repr(ptr) assert m.print_opaque_list(ptr) == "Opaque list: [some value]" + + +def test_unions(): + int_float_union = m.IntFloat() + int_float_union.i = 42 + assert int_float_union.i == 42 + int_float_union.f = 3.0 + assert int_float_union.f == 3.0 diff --git a/tests/test_operator_overloading.cpp b/tests/test_operator_overloading.cpp index f3c2eaafa9..0a27bfd57b 100644 --- a/tests/test_operator_overloading.cpp +++ b/tests/test_operator_overloading.cpp @@ -73,7 +73,7 @@ namespace std { // Not a good hash function, but easy to test size_t operator()(const Vector2 &) { return 4; } }; -} +} // namespace std // Not a good abs function, but easy to test. std::string abs(const Vector2&) { @@ -88,11 +88,11 @@ std::string abs(const Vector2&) { // Here, we suppress the warning using `#pragma diagnostic`. // Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 // TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). - #if (__APPLE__) && (__clang__) - #if (__clang_major__ >= 10) && (__clang_minor__ >= 0) && (__clang_patchlevel__ >= 1) + #if defined(__APPLE__) && defined(__clang__) + #if (__clang_major__ >= 10) #pragma GCC diagnostic ignored "-Wself-assign-overloaded" #endif - #elif (__clang__) + #elif defined(__clang__) #if (__clang_major__ >= 7) #pragma GCC diagnostic ignored "-Wself-assign-overloaded" #endif diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index 39e3aee271..5dbfb32c8a 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -56,23 +56,23 @@ def test_operator_overloading(): del v3 assert cstats.alive() == 0 assert cstats.values() == [ - '[1.000000, 2.000000]', - '[3.000000, -1.000000]', - '[1.000000, 2.000000]', - '[-3.000000, 1.000000]', - '[4.000000, 1.000000]', - '[-2.000000, 3.000000]', - '[-7.000000, -6.000000]', - '[9.000000, 10.000000]', - '[8.000000, 16.000000]', - '[0.125000, 0.250000]', - '[7.000000, 6.000000]', - '[9.000000, 10.000000]', - '[8.000000, 16.000000]', - '[8.000000, 4.000000]', - '[3.000000, -2.000000]', - '[3.000000, -0.500000]', - '[6.000000, -2.000000]', + "[1.000000, 2.000000]", + "[3.000000, -1.000000]", + "[1.000000, 2.000000]", + "[-3.000000, 1.000000]", + "[4.000000, 1.000000]", + "[-2.000000, 3.000000]", + "[-7.000000, -6.000000]", + "[9.000000, 10.000000]", + "[8.000000, 16.000000]", + "[0.125000, 0.250000]", + "[7.000000, 6.000000]", + "[9.000000, 10.000000]", + "[8.000000, 16.000000]", + "[8.000000, 4.000000]", + "[3.000000, -2.000000]", + "[3.000000, -0.500000]", + "[6.000000, -2.000000]", ] assert cstats.default_constructions == 0 assert cstats.copy_constructions == 0 diff --git a/tests/test_ownership_transfer.cpp b/tests/test_ownership_transfer.cpp index bedae4c8fb..49724e04eb 100644 --- a/tests/test_ownership_transfer.cpp +++ b/tests/test_ownership_transfer.cpp @@ -95,27 +95,27 @@ class DefinePyBaseWrapped : public py::wrapper> { }; // BaseBad - No wrapper alias. -typedef DefineBase BaseBad; -typedef DefineBaseContainer BaseBadContainer; -typedef Stats ChildBadStats; +using BaseBad = DefineBase; +using BaseBadContainer = DefineBaseContainer; +using ChildBadStats = Stats; // Base - wrapper alias used in pybind definition. -typedef DefineBase Base; -typedef DefinePyBase PyBase; -typedef DefineBaseContainer BaseContainer; -typedef Stats ChildStats; +using Base = DefineBase; +using PyBase = DefinePyBase; +using BaseContainer = DefineBaseContainer; +using ChildStats = Stats; // - Unique Ptr // BaseBad - No wrapper alias. -typedef DefineBase BaseBadUnique; -typedef DefineBaseUniqueContainer BaseBadUniqueContainer; -typedef Stats ChildBadUniqueStats; +using BaseBadUnique = DefineBase; +using BaseBadUniqueContainer = DefineBaseUniqueContainer; +using ChildBadUniqueStats = Stats; // Base - wrapper alias used directly. -typedef DefineBase BaseUnique; -typedef DefinePyBaseWrapped PyBaseUnique; -typedef DefineBaseUniqueContainer BaseUniqueContainer; -typedef Stats ChildUniqueStats; +using BaseUnique = DefineBase; +using PyBaseUnique = DefinePyBaseWrapped; +using BaseUniqueContainer = DefineBaseUniqueContainer; +using ChildUniqueStats = Stats; class PyInstanceStats { public: diff --git a/tests/test_ownership_transfer.py b/tests/test_ownership_transfer.py index c8a6edfe91..f48add15bd 100644 --- a/tests/test_ownership_transfer.py +++ b/tests/test_ownership_transfer.py @@ -2,6 +2,8 @@ from pybind11_tests import ownership_transfer as m from pybind11_tests import ConstructorStats +import env # noqa: F401 + import pytest import sys import weakref @@ -32,19 +34,17 @@ def get_cstats(): return ChildT -ChildBad = define_child('ChildBad', m.BaseBad, m.ChildBadStats) -Child = define_child('Child', m.Base, m.ChildStats) +ChildBad = define_child("ChildBad", m.BaseBad, m.ChildBadStats) +Child = define_child("Child", m.Base, m.ChildStats) -ChildBadUnique = define_child( - 'ChildBadUnique', m.BaseBadUnique, m.ChildBadUniqueStats) -ChildUnique = define_child( - 'ChildUnique', m.BaseUnique, m.ChildUniqueStats) +ChildBadUnique = define_child("ChildBadUnique", m.BaseBadUnique, m.ChildBadUniqueStats) +ChildUnique = define_child("ChildUnique", m.BaseUnique, m.ChildUniqueStats) # TODO(eric.cousineau): See if this is at all possibly on PyPy. # Placing `pytest.gc_collect` near `del` statements indicates that we are not # capturing deletion properly. -@pytest.unsupported_on_pypy +@pytest.mark.skipif(env.PYPY, reason="Unsupported on PyPy") def test_shared_ptr_derived_slicing(capture): leaked_count = [0] is_py38 = sys.version_info[:2] >= (3, 8) @@ -129,7 +129,7 @@ def cstats_alive_except_leaked(): assert cstats_alive_except_leaked() == 0 -@pytest.unsupported_on_pypy +@pytest.mark.skipif(env.PYPY, reason="Unsupported on PyPy") def test_unique_ptr_derived_slicing(capture): # [ Bad ] cstats = ChildBadUnique.get_cstats() diff --git a/tests/test_pickling.cpp b/tests/test_pickling.cpp index 9dc63bda3b..1a48595af6 100644 --- a/tests/test_pickling.cpp +++ b/tests/test_pickling.cpp @@ -31,7 +31,8 @@ TEST_SUBMODULE(pickling, m) { using Pickleable::Pickleable; }; - py::class_(m, "Pickleable") + py::class_ pyPickleable(m, "Pickleable"); + pyPickleable .def(py::init()) .def("value", &Pickleable::value) .def("extra1", &Pickleable::extra1) @@ -43,17 +44,20 @@ TEST_SUBMODULE(pickling, m) { .def("__getstate__", [](const Pickleable &p) { /* Return a tuple that fully encodes the state of the object */ return py::make_tuple(p.value(), p.extra1(), p.extra2()); - }) - .def("__setstate__", [](Pickleable &p, py::tuple t) { - if (t.size() != 3) - throw std::runtime_error("Invalid state!"); - /* Invoke the constructor (need to use in-place version) */ - new (&p) Pickleable(t[0].cast()); - - /* Assign any additional state */ - p.setExtra1(t[1].cast()); - p.setExtra2(t[2].cast()); }); + ignoreOldStyleInitWarnings([&pyPickleable]() { + pyPickleable + .def("__setstate__", [](Pickleable &p, py::tuple t) { + if (t.size() != 3) + throw std::runtime_error("Invalid state!"); + /* Invoke the constructor (need to use in-place version) */ + new (&p) Pickleable(t[0].cast()); + + /* Assign any additional state */ + p.setExtra1(t[1].cast()); + p.setExtra2(t[2].cast()); + }); + }); py::class_(m, "PickleableNew") .def(py::init()) @@ -87,27 +91,31 @@ TEST_SUBMODULE(pickling, m) { using PickleableWithDict::PickleableWithDict; }; - py::class_(m, "PickleableWithDict", py::dynamic_attr()) + py::class_ pyPickleableWithDict(m, "PickleableWithDict", py::dynamic_attr()); + pyPickleableWithDict .def(py::init()) .def_readwrite("value", &PickleableWithDict::value) .def_readwrite("extra", &PickleableWithDict::extra) .def("__getstate__", [](py::object self) { /* Also include __dict__ in state */ return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__")); - }) - .def("__setstate__", [](py::object self, py::tuple t) { - if (t.size() != 3) - throw std::runtime_error("Invalid state!"); - /* Cast and construct */ - auto& p = self.cast(); - new (&p) PickleableWithDict(t[0].cast()); - - /* Assign C++ state */ - p.extra = t[1].cast(); - - /* Assign Python state */ - self.attr("__dict__") = t[2]; }); + ignoreOldStyleInitWarnings([&pyPickleableWithDict]() { + pyPickleableWithDict + .def("__setstate__", [](py::object self, py::tuple t) { + if (t.size() != 3) + throw std::runtime_error("Invalid state!"); + /* Cast and construct */ + auto& p = self.cast(); + new (&p) PickleableWithDict(t[0].cast()); + + /* Assign C++ state */ + p.extra = t[1].cast(); + + /* Assign Python state */ + self.attr("__dict__") = t[2]; + }); + }); py::class_(m, "PickleableWithDictNew") .def(py::init()) diff --git a/tests/test_pickling.py b/tests/test_pickling.py index 58d67a6339..6b27a73a57 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- import pytest + +import env # noqa: F401 + from pybind11_tests import pickling as m try: @@ -22,7 +25,7 @@ def test_roundtrip(cls_name): assert p2.extra2() == p.extra2() -@pytest.unsupported_on_pypy +@pytest.mark.xfail("env.PYPY") @pytest.mark.parametrize("cls_name", ["PickleableWithDict", "PickleableWithDictNew"]) def test_roundtrip_with_dict(cls_name): cls = getattr(m, cls_name) @@ -39,5 +42,6 @@ def test_roundtrip_with_dict(cls_name): def test_enum_pickle(): from pybind11_tests import enums as e + data = pickle.dumps(e.EOne, 2) assert e.EOne == pickle.loads(data) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 9f7bc37dc6..6921796aa0 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -80,6 +80,7 @@ TEST_SUBMODULE(pytypes, m) { m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); m.def("str_from_object", [](const py::object& obj) { return py::str(obj); }); m.def("repr_from_object", [](const py::object& obj) { return py::repr(obj); }); + m.def("str_from_handle", [](py::handle h) { return py::str(h); }); m.def("str_format", []() { auto s1 = "{} + {} = {}"_s.format(1, 2, 3); @@ -91,6 +92,10 @@ TEST_SUBMODULE(pytypes, m) { m.def("bytes_from_string", []() { return py::bytes(std::string("foo")); }); m.def("bytes_from_str", []() { return py::bytes(py::str("bar", 3)); }); + // test bytearray + m.def("bytearray_from_string", []() { return py::bytearray(std::string("foo")); }); + m.def("bytearray_size", []() { return py::bytearray("foo").size(); }); + // test_capsule m.def("return_capsule_with_destructor", []() { py::print("creating capsule"); @@ -107,7 +112,7 @@ TEST_SUBMODULE(pytypes, m) { }); m.def("return_capsule_with_name_and_destructor", []() { - auto capsule = py::capsule((void *) 1234, "pointer type description", [](PyObject *ptr) { + auto capsule = py::capsule((void *) 12345, "pointer type description", [](PyObject *ptr) { if (ptr) { auto name = PyCapsule_GetName(ptr); py::print("destructing capsule ({}, '{}')"_s.format( @@ -115,8 +120,19 @@ TEST_SUBMODULE(pytypes, m) { )); } }); - void *contents = capsule; - py::print("created capsule ({}, '{}')"_s.format((size_t) contents, capsule.name())); + + capsule.set_pointer((void *) 1234); + + // Using get_pointer() + void* contents1 = static_cast(capsule); + void* contents2 = capsule.get_pointer(); + void* contents3 = capsule.get_pointer(); + + auto result1 = reinterpret_cast(contents1); + auto result2 = reinterpret_cast(contents2); + auto result3 = reinterpret_cast(contents3); + + py::print("created capsule ({}, '{}')"_s.format(result1 & result2 & result3, capsule.name())); return capsule; }); @@ -127,7 +143,7 @@ TEST_SUBMODULE(pytypes, m) { d["basic_attr"] = o.attr("basic_attr"); auto l = py::list(); - for (const auto &item : o.attr("begin_end")) { + for (auto item : o.attr("begin_end")) { l.append(item); } d["begin_end"] = l; @@ -197,6 +213,8 @@ TEST_SUBMODULE(pytypes, m) { // test_constructors m.def("default_constructors", []() { return py::dict( + "bytes"_a=py::bytes(), + "bytearray"_a=py::bytearray(), "str"_a=py::str(), "bool"_a=py::bool_(), "int"_a=py::int_(), @@ -210,6 +228,8 @@ TEST_SUBMODULE(pytypes, m) { m.def("converting_constructors", [](py::dict d) { return py::dict( + "bytes"_a=py::bytes(d["bytes"]), + "bytearray"_a=py::bytearray(d["bytearray"]), "str"_a=py::str(d["str"]), "bool"_a=py::bool_(d["bool"]), "int"_a=py::int_(d["int"]), @@ -225,6 +245,8 @@ TEST_SUBMODULE(pytypes, m) { m.def("cast_functions", [](py::dict d) { // When converting between Python types, obj.cast() should be the same as T(obj) return py::dict( + "bytes"_a=d["bytes"].cast(), + "bytearray"_a=d["bytearray"].cast(), "str"_a=d["str"].cast(), "bool"_a=d["bool"].cast(), "int"_a=d["int"].cast(), @@ -237,6 +259,24 @@ TEST_SUBMODULE(pytypes, m) { ); }); + m.def("convert_to_pybind11_str", [](py::object o) { return py::str(o); }); + + m.def("nonconverting_constructor", [](std::string type, py::object value, bool move) -> py::object { + if (type == "bytes") { + return move ? py::bytes(std::move(value)) : py::bytes(value); + } + else if (type == "none") { + return move ? py::none(std::move(value)) : py::none(value); + } + else if (type == "ellipsis") { + return move ? py::ellipsis(std::move(value)) : py::ellipsis(value); + } + else if (type == "type") { + return move ? py::type(std::move(value)) : py::type(value); + } + throw std::runtime_error("Invalid type"); + }); + m.def("get_implicit_casting", []() { py::dict d; d["char*_i1"] = "abc"; @@ -283,7 +323,7 @@ TEST_SUBMODULE(pytypes, m) { py::print("no new line here", "end"_a=" -- "); py::print("next print"); - auto py_stderr = py::module::import("sys").attr("stderr"); + auto py_stderr = py::module_::import("sys").attr("stderr"); py::print("this goes to stderr", "file"_a=py_stderr); py::print("flush", "flush"_a=true); @@ -319,6 +359,16 @@ TEST_SUBMODULE(pytypes, m) { return a[py::slice(0, -1, 2)]; }); + // See #2361 + m.def("issue2361_str_implicit_copy_none", []() { + py::str is_this_none = py::none(); + return is_this_none; + }); + m.def("issue2361_dict_implicit_copy_none", []() { + py::dict is_this_none = py::none(); + return is_this_none; + }); + m.def("test_memoryview_object", [](py::buffer b) { return py::memoryview(b); }); @@ -364,7 +414,31 @@ TEST_SUBMODULE(pytypes, m) { m.def("test_memoryview_from_memory", []() { const char* buf = "\xff\xe1\xab\x37"; return py::memoryview::from_memory( - buf, static_cast(strlen(buf))); + buf, static_cast(strlen(buf))); }); #endif + + // test_builtin_functions + m.def("get_len", [](py::handle h) { return py::len(h); }); + +#ifdef PYBIND11_STR_LEGACY_PERMISSIVE + m.attr("PYBIND11_STR_LEGACY_PERMISSIVE") = true; +#endif + + m.def("isinstance_pybind11_bytes", [](py::object o) { return py::isinstance(o); }); + m.def("isinstance_pybind11_str", [](py::object o) { return py::isinstance(o); }); + + m.def("pass_to_pybind11_bytes", [](py::bytes b) { return py::len(b); }); + m.def("pass_to_pybind11_str", [](py::str s) { return py::len(s); }); + m.def("pass_to_std_string", [](std::string s) { return s.size(); }); + + // test_weakref + m.def("weakref_from_handle", + [](py::handle h) { return py::weakref(h); }); + m.def("weakref_from_handle_and_function", + [](py::handle h, py::function f) { return py::weakref(h, f); }); + m.def("weakref_from_object", + [](py::object o) { return py::weakref(o); }); + m.def("weakref_from_object_and_function", + [](py::object o, py::function f) { return py::weakref(o, f); }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 4cfc707a32..f17bf76b77 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -3,6 +3,8 @@ import pytest import sys +import env # noqa: F401 + from pybind11_tests import pytypes as m from pybind11_tests import debug_enabled @@ -26,13 +28,16 @@ def test_list(capture, doc): lst.append("value2") m.print_list(lst) - assert capture.unordered == """ + assert ( + capture.unordered + == """ Entry at position 0: value list item 0: inserted-0 list item 1: overwritten list item 2: inserted-2 list item 3: value2 """ + ) assert doc(m.get_list) == "get_list() -> list" assert doc(m.print_list) == "print_list(arg0: list) -> None" @@ -50,12 +55,15 @@ def test_set(capture, doc): with capture: s.add("key4") m.print_set(s) - assert capture.unordered == """ + assert ( + capture.unordered + == """ key: key1 key: key2 key: key3 key: key4 """ + ) assert not m.set_contains(set([]), 42) assert m.set_contains({42}, 42) @@ -72,10 +80,13 @@ def test_dict(capture, doc): with capture: d["key2"] = "value2" m.print_dict(d) - assert capture.unordered == """ + assert ( + capture.unordered + == """ key: key, value=value key: key2, value=value2 """ + ) assert not m.dict_contains({}, 42) assert m.dict_contains({42: None}, 42) @@ -102,49 +113,78 @@ def __repr__(self): assert m.str_from_object(A()) == "this is a str" assert m.repr_from_object(A()) == "this is a repr" + assert m.str_from_handle(A()) == "this is a str" s1, s2 = m.str_format() assert s1 == "1 + 2 = 3" assert s1 == s2 + malformed_utf8 = b"\x80" + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert m.str_from_object(malformed_utf8) is malformed_utf8 + elif env.PY2: + with pytest.raises(UnicodeDecodeError): + m.str_from_object(malformed_utf8) + else: + assert m.str_from_object(malformed_utf8) == "b'\\x80'" + if env.PY2: + with pytest.raises(UnicodeDecodeError): + m.str_from_handle(malformed_utf8) + else: + assert m.str_from_handle(malformed_utf8) == "b'\\x80'" + def test_bytes(doc): assert m.bytes_from_string().decode() == "foo" assert m.bytes_from_str().decode() == "bar" assert doc(m.bytes_from_str) == "bytes_from_str() -> {}".format( - "bytes" if sys.version_info[0] == 3 else "str" + "str" if env.PY2 else "bytes" ) +def test_bytearray(doc): + assert m.bytearray_from_string().decode() == "foo" + assert m.bytearray_size() == len("foo") + + def test_capsule(capture): pytest.gc_collect() with capture: a = m.return_capsule_with_destructor() del a pytest.gc_collect() - assert capture.unordered == """ + assert ( + capture.unordered + == """ creating capsule destructing capsule """ + ) with capture: a = m.return_capsule_with_destructor_2() del a pytest.gc_collect() - assert capture.unordered == """ + assert ( + capture.unordered + == """ creating capsule destructing capsule: 1234 """ + ) with capture: a = m.return_capsule_with_name_and_destructor() del a pytest.gc_collect() - assert capture.unordered == """ + assert ( + capture.unordered + == """ created capsule (1234, 'pointer type description') destructing capsule (1234, 'pointer type description') """ + ) def test_accessors(): @@ -188,11 +228,18 @@ def func(self, x, *args): def test_constructors(): """C++ default and converting constructors are equivalent to type calls in Python""" - types = [str, bool, int, float, tuple, list, dict, set] + types = [bytes, bytearray, str, bool, int, float, tuple, list, dict, set] expected = {t.__name__: t() for t in types} + if env.PY2: + # Note that bytes.__name__ == 'str' in Python 2. + # pybind11::str is unicode even under Python 2. + expected["bytes"] = bytes() + expected["str"] = unicode() # noqa: F821 assert m.default_constructors() == expected data = { + bytes: b"41", # Currently no supported or working conversions. + bytearray: bytearray(b"41"), str: 42, bool: "Not empty", int: "42", @@ -201,10 +248,15 @@ def test_constructors(): list: range(3), dict: [("two", 2), ("one", 1), ("three", 3)], set: [4, 4, 5, 6, 6, 6], - memoryview: b'abc' + memoryview: b"abc", } inputs = {k.__name__: v for k, v in data.items()} expected = {k.__name__: k(v) for k, v in data.items()} + if env.PY2: # Similar to the above. See comments above. + inputs["bytes"] = b"41" + inputs["str"] = 42 + expected["bytes"] = b"41" + expected["str"] = u"42" assert m.converting_constructors(inputs) == expected assert m.cast_functions(inputs) == expected @@ -220,21 +272,94 @@ def test_constructors(): assert noconv2[k] is expected[k] +def test_non_converting_constructors(): + non_converting_test_cases = [ + ("bytes", range(10)), + ("none", 42), + ("ellipsis", 42), + ("type", 42), + ] + for t, v in non_converting_test_cases: + for move in [True, False]: + with pytest.raises(TypeError) as excinfo: + m.nonconverting_constructor(t, v, move) + expected_error = "Object of type '{}' is not an instance of '{}'".format( + type(v).__name__, t + ) + assert str(excinfo.value) == expected_error + + +def test_pybind11_str_raw_str(): + # specifically to exercise pybind11::str::raw_str + cvt = m.convert_to_pybind11_str + assert cvt(u"Str") == u"Str" + assert cvt(b"Bytes") == u"Bytes" if env.PY2 else "b'Bytes'" + assert cvt(None) == u"None" + assert cvt(False) == u"False" + assert cvt(True) == u"True" + assert cvt(42) == u"42" + assert cvt(2 ** 65) == u"36893488147419103232" + assert cvt(-1.50) == u"-1.5" + assert cvt(()) == u"()" + assert cvt((18,)) == u"(18,)" + assert cvt([]) == u"[]" + assert cvt([28]) == u"[28]" + assert cvt({}) == u"{}" + assert cvt({3: 4}) == u"{3: 4}" + assert cvt(set()) == u"set([])" if env.PY2 else "set()" + assert cvt({3, 3}) == u"set([3])" if env.PY2 else "{3}" + + valid_orig = u"DZ" + valid_utf8 = valid_orig.encode("utf-8") + valid_cvt = cvt(valid_utf8) + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert valid_cvt is valid_utf8 + else: + assert type(valid_cvt) is unicode if env.PY2 else str # noqa: F821 + if env.PY2: + assert valid_cvt == valid_orig + else: + assert valid_cvt == "b'\\xc7\\xb1'" + + malformed_utf8 = b"\x80" + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert cvt(malformed_utf8) is malformed_utf8 + else: + if env.PY2: + with pytest.raises(UnicodeDecodeError): + cvt(malformed_utf8) + else: + malformed_cvt = cvt(malformed_utf8) + assert type(malformed_cvt) is str + assert malformed_cvt == "b'\\x80'" + + def test_implicit_casting(): """Tests implicit casting when assigning or appending to dicts and lists.""" z = m.get_implicit_casting() - assert z['d'] == { - 'char*_i1': 'abc', 'char*_i2': 'abc', 'char*_e': 'abc', 'char*_p': 'abc', - 'str_i1': 'str', 'str_i2': 'str1', 'str_e': 'str2', 'str_p': 'str3', - 'int_i1': 42, 'int_i2': 42, 'int_e': 43, 'int_p': 44 + assert z["d"] == { + "char*_i1": "abc", + "char*_i2": "abc", + "char*_e": "abc", + "char*_p": "abc", + "str_i1": "str", + "str_i2": "str1", + "str_e": "str2", + "str_p": "str3", + "int_i1": 42, + "int_i2": 42, + "int_e": 43, + "int_p": 44, } - assert z['l'] == [3, 6, 9, 12, 15] + assert z["l"] == [3, 6, 9, 12, 15] def test_print(capture): with capture: m.print_function() - assert capture == """ + assert ( + capture + == """ Hello, World! 1 2.0 three True -- multiple args *args-and-a-custom-separator @@ -242,14 +367,15 @@ def test_print(capture): flush py::print + str.format = this """ + ) assert capture.stderr == "this goes to stderr" with pytest.raises(RuntimeError) as excinfo: m.print_failure() assert str(excinfo.value) == "make_tuple(): unable to convert " + ( "argument of type 'UnregisteredType' to Python object" - if debug_enabled else - "arguments to Python object (compile in debug mode for details)" + if debug_enabled + else "arguments to Python object (compile in debug mode for details)" ) @@ -271,8 +397,23 @@ class Unhashable(object): def test_number_protocol(): for a, b in [(1, 1), (3, 5)]: - li = [a == b, a != b, a < b, a <= b, a > b, a >= b, a + b, - a - b, a * b, a / b, a | b, a & b, a ^ b, a >> b, a << b] + li = [ + a == b, + a != b, + a < b, + a <= b, + a > b, + a >= b, + a + b, + a - b, + a * b, + a / b, + a | b, + a & b, + a ^ b, + a >> b, + a << b, + ] assert m.test_number_protocol(a, b) == li @@ -281,18 +422,29 @@ def test_list_slicing(): assert li[::2] == m.test_list_slicing(li) -@pytest.mark.parametrize('method, args, fmt, expected_view', [ - (m.test_memoryview_object, (b'red',), 'B', b'red'), - (m.test_memoryview_buffer_info, (b'green',), 'B', b'green'), - (m.test_memoryview_from_buffer, (False,), 'h', [3, 1, 4, 1, 5]), - (m.test_memoryview_from_buffer, (True,), 'H', [2, 7, 1, 8]), - (m.test_memoryview_from_buffer_nativeformat, (), '@i', [4, 7, 5]), -]) +def test_issue2361(): + # See issue #2361 + assert m.issue2361_str_implicit_copy_none() == "None" + with pytest.raises(TypeError) as excinfo: + assert m.issue2361_dict_implicit_copy_none() + assert "'NoneType' object is not iterable" in str(excinfo.value) + + +@pytest.mark.parametrize( + "method, args, fmt, expected_view", + [ + (m.test_memoryview_object, (b"red",), "B", b"red"), + (m.test_memoryview_buffer_info, (b"green",), "B", b"green"), + (m.test_memoryview_from_buffer, (False,), "h", [3, 1, 4, 1, 5]), + (m.test_memoryview_from_buffer, (True,), "H", [2, 7, 1, 8]), + (m.test_memoryview_from_buffer_nativeformat, (), "@i", [4, 7, 5]), + ], +) def test_memoryview(method, args, fmt, expected_view): view = method(*args) assert isinstance(view, memoryview) assert view.format == fmt - if isinstance(expected_view, bytes) or sys.version_info[0] >= 3: + if isinstance(expected_view, bytes) or not env.PY2: view_as_list = list(view) else: # Using max to pick non-zero byte (big-endian vs little-endian). @@ -300,15 +452,16 @@ def test_memoryview(method, args, fmt, expected_view): assert view_as_list == list(expected_view) -@pytest.mark.skipif( - not hasattr(sys, 'getrefcount'), - reason='getrefcount is not available') -@pytest.mark.parametrize('method', [ - m.test_memoryview_object, - m.test_memoryview_buffer_info, -]) +@pytest.mark.xfail("env.PYPY", reason="getrefcount is not available") +@pytest.mark.parametrize( + "method", + [ + m.test_memoryview_object, + m.test_memoryview_buffer_info, + ], +) def test_memoryview_refcount(method): - buf = b'\x0a\x0b\x0c\x0d' + buf = b"\x0a\x0b\x0c\x0d" ref_before = sys.getrefcount(buf) view = method(buf) ref_after = sys.getrefcount(buf) @@ -319,13 +472,13 @@ def test_memoryview_refcount(method): def test_memoryview_from_buffer_empty_shape(): view = m.test_memoryview_from_buffer_empty_shape() assert isinstance(view, memoryview) - assert view.format == 'B' - if sys.version_info.major < 3: + assert view.format == "B" + if env.PY2: # Python 2 behavior is weird, but Python 3 (the future) is fine. # PyPy3 has #include +#include + template class NonZeroIterator { const T* ptr_; @@ -81,7 +83,7 @@ TEST_SUBMODULE(sequences_and_iterators, m) { py::class_(m,"Sliceable") .def(py::init()) .def("__getitem__",[](const Sliceable &s, py::slice slice) { - ssize_t start, stop, step, slicelength; + py::ssize_t start, stop, step, slicelength; if (!slice.compute(s.size, &start, &stop, &step, &slicelength)) throw py::error_already_set(); int istart = static_cast(start); @@ -198,7 +200,7 @@ TEST_SUBMODULE(sequences_and_iterators, m) { size_t start, stop, step, slicelength; if (!slice.compute(s.size(), &start, &stop, &step, &slicelength)) throw py::error_already_set(); - Sequence *seq = new Sequence(slicelength); + auto *seq = new Sequence(slicelength); for (size_t i = 0; i < slicelength; ++i) { (*seq)[i] = s[start]; start += step; } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index 8f6c0c4bbd..c3b608c436 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -10,7 +10,9 @@ def isclose(a, b, rel_tol=1e-05, abs_tol=0.0): def allclose(a_list, b_list, rel_tol=1e-05, abs_tol=0.0): - return all(isclose(a, b, rel_tol=rel_tol, abs_tol=abs_tol) for a, b in zip(a_list, b_list)) + return all( + isclose(a, b, rel_tol=rel_tol, abs_tol=abs_tol) for a, b in zip(a_list, b_list) + ) def test_generalized_iterators(): @@ -51,7 +53,7 @@ def test_sequence(): cstats = ConstructorStats.get(m.Sequence) s = m.Sequence(5) - assert cstats.values() == ['of size', '5'] + assert cstats.values() == ["of size", "5"] assert "Sequence" in repr(s) assert len(s) == 5 @@ -62,16 +64,16 @@ def test_sequence(): assert isclose(s[0], 12.34) and isclose(s[3], 56.78) rev = reversed(s) - assert cstats.values() == ['of size', '5'] + assert cstats.values() == ["of size", "5"] rev2 = s[::-1] - assert cstats.values() == ['of size', '5'] + assert cstats.values() == ["of size", "5"] it = iter(m.Sequence(0)) for _ in range(3): # __next__ must continue to raise StopIteration with pytest.raises(StopIteration): next(it) - assert cstats.values() == ['of size', '0'] + assert cstats.values() == ["of size", "0"] expected = [0, 56.78, 0, 0, 12.34] assert allclose(rev, expected) @@ -79,7 +81,7 @@ def test_sequence(): assert rev == rev2 rev[0::2] = m.Sequence([2.0, 2.0, 2.0]) - assert cstats.values() == ['of size', '3', 'from std::vector'] + assert cstats.values() == ["of size", "3", "from std::vector"] assert allclose(rev, [2, 56.78, 2, 0, 2]) @@ -103,10 +105,11 @@ def test_sequence(): def test_sequence_length(): """#2076: Exception raised by len(arg) should be propagated """ + class BadLen(RuntimeError): pass - class SequenceLike(): + class SequenceLike: def __getitem__(self, i): return None @@ -121,17 +124,17 @@ def __len__(self): def test_map_iterator(): - sm = m.StringMap({'hi': 'bye', 'black': 'white'}) - assert sm['hi'] == 'bye' + sm = m.StringMap({"hi": "bye", "black": "white"}) + assert sm["hi"] == "bye" assert len(sm) == 2 - assert sm['black'] == 'white' + assert sm["black"] == "white" with pytest.raises(KeyError): - assert sm['orange'] - sm['orange'] = 'banana' - assert sm['orange'] == 'banana' + assert sm["orange"] + sm["orange"] = "banana" + assert sm["orange"] == "banana" - expected = {'hi': 'bye', 'black': 'white', 'orange': 'banana'} + expected = {"hi": "bye", "black": "white", "orange": "banana"} for k in sm: assert sm[k] == expected[k] for k, v in sm.items(): @@ -179,7 +182,8 @@ def test_iterator_passthrough(): """#181: iterator passthrough did not compile""" from pybind11_tests.sequences_and_iterators import iterator_passthrough - assert list(iterator_passthrough(iter([3, 5, 7, 9, 11, 13, 15]))) == [3, 5, 7, 9, 11, 13, 15] + values = [3, 5, 7, 9, 11, 13, 15] + assert list(iterator_passthrough(iter(values))) == values def test_iterator_rvp(): diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp index afd5cc696e..157621932a 100644 --- a/tests/test_smart_ptr.cpp +++ b/tests/test_smart_ptr.cpp @@ -8,29 +8,14 @@ BSD-style license that can be found in the LICENSE file. */ -#if defined(_MSC_VER) && _MSC_VER < 1910 -# pragma warning(disable: 4702) // unreachable code in system header +#if defined(_MSC_VER) && _MSC_VER < 1910 // VS 2015's MSVC +# pragma warning(disable: 4702) // unreachable code in system header (xatomic.h(382)) #endif #include "pybind11_tests.h" #include "object.h" -// Make pybind aware of the ref-counted wrapper type (s): - -// ref is a wrapper for 'Object' which uses intrusive reference counting -// It is always possible to construct a ref from an Object* pointer without -// possible inconsistencies, hence the 'true' argument at the end. -PYBIND11_DECLARE_HOLDER_TYPE(T, ref, true); -// Make pybind11 aware of the non-standard getter member function -namespace pybind11 { namespace detail { - template - struct holder_helper> { - static const T* get(const ref &p) { return p.get_ptr(); } - }; -}} - -// The following is not required anymore for std::shared_ptr, but it should compile without error: -PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr); +namespace { // This is just a wrapper around unique_ptr, but with extra fields to deliberately bloat up the // holder size to trigger the non-simple-layout internal instance layout for single inheritance with @@ -40,10 +25,8 @@ template class huge_unique_ptr { uint64_t padding[10]; public: huge_unique_ptr(T *p) : ptr(p) {}; - T* get() const { return ptr.get(); } - T* release() { return ptr.release(); } + T *get() { return ptr.get(); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr); // Simple custom holder that works like unique_ptr template @@ -52,9 +35,8 @@ class custom_unique_ptr { public: custom_unique_ptr(T* p) : impl(p) { } T* get() const { return impl.get(); } - T* release() { return impl.release(); } + T* release_ptr() { return impl.release(); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr); // Simple custom holder that works like shared_ptr and has operator& overload // To obtain address of an instance of this holder pybind should use std::addressof @@ -68,7 +50,6 @@ class shared_ptr_with_addressof_operator { T* get() const { return impl.get(); } T** operator&() { throw std::logic_error("Call of overloaded operator& is not expected"); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator); // Simple custom holder that works like unique_ptr and has operator& overload // To obtain address of an instance of this holder pybind should use std::addressof @@ -83,10 +64,218 @@ class unique_ptr_with_addressof_operator { T* release_ptr() { return impl.release(); } T** operator&() { throw std::logic_error("Call of overloaded operator& is not expected"); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator); +// Custom object with builtin reference counting (see 'object.h' for the implementation) +class MyObject1 : public Object { +public: + MyObject1(int value) : value(value) { print_created(this, toString()); } + std::string toString() const override { return "MyObject1[" + std::to_string(value) + "]"; } +protected: + ~MyObject1() override { print_destroyed(this); } +private: + int value; +}; + +// Object managed by a std::shared_ptr<> +class MyObject2 { +public: + MyObject2(const MyObject2 &) = default; + MyObject2(int value) : value(value) { print_created(this, toString()); } + std::string toString() const { return "MyObject2[" + std::to_string(value) + "]"; } + virtual ~MyObject2() { print_destroyed(this); } +private: + int value; +}; + +// Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<> +class MyObject3 : public std::enable_shared_from_this { +public: + MyObject3(const MyObject3 &) = default; + MyObject3(int value) : value(value) { print_created(this, toString()); } + std::string toString() const { return "MyObject3[" + std::to_string(value) + "]"; } + virtual ~MyObject3() { print_destroyed(this); } +private: + int value; +}; + +// test_unique_nodelete +// Object with a private destructor +class MyObject4; +static std::unordered_set myobject4_instances; +class MyObject4 { +public: + MyObject4(int value) : value{value} { + print_created(this); + myobject4_instances.insert(this); + } + int value; + + static void cleanupAllInstances() { + auto tmp = std::move(myobject4_instances); + myobject4_instances.clear(); + for (auto o : tmp) + delete o; + } +private: + ~MyObject4() { + myobject4_instances.erase(this); + print_destroyed(this); + } +}; + +// test_unique_deleter +// Object with std::unique_ptr where D is not matching the base class +// Object with a protected destructor +class MyObject4a; +static std::unordered_set myobject4a_instances; +class MyObject4a { +public: + MyObject4a(int i) { + value = i; + print_created(this); + myobject4a_instances.insert(this); + }; + int value; + + static void cleanupAllInstances() { + auto tmp = std::move(myobject4a_instances); + myobject4a_instances.clear(); + for (auto o : tmp) + delete o; + } +protected: + virtual ~MyObject4a() { + myobject4a_instances.erase(this); + print_destroyed(this); + } +}; + +// Object derived but with public destructor and no Deleter in default holder +class MyObject4b : public MyObject4a { +public: + MyObject4b(int i) : MyObject4a(i) { print_created(this); } + ~MyObject4b() override { print_destroyed(this); } +}; + +// test_large_holder +class MyObject5 { // managed by huge_unique_ptr +public: + MyObject5(int value) : value{value} { print_created(this); } + ~MyObject5() { print_destroyed(this); } + int value; +}; + +// test_shared_ptr_and_references +struct SharedPtrRef { + struct A { + A() { print_created(this); } + A(const A &) { print_copy_created(this); } + A(A &&) { print_move_created(this); } + ~A() { print_destroyed(this); } + }; + + A value = {}; + std::shared_ptr shared = std::make_shared(); +}; + +// test_shared_ptr_from_this_and_references +struct SharedFromThisRef { + struct B : std::enable_shared_from_this { + B() { print_created(this); } + B(const B &) : std::enable_shared_from_this() { print_copy_created(this); } + B(B &&) : std::enable_shared_from_this() { print_move_created(this); } + ~B() { print_destroyed(this); } + }; + + B value = {}; + std::shared_ptr shared = std::make_shared(); +}; + +// Issue #865: shared_from_this doesn't work with virtual inheritance +struct SharedFromThisVBase : std::enable_shared_from_this { + SharedFromThisVBase() = default; + SharedFromThisVBase(const SharedFromThisVBase &) = default; + virtual ~SharedFromThisVBase() = default; +}; +struct SharedFromThisVirt : virtual SharedFromThisVBase {}; + +// test_move_only_holder +struct C { + C() { print_created(this); } + ~C() { print_destroyed(this); } +}; + +// test_holder_with_addressof_operator +struct TypeForHolderWithAddressOf { + TypeForHolderWithAddressOf() { print_created(this); } + TypeForHolderWithAddressOf(const TypeForHolderWithAddressOf &) { print_copy_created(this); } + TypeForHolderWithAddressOf(TypeForHolderWithAddressOf &&) { print_move_created(this); } + ~TypeForHolderWithAddressOf() { print_destroyed(this); } + std::string toString() const { + return "TypeForHolderWithAddressOf[" + std::to_string(value) + "]"; + } + int value = 42; +}; + +// test_move_only_holder_with_addressof_operator +struct TypeForMoveOnlyHolderWithAddressOf { + TypeForMoveOnlyHolderWithAddressOf(int value) : value{value} { print_created(this); } + ~TypeForMoveOnlyHolderWithAddressOf() { print_destroyed(this); } + std::string toString() const { + return "MoveOnlyHolderWithAddressOf[" + std::to_string(value) + "]"; + } + int value; +}; + +// test_smart_ptr_from_default +struct HeldByDefaultHolder { }; + +// test_shared_ptr_gc +// #187: issue involving std::shared_ptr<> return value policy & garbage collection +struct ElementBase { + virtual ~ElementBase() = default; /* Force creation of virtual table */ + ElementBase() = default; + ElementBase(const ElementBase&) = delete; +}; + +struct ElementA : ElementBase { + ElementA(int v) : v(v) { } + int value() { return v; } + int v; +}; + +struct ElementList { + void add(std::shared_ptr e) { l.push_back(e); } + std::vector> l; +}; + +} // namespace + +// ref is a wrapper for 'Object' which uses intrusive reference counting +// It is always possible to construct a ref from an Object* pointer without +// possible inconsistencies, hence the 'true' argument at the end. +// Make pybind11 aware of the non-standard getter member function +namespace pybind11 { namespace detail { + template + struct holder_helper> { + static const T *get(const ref &p) { return p.get_ptr(); } + }; +} // namespace detail +} // namespace pybind11 + +// Make pybind aware of the ref-counted wrapper type (s): +PYBIND11_DECLARE_HOLDER_TYPE(T, ref, true); +// The following is not required anymore for std::shared_ptr, but it should compile without error: +PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr); +PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr); +PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr); +PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator); +PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator); TEST_SUBMODULE(smart_ptr, m) { + // Please do not interleave `struct` and `class` definitions with bindings code, + // but implement `struct`s and `class`es in the anonymous namespace above. + // This helps keeping the smart_holder branch in sync with master. // test_smart_ptr @@ -94,16 +283,6 @@ TEST_SUBMODULE(smart_ptr, m) { py::class_> obj(m, "Object"); obj.def("getRefCount", &Object::getRefCount); - // Custom object with builtin reference counting (see 'object.h' for the implementation) - class MyObject1 : public Object { - public: - MyObject1(int value) : value(value) { print_created(this, toString()); } - std::string toString() const { return "MyObject1[" + std::to_string(value) + "]"; } - protected: - virtual ~MyObject1() { print_destroyed(this); } - private: - int value; - }; py::class_>(m, "MyObject1", obj) .def(py::init()); py::implicitly_convertible(); @@ -124,17 +303,6 @@ TEST_SUBMODULE(smart_ptr, m) { // Expose constructor stats for the ref type m.def("cstats_ref", &ConstructorStats::get); - - // Object managed by a std::shared_ptr<> - class MyObject2 { - public: - MyObject2(const MyObject2 &) = default; - MyObject2(int value) : value(value) { print_created(this, toString()); } - std::string toString() const { return "MyObject2[" + std::to_string(value) + "]"; } - virtual ~MyObject2() { print_destroyed(this); } - private: - int value; - }; py::class_>(m, "MyObject2") .def(py::init()); m.def("make_myobject2_1", []() { return new MyObject2(6); }); @@ -144,16 +312,6 @@ TEST_SUBMODULE(smart_ptr, m) { m.def("print_myobject2_3", [](const std::shared_ptr &obj) { py::print(obj->toString()); }); m.def("print_myobject2_4", [](const std::shared_ptr *obj) { py::print((*obj)->toString()); }); - // Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<> - class MyObject3 : public std::enable_shared_from_this { - public: - MyObject3(const MyObject3 &) = default; - MyObject3(int value) : value(value) { print_created(this, toString()); } - std::string toString() const { return "MyObject3[" + std::to_string(value) + "]"; } - virtual ~MyObject3() { print_destroyed(this); } - private: - int value; - }; py::class_>(m, "MyObject3") .def(py::init()); m.def("make_myobject3_1", []() { return new MyObject3(8); }); @@ -175,70 +333,29 @@ TEST_SUBMODULE(smart_ptr, m) { }); // test_unique_nodelete - // Object with a private destructor - class MyObject4 { - public: - MyObject4(int value) : value{value} { print_created(this); } - int value; - private: - ~MyObject4() { print_destroyed(this); } - }; py::class_>(m, "MyObject4") .def(py::init()) - .def_readwrite("value", &MyObject4::value); + .def_readwrite("value", &MyObject4::value) + .def_static("cleanup_all_instances", &MyObject4::cleanupAllInstances); // test_unique_deleter - // Object with std::unique_ptr where D is not matching the base class - // Object with a protected destructor - class MyObject4a { - public: - MyObject4a(int i) { - value = i; - print_created(this); - }; - int value; - protected: - virtual ~MyObject4a() { print_destroyed(this); } - }; py::class_>(m, "MyObject4a") .def(py::init()) - .def_readwrite("value", &MyObject4a::value); + .def_readwrite("value", &MyObject4a::value) + .def_static("cleanup_all_instances", &MyObject4a::cleanupAllInstances); - // Object derived but with public destructor and no Deleter in default holder - class MyObject4b : public MyObject4a { - public: - MyObject4b(int i) : MyObject4a(i) { print_created(this); } - ~MyObject4b() { print_destroyed(this); } - }; - py::class_(m, "MyObject4b") + py::class_>(m, "MyObject4b") .def(py::init()); // test_large_holder - class MyObject5 { // managed by huge_unique_ptr - public: - MyObject5(int value) : value{value} { print_created(this); } - ~MyObject5() { print_destroyed(this); } - int value; - }; py::class_>(m, "MyObject5") .def(py::init()) .def_readwrite("value", &MyObject5::value); // test_shared_ptr_and_references - struct SharedPtrRef { - struct A { - A() { print_created(this); } - A(const A &) { print_copy_created(this); } - A(A &&) { print_move_created(this); } - ~A() { print_destroyed(this); } - }; - - A value = {}; - std::shared_ptr shared = std::make_shared(); - }; using A = SharedPtrRef::A; py::class_>(m, "A"); - py::class_(m, "SharedPtrRef") + py::class_>(m, "SharedPtrRef") .def(py::init<>()) .def_readonly("ref", &SharedPtrRef::value) .def_property_readonly("copy", [](const SharedPtrRef &s) { return s.value; }, @@ -250,20 +367,9 @@ TEST_SUBMODULE(smart_ptr, m) { .def("set_holder", [](SharedPtrRef &, std::shared_ptr) { return true; }); // test_shared_ptr_from_this_and_references - struct SharedFromThisRef { - struct B : std::enable_shared_from_this { - B() { print_created(this); } - B(const B &) : std::enable_shared_from_this() { print_copy_created(this); } - B(B &&) : std::enable_shared_from_this() { print_move_created(this); } - ~B() { print_destroyed(this); } - }; - - B value = {}; - std::shared_ptr shared = std::make_shared(); - }; using B = SharedFromThisRef::B; py::class_>(m, "B"); - py::class_(m, "SharedFromThisRef") + py::class_>(m, "SharedFromThisRef") .def(py::init<>()) .def_readonly("bad_wp", &SharedFromThisRef::value) .def_property_readonly("ref", [](const SharedFromThisRef &s) -> const B & { return *s.shared; }) @@ -276,36 +382,16 @@ TEST_SUBMODULE(smart_ptr, m) { .def("set_holder", [](SharedFromThisRef &, std::shared_ptr) { return true; }); // Issue #865: shared_from_this doesn't work with virtual inheritance - struct SharedFromThisVBase : std::enable_shared_from_this { - SharedFromThisVBase() = default; - SharedFromThisVBase(const SharedFromThisVBase &) = default; - virtual ~SharedFromThisVBase() = default; - }; - struct SharedFromThisVirt : virtual SharedFromThisVBase {}; static std::shared_ptr sft(new SharedFromThisVirt()); py::class_>(m, "SharedFromThisVirt") .def_static("get", []() { return sft.get(); }); // test_move_only_holder - struct C { - C() { print_created(this); } - ~C() { print_destroyed(this); } - }; py::class_>(m, "TypeWithMoveOnlyHolder") .def_static("make", []() { return custom_unique_ptr(new C); }) .def_static("make_as_object", []() { return py::cast(custom_unique_ptr(new C)); }); // test_holder_with_addressof_operator - struct TypeForHolderWithAddressOf { - TypeForHolderWithAddressOf() { print_created(this); } - TypeForHolderWithAddressOf(const TypeForHolderWithAddressOf &) { print_copy_created(this); } - TypeForHolderWithAddressOf(TypeForHolderWithAddressOf &&) { print_move_created(this); } - ~TypeForHolderWithAddressOf() { print_destroyed(this); } - std::string toString() const { - return "TypeForHolderWithAddressOf[" + std::to_string(value) + "]"; - } - int value = 42; - }; using HolderWithAddressOf = shared_ptr_with_addressof_operator; py::class_(m, "TypeForHolderWithAddressOf") .def_static("make", []() { return HolderWithAddressOf(new TypeForHolderWithAddressOf); }) @@ -316,14 +402,6 @@ TEST_SUBMODULE(smart_ptr, m) { .def("print_object_4", [](const HolderWithAddressOf *obj) { py::print((*obj).get()->toString()); }); // test_move_only_holder_with_addressof_operator - struct TypeForMoveOnlyHolderWithAddressOf { - TypeForMoveOnlyHolderWithAddressOf(int value) : value{value} { print_created(this); } - ~TypeForMoveOnlyHolderWithAddressOf() { print_destroyed(this); } - std::string toString() const { - return "MoveOnlyHolderWithAddressOf[" + std::to_string(value) + "]"; - } - int value; - }; using MoveOnlyHolderWithAddressOf = unique_ptr_with_addressof_operator; py::class_(m, "TypeForMoveOnlyHolderWithAddressOf") .def_static("make", []() { return MoveOnlyHolderWithAddressOf(new TypeForMoveOnlyHolderWithAddressOf(0)); }) @@ -331,33 +409,18 @@ TEST_SUBMODULE(smart_ptr, m) { .def("print_object", [](const TypeForMoveOnlyHolderWithAddressOf *obj) { py::print(obj->toString()); }); // test_smart_ptr_from_default - struct HeldByDefaultHolder { }; - py::class_(m, "HeldByDefaultHolder") + py::class_>(m, "HeldByDefaultHolder") .def(py::init<>()) .def_static("load_shared_ptr", [](std::shared_ptr) {}); // test_shared_ptr_gc // #187: issue involving std::shared_ptr<> return value policy & garbage collection - struct ElementBase { - virtual ~ElementBase() { } /* Force creation of virtual table */ - ElementBase() = default; - ElementBase(const ElementBase&) = delete; - }; py::class_>(m, "ElementBase"); - struct ElementA : ElementBase { - ElementA(int v) : v(v) { } - int value() { return v; } - int v; - }; py::class_>(m, "ElementA") .def(py::init()) .def("value", &ElementA::value); - struct ElementList { - void add(std::shared_ptr e) { l.push_back(e); } - std::vector> l; - }; py::class_>(m, "ElementList") .def(py::init<>()) .def("add", &ElementList::add) diff --git a/tests/test_smart_ptr.py b/tests/test_smart_ptr.py index 8c97e5ab90..ce07ecf952 100644 --- a/tests/test_smart_ptr.py +++ b/tests/test_smart_ptr.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- import pytest -from pybind11_tests import smart_ptr as m -from pybind11_tests import ConstructorStats + +m = pytest.importorskip("pybind11_tests.smart_ptr") +from pybind11_tests import ConstructorStats # noqa: E402 def test_smart_ptr(capture): # Object1 - for i, o in enumerate([m.make_object_1(), m.make_object_2(), m.MyObject1(3)], start=1): + for i, o in enumerate( + [m.make_object_1(), m.make_object_2(), m.MyObject1(3)], start=1 + ): assert o.getRefCount() == 1 with capture: m.print_object_1(o) @@ -15,8 +18,9 @@ def test_smart_ptr(capture): m.print_object_4(o) assert capture == "MyObject1[{i}]\n".format(i=i) * 4 - for i, o in enumerate([m.make_myobject1_1(), m.make_myobject1_2(), m.MyObject1(6), 7], - start=4): + for i, o in enumerate( + [m.make_myobject1_1(), m.make_myobject1_2(), m.MyObject1(6), 7], start=4 + ): print(o) with capture: if not isinstance(o, int): @@ -28,11 +32,15 @@ def test_smart_ptr(capture): m.print_myobject1_2(o) m.print_myobject1_3(o) m.print_myobject1_4(o) - assert capture == "MyObject1[{i}]\n".format(i=i) * (4 if isinstance(o, int) else 8) + + times = 4 if isinstance(o, int) else 8 + assert capture == "MyObject1[{i}]\n".format(i=i) * times cstats = ConstructorStats.get(m.MyObject1) assert cstats.alive() == 0 - expected_values = ['MyObject1[{}]'.format(i) for i in range(1, 7)] + ['MyObject1[7]'] * 4 + expected_values = ["MyObject1[{}]".format(i) for i in range(1, 7)] + [ + "MyObject1[7]" + ] * 4 assert cstats.values() == expected_values assert cstats.default_constructions == 0 assert cstats.copy_constructions == 0 @@ -41,7 +49,9 @@ def test_smart_ptr(capture): assert cstats.move_assignments == 0 # Object2 - for i, o in zip([8, 6, 7], [m.MyObject2(8), m.make_myobject2_1(), m.make_myobject2_2()]): + for i, o in zip( + [8, 6, 7], [m.MyObject2(8), m.make_myobject2_1(), m.make_myobject2_2()] + ): print(o) with capture: m.print_myobject2_1(o) @@ -54,7 +64,7 @@ def test_smart_ptr(capture): assert cstats.alive() == 1 o = None assert cstats.alive() == 0 - assert cstats.values() == ['MyObject2[8]', 'MyObject2[6]', 'MyObject2[7]'] + assert cstats.values() == ["MyObject2[8]", "MyObject2[6]", "MyObject2[7]"] assert cstats.default_constructions == 0 assert cstats.copy_constructions == 0 # assert cstats.move_constructions >= 0 # Doesn't invoke any @@ -62,7 +72,9 @@ def test_smart_ptr(capture): assert cstats.move_assignments == 0 # Object3 - for i, o in zip([9, 8, 9], [m.MyObject3(9), m.make_myobject3_1(), m.make_myobject3_2()]): + for i, o in zip( + [9, 8, 9], [m.MyObject3(9), m.make_myobject3_1(), m.make_myobject3_2()] + ): print(o) with capture: m.print_myobject3_1(o) @@ -75,7 +87,7 @@ def test_smart_ptr(capture): assert cstats.alive() == 1 o = None assert cstats.alive() == 0 - assert cstats.values() == ['MyObject3[9]', 'MyObject3[8]', 'MyObject3[9]'] + assert cstats.values() == ["MyObject3[9]", "MyObject3[8]", "MyObject3[9]"] assert cstats.default_constructions == 0 assert cstats.copy_constructions == 0 # assert cstats.move_constructions >= 0 # Doesn't invoke any @@ -95,7 +107,7 @@ def test_smart_ptr(capture): # ref<> cstats = m.cstats_ref() assert cstats.alive() == 0 - assert cstats.values() == ['from pointer'] * 10 + assert cstats.values() == ["from pointer"] * 10 assert cstats.default_constructions == 30 assert cstats.copy_constructions == 12 # assert cstats.move_constructions >= 0 # Doesn't invoke any @@ -113,7 +125,9 @@ def test_unique_nodelete(): cstats = ConstructorStats.get(m.MyObject4) assert cstats.alive() == 1 del o - assert cstats.alive() == 1 # Leak, but that's intentional + assert cstats.alive() == 1 + m.MyObject4.cleanup_all_instances() + assert cstats.alive() == 0 def test_unique_nodelete4a(): @@ -122,19 +136,25 @@ def test_unique_nodelete4a(): cstats = ConstructorStats.get(m.MyObject4a) assert cstats.alive() == 1 del o - assert cstats.alive() == 1 # Leak, but that's intentional + assert cstats.alive() == 1 + m.MyObject4a.cleanup_all_instances() + assert cstats.alive() == 0 def test_unique_deleter(): + m.MyObject4a(0) o = m.MyObject4b(23) assert o.value == 23 cstats4a = ConstructorStats.get(m.MyObject4a) - assert cstats4a.alive() == 2 # Two because of previous test + assert cstats4a.alive() == 2 cstats4b = ConstructorStats.get(m.MyObject4b) assert cstats4b.alive() == 1 del o - assert cstats4a.alive() == 1 # Should now only be one leftover from previous test + assert cstats4a.alive() == 1 # Should now only be one leftover assert cstats4b.alive() == 0 # Should be deleted + m.MyObject4a.cleanup_all_instances() + assert cstats4a.alive() == 0 + assert cstats4b.alive() == 0 def test_large_holder(): @@ -185,7 +205,9 @@ def test_shared_ptr_from_this_and_references(): ref = s.ref # init_holder_helper(holder_ptr=false, owned=false, bad_wp=false) assert stats.alive() == 2 assert s.set_ref(ref) - assert s.set_holder(ref) # std::enable_shared_from_this can create a holder from a reference + assert s.set_holder( + ref + ) # std::enable_shared_from_this can create a holder from a reference bad_wp = s.bad_wp # init_holder_helper(holder_ptr=false, owned=false, bad_wp=true) assert stats.alive() == 2 @@ -199,12 +221,16 @@ def test_shared_ptr_from_this_and_references(): assert s.set_ref(copy) assert s.set_holder(copy) - holder_ref = s.holder_ref # init_holder_helper(holder_ptr=true, owned=false, bad_wp=false) + holder_ref = ( + s.holder_ref + ) # init_holder_helper(holder_ptr=true, owned=false, bad_wp=false) assert stats.alive() == 3 assert s.set_ref(holder_ref) assert s.set_holder(holder_ref) - holder_copy = s.holder_copy # init_holder_helper(holder_ptr=true, owned=true, bad_wp=false) + holder_copy = ( + s.holder_copy + ) # init_holder_helper(holder_ptr=true, owned=true, bad_wp=false) assert stats.alive() == 3 assert s.set_ref(holder_copy) assert s.set_holder(holder_copy) @@ -276,8 +302,10 @@ def test_smart_ptr_from_default(): instance = m.HeldByDefaultHolder() with pytest.raises(RuntimeError) as excinfo: m.HeldByDefaultHolder.load_shared_ptr(instance) - assert "Unable to load a custom holder type from a " \ - "default-holder instance" in str(excinfo.value) + assert ( + "Unable to load a custom holder type from a " + "default-holder instance" in str(excinfo.value) + ) def test_shared_ptr_gc(): @@ -343,7 +371,6 @@ def test_unique_ptr_overload_fail(): def test_unique_ptr_held_container_from_cpp(): - def check_reset(obj_new): c = m.UniquePtrHeldContainer() obj = c.get() diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 928635788e..0590162770 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -15,7 +15,7 @@ #include // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14 -#if PYBIND11_HAS_VARIANT +#if defined(PYBIND11_HAS_VARIANT) using std::variant; #elif defined(PYBIND11_TEST_BOOST) && (!defined(_MSC_VER) || _MSC_VER >= 1910) # include @@ -47,7 +47,7 @@ struct TplCtorClass { namespace std { template <> struct hash { size_t operator()(const TplCtorClass &) const { return 0; } }; -} +} // namespace std template