From edcae5e6944bcb8d48e0c87e1ed15c0a0853b76c Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:11:53 +0100 Subject: [PATCH 01/12] Run tests via a reuseable custom Action. Let requirements.test.txt define Pytest version etc. (remove repeated installation). --- .github/actions/test/action.yml | 73 +++++++++++++++++++++++++++++++++ .github/workflows/build.yml | 32 ++++++--------- 2 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 .github/actions/test/action.yml diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000..ebb7f437 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,73 @@ +name: + Test + +description: + Run pytest, and run the doctest runner (shapefile.py as a script). + +inputs: + python_version: + required: false + default: "3.13" + description: 'The Python version to run the tests on. ' + + +runs: + using: "composite" + steps: + # The Repo is required to already be checked out, e.g. by the calling workflow + - uses: LizardByte/setup-python-action@master + with: + python-version: ${{ inputs.python_version }} + + - name: Doctests + run: python shapefile.py + + - name: Install test dependencies. + run: | + python -m pip install --upgrade pip + pip install -r requirements.test.txt + + - name: Pytest + run: | + pytest + + - name: Show versions. + shell: ${{ inputs.shell }} + working-directory: '.\sDNA\sdna_vs2008\tests\pytest' + run: | + python --version + python -c "import numpy; print(numpy.__version__)" + python -m pytest --version + + + # - name: Run debug_test.py directly. + # shell: cmd + # working-directory: '.\sDNA\sdna_vs2008\tests' + # # run: python -u debug_test.py + # run: set "sdnadll=c:\Program Files (x86)\sDNA\x64\sdna_vs2008.dll" & python -u debug_test.py + + # - name: Run diff test number 0 directly via + # test_gathering_windows_command_line_tests.py, by test index + # working-directory: '.\sDNA\sdna_vs2008\tests\pytest' + # run: python test_gathering_windows_command_line_tests.py 0 + + - name: Run all the diff tests with Pytest. + shell: ${{ inputs.shell }} + working-directory: .\sDNA\sdna_vs2008\tests\pytest + env: + DONT_TEST_N_LINK_SUBSYSTEMS_ORDER: ${{ inputs.DONT_TEST_N_LINK_SUBSYSTEMS_ORDER }} + ALLOW_NEGATIVE_FORMULA_ERROR_ON_ANY_LINK_PRESENT: ${{ inputs.ALLOW_NEGATIVE_FORMULA_ERROR_ON_ANY_LINK_PRESENT }} + sdnadll: ${{ inputs.sdnadll }} + sdna_bin_dir: ${{ inputs.sdna_bin_dir }} + sdna_debug: ${{ inputs.sdna_debug }} + run: pytest -rA + # -rA shows summary of tests, one per line, even when they all pass. + # + # To run the same tests locally, the command is something like: + # run: > + # set "sdnadll=c:\Program Files (x86)\sDNA\x64\sdna_vs2008.dll" & + # set "sdna_bin_dir=c:\Program Files (x86)\sDNA\bin" & + # set sdna_debug=False & + # set DONT_TEST_N_LINK_SUBSYSTEMS_ORDER=1 & + # set ALLOW_NEGATIVE_FORMULA_ERROR_ON_ANY_LINK_PRESENT=1 & + # pytest -rA \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b9f58955..9908e619 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will run the pre-commit hooks (including linters), and the tests with a variety of Python versions name: build @@ -16,8 +15,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - uses: pre-commit/action@v3.0.1 - test: + test: strategy: fail-fast: false matrix: @@ -33,21 +32,16 @@ jobs: "3.12", "3.13.0a2", ] - - runs-on: ubuntu-latest - container: - image: python:${{ matrix.python-version }}-slim - + os: [ + "ubuntu-latest", + "windows-latest", + "macos-latest", + ] + runs-on: matrix.os steps: - uses: actions/checkout@v4 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install pytest - if [ -f requirements.test.txt ]; then pip install -r requirements.test.txt; fi - - name: Test with doctest - run: | - python shapefile.py - - name: Test with pytest - run: | - pytest + + - name: Run tests + uses: ./.github/actions/test + with: + python_version: ${{ matrix.python_version }} From 18354e237d1376280a04d4c911112fd9cafda5a8 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:26:31 +0100 Subject: [PATCH 02/12] Don't test all Python versions on Windows and MacOS due to Github hosted runner limits. --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9908e619..8cdf134c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,9 +34,13 @@ jobs: ] os: [ "ubuntu-latest", - "windows-latest", - "macos-latest", ] + include: + - os: "windows-latest" + python-version: "3.13" + - os: "macos-latest" + python-version: "3.13" + runs-on: matrix.os steps: - uses: actions/checkout@v4 From de4f6afef6f61557c627749fbb9b781576de2528 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:28:24 +0100 Subject: [PATCH 03/12] Don't test on Windows or MacOS at all yet. --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8cdf134c..3538aa94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,11 +35,11 @@ jobs: os: [ "ubuntu-latest", ] - include: - - os: "windows-latest" - python-version: "3.13" - - os: "macos-latest" - python-version: "3.13" + # include: + # - os: "windows-latest" + # python-version: "3.13" + # - os: "macos-latest" + # python-version: "3.13" runs-on: matrix.os steps: From a79a6125b2c6ef99dcf4c58cba8c1b1c040ee1a4 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:42:32 +0100 Subject: [PATCH 04/12] Run Python version tests in containers. --- .github/workflows/build.yml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3538aa94..16f11e36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/setup-python@v5 - uses: pre-commit/action@v3.0.1 - test: + test_in_slim_Python_containers: strategy: fail-fast: false matrix: @@ -32,15 +32,40 @@ jobs: "3.12", "3.13.0a2", ] + + runs-on: ubuntu-latest + container: + image: python:${{ matrix.python-version }}-slim + + steps: + - uses: actions/checkout@v4 + + - name: Run tests + uses: ./.github/actions/test + with: + python_version: ${{ matrix.python_version }} + + + test_on_MacOS_and_Windows: + + strategy: + fail-fast: false + matrix: + python-version: [ + "3.12", + ] os: [ - "ubuntu-latest", + "macos-latest", + "windows-latest", ] + # include: # - os: "windows-latest" # python-version: "3.13" # - os: "macos-latest" # python-version: "3.13" + runs-on: matrix.os steps: - uses: actions/checkout@v4 From a1e7642bdf6ec5a202179784ee1b33c1f3c04691 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:44:40 +0100 Subject: [PATCH 05/12] Update action.yml --- .github/actions/test/action.yml | 37 +-------------------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index ebb7f437..ec03ac84 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -31,43 +31,8 @@ runs: run: | pytest - - name: Show versions. - shell: ${{ inputs.shell }} - working-directory: '.\sDNA\sdna_vs2008\tests\pytest' + - name: Show versions for logs. run: | python --version - python -c "import numpy; print(numpy.__version__)" python -m pytest --version - - # - name: Run debug_test.py directly. - # shell: cmd - # working-directory: '.\sDNA\sdna_vs2008\tests' - # # run: python -u debug_test.py - # run: set "sdnadll=c:\Program Files (x86)\sDNA\x64\sdna_vs2008.dll" & python -u debug_test.py - - # - name: Run diff test number 0 directly via - # test_gathering_windows_command_line_tests.py, by test index - # working-directory: '.\sDNA\sdna_vs2008\tests\pytest' - # run: python test_gathering_windows_command_line_tests.py 0 - - - name: Run all the diff tests with Pytest. - shell: ${{ inputs.shell }} - working-directory: .\sDNA\sdna_vs2008\tests\pytest - env: - DONT_TEST_N_LINK_SUBSYSTEMS_ORDER: ${{ inputs.DONT_TEST_N_LINK_SUBSYSTEMS_ORDER }} - ALLOW_NEGATIVE_FORMULA_ERROR_ON_ANY_LINK_PRESENT: ${{ inputs.ALLOW_NEGATIVE_FORMULA_ERROR_ON_ANY_LINK_PRESENT }} - sdnadll: ${{ inputs.sdnadll }} - sdna_bin_dir: ${{ inputs.sdna_bin_dir }} - sdna_debug: ${{ inputs.sdna_debug }} - run: pytest -rA - # -rA shows summary of tests, one per line, even when they all pass. - # - # To run the same tests locally, the command is something like: - # run: > - # set "sdnadll=c:\Program Files (x86)\sDNA\x64\sdna_vs2008.dll" & - # set "sdna_bin_dir=c:\Program Files (x86)\sDNA\bin" & - # set sdna_debug=False & - # set DONT_TEST_N_LINK_SUBSYSTEMS_ORDER=1 & - # set ALLOW_NEGATIVE_FORMULA_ERROR_ON_ANY_LINK_PRESENT=1 & - # pytest -rA \ No newline at end of file From 67206790b2598680ec338757df049aabeeecd9eb Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:01:32 +0100 Subject: [PATCH 06/12] Drop action input varibles. Use official setup-python action. --- .github/actions/test/action.yml | 9 +-------- .github/workflows/build.yml | 7 +++---- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index ec03ac84..92d314ce 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -4,20 +4,13 @@ name: description: Run pytest, and run the doctest runner (shapefile.py as a script). -inputs: - python_version: - required: false - default: "3.13" - description: 'The Python version to run the tests on. ' runs: using: "composite" steps: # The Repo is required to already be checked out, e.g. by the calling workflow - - uses: LizardByte/setup-python-action@master - with: - python-version: ${{ inputs.python_version }} + # The Python to be tested with is required to already be setup, with "python" and "pip" on the system Path - name: Doctests run: python shapefile.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16f11e36..231dbcd1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,8 +42,6 @@ jobs: - name: Run tests uses: ./.github/actions/test - with: - python_version: ${{ matrix.python_version }} test_on_MacOS_and_Windows: @@ -70,7 +68,8 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} - name: Run tests uses: ./.github/actions/test - with: - python_version: ${{ matrix.python_version }} From 6b62dc6271b7c7bd18f00714da3fdceeed9da727 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:13:01 +0100 Subject: [PATCH 07/12] Delete action to clear bad ref in cache --- .github/actions/test/action.yml | 31 --------------------------- .github/workflows/build.yml | 38 +++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 35 deletions(-) delete mode 100644 .github/actions/test/action.yml diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml deleted file mode 100644 index 92d314ce..00000000 --- a/.github/actions/test/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: - Test - -description: - Run pytest, and run the doctest runner (shapefile.py as a script). - - - -runs: - using: "composite" - steps: - # The Repo is required to already be checked out, e.g. by the calling workflow - # The Python to be tested with is required to already be setup, with "python" and "pip" on the system Path - - - name: Doctests - run: python shapefile.py - - - name: Install test dependencies. - run: | - python -m pip install --upgrade pip - pip install -r requirements.test.txt - - - name: Pytest - run: | - pytest - - - name: Show versions for logs. - run: | - python --version - python -m pytest --version - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 231dbcd1..d29da30a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,8 +40,23 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Run tests - uses: ./.github/actions/test + + - name: Doctests + run: python shapefile.py + + - name: Install test dependencies. + run: | + python -m pip install --upgrade pip + pip install -r requirements.test.txt + + - name: Pytest + run: | + pytest + + - name: Show versions for logs. + run: | + python --version + python -m pytest --version test_on_MacOS_and_Windows: @@ -71,5 +86,20 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} - - name: Run tests - uses: ./.github/actions/test + + - name: Doctests + run: python shapefile.py + + - name: Install test dependencies. + run: | + python -m pip install --upgrade pip + pip install -r requirements.test.txt + + - name: Pytest + run: | + pytest + + - name: Show versions for logs. + run: | + python --version + python -m pytest --version From 658f30ca7bc56f4740ce511976a681753e26914e Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:20:12 +0100 Subject: [PATCH 08/12] Re-use and re-instate the Action file --- .github/workflows/actions/test/action.yml | 29 ++++++++++++++++ .github/workflows/build.yml | 40 +++-------------------- 2 files changed, 33 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/actions/test/action.yml diff --git a/.github/workflows/actions/test/action.yml b/.github/workflows/actions/test/action.yml new file mode 100644 index 00000000..d8883bc5 --- /dev/null +++ b/.github/workflows/actions/test/action.yml @@ -0,0 +1,29 @@ +name: + Test + +description: + Run pytest, and run the doctest runner (shapefile.py as a script). + +runs: + using: "composite" + steps: + # The Repo is required to already be checked out, e.g. by the calling workflow + + # The Python to be tested with is required to already be setup, with "python" and "pip" on the system Path + + - name: Doctests + run: python shapefile.py + + - name: Install test dependencies. + run: | + python -m pip install --upgrade pip + pip install -r requirements.test.txt + + - name: Pytest + run: | + pytest + + - name: Show versions for logs. + run: | + python --version + python -m pytest --version \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d29da30a..0d2df3f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,27 +40,11 @@ jobs: steps: - uses: actions/checkout@v4 - - - name: Doctests - run: python shapefile.py - - - name: Install test dependencies. - run: | - python -m pip install --upgrade pip - pip install -r requirements.test.txt - - - name: Pytest - run: | - pytest - - - name: Show versions for logs. - run: | - python --version - python -m pytest --version + - name: Run tests + uses: ./.github/actions/test test_on_MacOS_and_Windows: - strategy: fail-fast: false matrix: @@ -71,14 +55,12 @@ jobs: "macos-latest", "windows-latest", ] - # include: # - os: "windows-latest" # python-version: "3.13" # - os: "macos-latest" # python-version: "3.13" - runs-on: matrix.os steps: - uses: actions/checkout@v4 @@ -87,19 +69,5 @@ jobs: with: python-version: ${{ inputs.python_version }} - - name: Doctests - run: python shapefile.py - - - name: Install test dependencies. - run: | - python -m pip install --upgrade pip - pip install -r requirements.test.txt - - - name: Pytest - run: | - pytest - - - name: Show versions for logs. - run: | - python --version - python -m pytest --version + - name: Run tests + uses: ./.github/actions/test From 7b56069e47f69cc48b261e232b25bc1af5248c62 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:22:18 +0100 Subject: [PATCH 09/12] Move Action file to correct location --- .github/{workflows => }/actions/test/action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => }/actions/test/action.yml (100%) diff --git a/.github/workflows/actions/test/action.yml b/.github/actions/test/action.yml similarity index 100% rename from .github/workflows/actions/test/action.yml rename to .github/actions/test/action.yml From 4b8821b586ccdaf7207947a6f890ae860c88b135 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:24:46 +0100 Subject: [PATCH 10/12] Specify bash shell --- .github/actions/test/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index d8883bc5..89a622fa 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -15,15 +15,18 @@ runs: run: python shapefile.py - name: Install test dependencies. + shell: bash run: | python -m pip install --upgrade pip pip install -r requirements.test.txt - name: Pytest + shell: bash run: | pytest - name: Show versions for logs. + shell: bash run: | python --version python -m pytest --version \ No newline at end of file From 69c54cb59bf21ed78bd59d50195c1b41176d32b5 Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:25:45 +0100 Subject: [PATCH 11/12] Specify Bash everywhere --- .github/actions/test/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 89a622fa..86ec93f3 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -12,6 +12,7 @@ runs: # The Python to be tested with is required to already be setup, with "python" and "pip" on the system Path - name: Doctests + shell: bash run: python shapefile.py - name: Install test dependencies. From 8c9bd567ebbe23c27792157a562ac757cb993e5a Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:42:07 +0100 Subject: [PATCH 12/12] Rename build.yml, as it doesn't build anything. Only test on Windows and MacOS in parent repo, due to limited Github hosted runner availability for forks. --- .github/workflows/deploy.yml | 11 +++++++++++ .../workflows/{build.yml => run_tests_and_hooks.yml} | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) rename .github/workflows/{build.yml => run_tests_and_hooks.yml} (93%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 70db5f72..057d8179 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,13 +23,24 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.x' + + # In general tests should be run after building a distribution, to test that distribution. + # However as long as PyShp is a pure Python library, with pure Python deps (or no deps) + # then it's not crucial. + + # Prevent deployment of releases that fail any hooks (e.g. linting) or that fail any tests. + - name: Run tests and hooks + uses: ./.github/workflows/run_tests_and_hooks.yml + - name: Install dependencies run: | python -m pip install --upgrade pip pip install build - name: Build package run: python -m build + - name: Publish package + if: github.repository == 'GeospatialPython/pyshp' uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 with: user: __token__ diff --git a/.github/workflows/build.yml b/.github/workflows/run_tests_and_hooks.yml similarity index 93% rename from .github/workflows/build.yml rename to .github/workflows/run_tests_and_hooks.yml index 0d2df3f2..438813d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/run_tests_and_hooks.yml @@ -1,11 +1,12 @@ # This workflow will run the pre-commit hooks (including linters), and the tests with a variety of Python versions -name: build +name: Run pre-commit hooks and tests on: push: pull_request: branches: [ master ] + workflow_call: workflow_dispatch: jobs: @@ -45,6 +46,7 @@ jobs: test_on_MacOS_and_Windows: + if: github.repository == 'GeospatialPython/pyshp' strategy: fail-fast: false matrix: