From 91cd4e5682a6290d15f6d57ee4cc75fc0c4af8c9 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 20:10:49 +1000 Subject: [PATCH 01/13] moved download of task template into separate job to avoid issues within tests --- .github/workflows/ci-cd.yml | 27 +++++++++++++++++++++- conftest.py | 8 +++++++ nipype2pydra/pkg_gen/tests/test_pkg_gen.py | 5 ++-- nipype2pydra/tests/test_package.py | 5 ++-- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b721f1d0..75edce94 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -13,6 +13,23 @@ defaults: shell: bash jobs: + + download-task-template: + runs-on: ubuntu-latest + steps: + + - name: Download task template + run: | + LATEST=$(curl -s https://api.github.com/repos/nipype/pydra-tasks-template/releases/latest | grep tag_name | awk -F\" '{print $4}') + curl -Lo pydra-task-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz + tar xzf pydra-task-template.tar.gz + mv pydra-tasks-template-* pydra-tasks-template + + - uses: actions/upload-artifact@v4 + with: + name: tasks-template + path: pydra-tasks-template + test: strategy: matrix: @@ -28,6 +45,12 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Download the pydra-tasks-template artefact + uses: actions/download-artifact@v4 + with: + name: pydra-tasks-template + path: $HOME/pydra-tasks-template + - name: Unset header # checkout@v2 adds a header that makes branch protection report errors # because the Github action bot is not a collaborator on the repo @@ -64,7 +87,9 @@ jobs: run: python3 -m pip install --break-system-packages .[test] - name: Pytest - run: pytest -vvs --cov nipype2pydra --cov-config .coveragerc --cov-report xml + run: >- + NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=$HOME/pydra-tasks-template + pytest -vvs --cov nipype2pydra --cov-config .coveragerc --cov-report xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 diff --git a/conftest.py b/conftest.py index cce075c8..101906ab 100644 --- a/conftest.py +++ b/conftest.py @@ -40,6 +40,14 @@ def invoke(*args, catch_exceptions=catch_cli_exceptions, **kwargs): return invoke +@pytest.fixture +def tasks_template_args(): + template = os.environ.get("NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE", None) + if template: + return ["--task-template", template] + return [] + + # For debugging in IDE's don't catch raised exceptions and let the IDE # break at it if os.getenv("_PYTEST_RAISE", "0") != "0": diff --git a/nipype2pydra/pkg_gen/tests/test_pkg_gen.py b/nipype2pydra/pkg_gen/tests/test_pkg_gen.py index 21115f5c..83a0c16a 100644 --- a/nipype2pydra/pkg_gen/tests/test_pkg_gen.py +++ b/nipype2pydra/pkg_gen/tests/test_pkg_gen.py @@ -15,7 +15,7 @@ def pkg_gen_spec_file(request): return EXAMPLE_PKG_GEN_DIR.joinpath(*request.param.split("-")).with_suffix(".yaml") -def test_pkg_gen(pkg_gen_spec_file, cli_runner, tmp_path): +def test_pkg_gen(pkg_gen_spec_file, cli_runner, tmp_path, tasks_template_args): outputs_dir = tmp_path / "output-dir" outputs_dir.mkdir() @@ -24,6 +24,7 @@ def test_pkg_gen(pkg_gen_spec_file, cli_runner, tmp_path): [ str(pkg_gen_spec_file), str(outputs_dir), - ], + ] + + tasks_template_args, ) assert result.exit_code == 0, show_cli_trace(result) diff --git a/nipype2pydra/tests/test_package.py b/nipype2pydra/tests/test_package.py index be625c2a..1b8865b2 100644 --- a/nipype2pydra/tests/test_package.py +++ b/nipype2pydra/tests/test_package.py @@ -33,7 +33,7 @@ def package_spec(request): @pytest.mark.xfail(reason="Fails due to missing dependencies on PyPI") -def test_package_complete(package_spec, cli_runner, tmp_path): +def test_package_complete(package_spec, cli_runner, tmp_path, tasks_template_args): pkg_name = package_spec.stem repo_output = tmp_path / "repo" repo_output.mkdir() @@ -43,7 +43,8 @@ def test_package_complete(package_spec, cli_runner, tmp_path): [ str(package_spec), str(repo_output), - ], + ] + + tasks_template_args, ) assert result.exit_code == 0, show_cli_trace(result) pkg_root = repo_output / f"pydra-{pkg_name}" From 1ebc139cea03287e4fcf381315cb468c743f90c1 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 20:19:20 +1000 Subject: [PATCH 02/13] fixed ci-cd --- .github/workflows/ci-cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 75edce94..47869d30 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -31,6 +31,7 @@ jobs: path: pydra-tasks-template test: + needs: [download-task-template] strategy: matrix: os: [macos-latest, ubuntu-latest] From ad8f6584d4205d80b7ff5faabd59ccb51a04911a Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 20:22:47 +1000 Subject: [PATCH 03/13] debugging ci-cd --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 47869d30..30f0283f 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -49,7 +49,7 @@ jobs: - name: Download the pydra-tasks-template artefact uses: actions/download-artifact@v4 with: - name: pydra-tasks-template + name: tasks-template path: $HOME/pydra-tasks-template - name: Unset header From 5a17c33cb89fdeac21614c46a0981ce4f6ee29ca Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 21:49:30 +1000 Subject: [PATCH 04/13] bugfixing github --- .github/workflows/ci-cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 30f0283f..84ea7d5a 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -23,12 +23,12 @@ jobs: LATEST=$(curl -s https://api.github.com/repos/nipype/pydra-tasks-template/releases/latest | grep tag_name | awk -F\" '{print $4}') curl -Lo pydra-task-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz tar xzf pydra-task-template.tar.gz - mv pydra-tasks-template-* pydra-tasks-template + mv pydra-tasks-template-* $HOME/pydra-tasks-template - uses: actions/upload-artifact@v4 with: name: tasks-template - path: pydra-tasks-template + path: $HOME/pydra-tasks-template test: needs: [download-task-template] From 740c333dc1a83c16a98e8e8161659e15f39cc686 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 21:55:12 +1000 Subject: [PATCH 05/13] debugging gha --- .github/workflows/ci-cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 84ea7d5a..a5a7c4c5 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -23,6 +23,7 @@ jobs: LATEST=$(curl -s https://api.github.com/repos/nipype/pydra-tasks-template/releases/latest | grep tag_name | awk -F\" '{print $4}') curl -Lo pydra-task-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz tar xzf pydra-task-template.tar.gz + ls mv pydra-tasks-template-* $HOME/pydra-tasks-template - uses: actions/upload-artifact@v4 From bbf5373260a62d49dd630074df0ee1b4f7430674 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 21:58:05 +1000 Subject: [PATCH 06/13] gha debugging --- .github/workflows/ci-cd.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a5a7c4c5..a4751b5e 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -21,10 +21,11 @@ jobs: - name: Download task template run: | LATEST=$(curl -s https://api.github.com/repos/nipype/pydra-tasks-template/releases/latest | grep tag_name | awk -F\" '{print $4}') - curl -Lo pydra-task-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz - tar xzf pydra-task-template.tar.gz + curl -Lo pydra-tasks-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz + tar xzf pydra-tasks-template.tar.gz ls - mv pydra-tasks-template-* $HOME/pydra-tasks-template + mv pydra-tasks-template-$LATEST $HOME/pydra-tasks-template + ls $HOME - uses: actions/upload-artifact@v4 with: From bfcf0cfc3437a3bfcbb70f41367f4716c7058550 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 21:59:20 +1000 Subject: [PATCH 07/13] gha action debug --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a4751b5e..7222eb9c 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -24,7 +24,7 @@ jobs: curl -Lo pydra-tasks-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz tar xzf pydra-tasks-template.tar.gz ls - mv pydra-tasks-template-$LATEST $HOME/pydra-tasks-template + mv pydra-tasks-template-${LATEST#v} $HOME/pydra-tasks-template ls $HOME - uses: actions/upload-artifact@v4 From 8e1c0d11d5d1fcc3809b0922ebfa5517e81e53bf Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 22:02:53 +1000 Subject: [PATCH 08/13] task template debugging --- .github/workflows/ci-cd.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 7222eb9c..74c5b6db 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -23,14 +23,15 @@ jobs: LATEST=$(curl -s https://api.github.com/repos/nipype/pydra-tasks-template/releases/latest | grep tag_name | awk -F\" '{print $4}') curl -Lo pydra-tasks-template.tar.gz https://github.com/nipype/pydra-tasks-template/archive/refs/tags/$LATEST.tar.gz tar xzf pydra-tasks-template.tar.gz - ls - mv pydra-tasks-template-${LATEST#v} $HOME/pydra-tasks-template - ls $HOME + rm pydra-tasks-template.tar.gz + mv pydra-tasks-template-${LATEST#v} pydra-tasks-template + tar czf $HOME/pydra-tasks-template.tar.gz pydra-tasks-template + - uses: actions/upload-artifact@v4 with: name: tasks-template - path: $HOME/pydra-tasks-template + path: $HOME/pydra-tasks-template.tar.gz test: needs: [download-task-template] @@ -52,7 +53,13 @@ jobs: uses: actions/download-artifact@v4 with: name: tasks-template - path: $HOME/pydra-tasks-template + path: $HOME/pydra-tasks-template.tar.gz + + - name: Extract the pydra-tasks-template + run: | + pushd $HOME + tar xzf pydra-tasks-template.tar.gz + popd - name: Unset header # checkout@v2 adds a header that makes branch protection report errors From c9ad0977726d32fa246d3f2ecb84308f76799713 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 22:12:14 +1000 Subject: [PATCH 09/13] debugging --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 74c5b6db..a6e2fa56 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -26,7 +26,7 @@ jobs: rm pydra-tasks-template.tar.gz mv pydra-tasks-template-${LATEST#v} pydra-tasks-template tar czf $HOME/pydra-tasks-template.tar.gz pydra-tasks-template - + echo $HOME - uses: actions/upload-artifact@v4 with: From 1d6991edfe92a343de4361dc844677b74513dfbb Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 22:15:32 +1000 Subject: [PATCH 10/13] debugging gha --- .github/workflows/ci-cd.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a6e2fa56..d9ef0cf7 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -25,13 +25,12 @@ jobs: tar xzf pydra-tasks-template.tar.gz rm pydra-tasks-template.tar.gz mv pydra-tasks-template-${LATEST#v} pydra-tasks-template - tar czf $HOME/pydra-tasks-template.tar.gz pydra-tasks-template - echo $HOME + tar czf /home/runner/pydra-tasks-template.tar.gz pydra-tasks-template - uses: actions/upload-artifact@v4 with: name: tasks-template - path: $HOME/pydra-tasks-template.tar.gz + path: /home/runner/pydra-tasks-template.tar.gz test: needs: [download-task-template] @@ -53,11 +52,11 @@ jobs: uses: actions/download-artifact@v4 with: name: tasks-template - path: $HOME/pydra-tasks-template.tar.gz + path: /home/runner/pydra-tasks-template.tar.gz - name: Extract the pydra-tasks-template run: | - pushd $HOME + pushd /home/runner tar xzf pydra-tasks-template.tar.gz popd @@ -98,7 +97,7 @@ jobs: - name: Pytest run: >- - NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=$HOME/pydra-tasks-template + NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=/home/runner/pydra-tasks-template pytest -vvs --cov nipype2pydra --cov-config .coveragerc --cov-report xml - name: Upload coverage to Codecov From 402df914c8c2970ceb3f4cabcc0f466b6e82906a Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 22:17:12 +1000 Subject: [PATCH 11/13] debugging gha --- .github/workflows/ci-cd.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index d9ef0cf7..4ee8ab6b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -25,12 +25,12 @@ jobs: tar xzf pydra-tasks-template.tar.gz rm pydra-tasks-template.tar.gz mv pydra-tasks-template-${LATEST#v} pydra-tasks-template - tar czf /home/runner/pydra-tasks-template.tar.gz pydra-tasks-template + tar czf /pydra-tasks-template.tar.gz pydra-tasks-template - uses: actions/upload-artifact@v4 with: name: tasks-template - path: /home/runner/pydra-tasks-template.tar.gz + path: /pydra-tasks-template.tar.gz test: needs: [download-task-template] @@ -52,11 +52,11 @@ jobs: uses: actions/download-artifact@v4 with: name: tasks-template - path: /home/runner/pydra-tasks-template.tar.gz + path: /pydra-tasks-template.tar.gz - name: Extract the pydra-tasks-template run: | - pushd /home/runner + pushd / tar xzf pydra-tasks-template.tar.gz popd @@ -97,7 +97,7 @@ jobs: - name: Pytest run: >- - NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=/home/runner/pydra-tasks-template + NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=/pydra-tasks-template pytest -vvs --cov nipype2pydra --cov-config .coveragerc --cov-report xml - name: Upload coverage to Codecov From a72f1e7df36aa0099a49fd5594fc7b4ee59f44a3 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 22:19:25 +1000 Subject: [PATCH 12/13] gh db --- .github/workflows/ci-cd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4ee8ab6b..50410940 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -25,7 +25,9 @@ jobs: tar xzf pydra-tasks-template.tar.gz rm pydra-tasks-template.tar.gz mv pydra-tasks-template-${LATEST#v} pydra-tasks-template - tar czf /pydra-tasks-template.tar.gz pydra-tasks-template + tar czf pydra-tasks-template.tar.gz pydra-tasks-template + echo $PWD + ls - uses: actions/upload-artifact@v4 with: From c9c0fcf7732b11110d507007bb9dde3467daac8c Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 17 May 2024 22:25:13 +1000 Subject: [PATCH 13/13] ghdb --- .github/workflows/ci-cd.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 50410940..6bd985a2 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -26,13 +26,11 @@ jobs: rm pydra-tasks-template.tar.gz mv pydra-tasks-template-${LATEST#v} pydra-tasks-template tar czf pydra-tasks-template.tar.gz pydra-tasks-template - echo $PWD - ls - uses: actions/upload-artifact@v4 with: name: tasks-template - path: /pydra-tasks-template.tar.gz + path: pydra-tasks-template.tar.gz test: needs: [download-task-template] @@ -54,13 +52,10 @@ jobs: uses: actions/download-artifact@v4 with: name: tasks-template - path: /pydra-tasks-template.tar.gz + path: . - name: Extract the pydra-tasks-template - run: | - pushd / - tar xzf pydra-tasks-template.tar.gz - popd + run: tar xzf pydra-tasks-template.tar.gz - name: Unset header # checkout@v2 adds a header that makes branch protection report errors @@ -99,7 +94,7 @@ jobs: - name: Pytest run: >- - NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=/pydra-tasks-template + NIPYPE2PYDRA_PYDRA_TASK_TEMPLATE=$PWD/pydra-tasks-template pytest -vvs --cov nipype2pydra --cov-config .coveragerc --cov-report xml - name: Upload coverage to Codecov