From 5c4c4893a285e57ad4ab5730e665157d5ffe1775 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Thu, 1 Jul 2021 13:42:16 +0200 Subject: [PATCH 1/5] Break tests into different groups --- .github/workflows/ci.yml | 16 ++- test/runtests.jl | 224 +++++++++++++++++++++------------------ 2 files changed, 133 insertions(+), 107 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9d9ad787..b3b778a2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: pull_request: jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.group }} runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.version == 'nightly' }} strategy: @@ -22,6 +22,13 @@ jobs: - windows-latest arch: - x64 + group: + - 'Transform' + - 'BaseKernels' + - 'Kernels' + - 'Matrix' + - 'MultiOutput' + - 'Others' steps: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 @@ -40,9 +47,14 @@ jobs: ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + with: + coverage: ${{ matrix.coverage }} || false + env: + GROUP: ${{ matrix.group }} - uses: julia-actions/julia-processcoverage@v1 + if: matrix.coverage - name: Send coverage - if: matrix.version == '1' && matrix.os == 'ubuntu-latest' + if: matrix.coverage uses: coverallsapp/github-action@master with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/test/runtests.jl b/test/runtests.jl index 51a72742b..a28eaa2be 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,6 +19,8 @@ using KernelFunctions: SimpleKernel, metric, kappa, ColVecs, RowVecs, TestUtils using KernelFunctions.TestUtils: test_interface +const GROUP = get(ENV, "GROUP", "All") + # Writing tests: # 1. The file structure of the test should match precisely the file structure of src. # Amongst other things, this means that there should be exactly 1 test file per src file. @@ -54,123 +56,135 @@ using KernelFunctions.TestUtils: test_interface include("test_utils.jl") @testset "KernelFunctions" begin - include("utils.jl") - - @testset "distances" begin - include(joinpath("distances", "pairwise.jl")) - include(joinpath("distances", "dotproduct.jl")) - include(joinpath("distances", "delta.jl")) - include(joinpath("distances", "sinus.jl")) + if GROUP == "All" || GROUP == "Transform" + @testset "transform" begin + include(joinpath("transform", "transform.jl")) + print(" ") + include(joinpath("transform", "scaletransform.jl")) + print(" ") + include(joinpath("transform", "ardtransform.jl")) + print(" ") + include(joinpath("transform", "lineartransform.jl")) + print(" ") + include(joinpath("transform", "functiontransform.jl")) + print(" ") + include(joinpath("transform", "selecttransform.jl")) + print(" ") + include(joinpath("transform", "chaintransform.jl")) + print(" ") + include(joinpath("transform", "periodic_transform.jl")) + print(" ") + end + @info "Ran tests on Transform" end - @info "Ran tests on Distances" - @testset "transform" begin - include(joinpath("transform", "transform.jl")) - print(" ") - include(joinpath("transform", "scaletransform.jl")) - print(" ") - include(joinpath("transform", "ardtransform.jl")) - print(" ") - include(joinpath("transform", "lineartransform.jl")) - print(" ") - include(joinpath("transform", "functiontransform.jl")) - print(" ") - include(joinpath("transform", "selecttransform.jl")) - print(" ") - include(joinpath("transform", "chaintransform.jl")) - print(" ") - include(joinpath("transform", "periodic_transform.jl")) - print(" ") + if GROUP == "All" || GROUP == "BaseKernels" + @testset "basekernels" begin + include(joinpath("basekernels", "constant.jl")) + print(" ") + include(joinpath("basekernels", "cosine.jl")) + print(" ") + include(joinpath("basekernels", "exponential.jl")) + print(" ") + include(joinpath("basekernels", "exponentiated.jl")) + print(" ") + include(joinpath("basekernels", "fbm.jl")) + print(" ") + include(joinpath("basekernels", "gabor.jl")) + print(" ") + include(joinpath("basekernels", "matern.jl")) + print(" ") + include(joinpath("basekernels", "nn.jl")) + print(" ") + include(joinpath("basekernels", "periodic.jl")) + print(" ") + include(joinpath("basekernels", "piecewisepolynomial.jl")) + print(" ") + include(joinpath("basekernels", "polynomial.jl")) + print(" ") + include(joinpath("basekernels", "rational.jl")) + print(" ") + include(joinpath("basekernels", "sm.jl")) + print(" ") + include(joinpath("basekernels", "wiener.jl")) + print(" ") + end + @info "Ran tests on BaseKernel" end - @info "Ran tests on Transform" - @testset "basekernels" begin - include(joinpath("basekernels", "constant.jl")) - print(" ") - include(joinpath("basekernels", "cosine.jl")) - print(" ") - include(joinpath("basekernels", "exponential.jl")) - print(" ") - include(joinpath("basekernels", "exponentiated.jl")) - print(" ") - include(joinpath("basekernels", "fbm.jl")) - print(" ") - include(joinpath("basekernels", "gabor.jl")) - print(" ") - include(joinpath("basekernels", "matern.jl")) - print(" ") - include(joinpath("basekernels", "nn.jl")) - print(" ") - include(joinpath("basekernels", "periodic.jl")) - print(" ") - include(joinpath("basekernels", "piecewisepolynomial.jl")) - print(" ") - include(joinpath("basekernels", "polynomial.jl")) - print(" ") - include(joinpath("basekernels", "rational.jl")) - print(" ") - include(joinpath("basekernels", "sm.jl")) - print(" ") - include(joinpath("basekernels", "wiener.jl")) - print(" ") + if GROUP == "All" || GROUP == "Kernels" + @testset "kernels" begin + include(joinpath("kernels", "kernelproduct.jl")) + include(joinpath("kernels", "kernelsum.jl")) + include(joinpath("kernels", "kerneltensorproduct.jl")) + include(joinpath("kernels", "overloads.jl")) + include(joinpath("kernels", "scaledkernel.jl")) + include(joinpath("kernels", "transformedkernel.jl")) + include(joinpath("kernels", "normalizedkernel.jl")) + include(joinpath("kernels", "neuralkernelnetwork.jl")) + end + @info "Ran tests on Kernel" end - @info "Ran tests on BaseKernel" - @testset "kernels" begin - include(joinpath("kernels", "kernelproduct.jl")) - include(joinpath("kernels", "kernelsum.jl")) - include(joinpath("kernels", "kerneltensorproduct.jl")) - include(joinpath("kernels", "overloads.jl")) - include(joinpath("kernels", "scaledkernel.jl")) - include(joinpath("kernels", "transformedkernel.jl")) - include(joinpath("kernels", "normalizedkernel.jl")) - include(joinpath("kernels", "neuralkernelnetwork.jl")) + if GROUP == "All" || GROUP == "Matrix" + @testset "matrix" begin + include(joinpath("matrix", "kernelmatrix.jl")) + include(joinpath("matrix", "kernelkroneckermat.jl")) + include(joinpath("matrix", "kernelpdmat.jl")) + end + @info "Ran tests on matrix" end - @info "Ran tests on Kernel" - @testset "matrix" begin - include(joinpath("matrix", "kernelmatrix.jl")) - include(joinpath("matrix", "kernelkroneckermat.jl")) - include(joinpath("matrix", "kernelpdmat.jl")) + if GROUP == "All" || GROUP == "MultiOutput" + @testset "multi_output" begin + include(joinpath("mokernels", "moinput.jl")) + include(joinpath("mokernels", "independent.jl")) + include(joinpath("mokernels", "slfm.jl")) + include(joinpath("mokernels", "intrinsiccoregion.jl")) + include(joinpath("mokernels", "lmm.jl")) + end + @info "Ran tests on Multi-Output Kernels" end - @info "Ran tests on matrix" - @testset "multi_output" begin - include(joinpath("mokernels", "moinput.jl")) - include(joinpath("mokernels", "independent.jl")) - include(joinpath("mokernels", "slfm.jl")) - include(joinpath("mokernels", "intrinsiccoregion.jl")) - include(joinpath("mokernels", "lmm.jl")) - end - @info "Ran tests on Multi-Output Kernels" + if GROUP == "All" || GROUP == "Others" + include("utils.jl") - @testset "approximations" begin - include(joinpath("approximations", "nystrom.jl")) - end + @testset "distances" begin + include(joinpath("distances", "pairwise.jl")) + include(joinpath("distances", "dotproduct.jl")) + include(joinpath("distances", "delta.jl")) + include(joinpath("distances", "sinus.jl")) + end + @info "Ran tests on Distances" + + @testset "approximations" begin + include(joinpath("approximations", "nystrom.jl")) + end - include("generic.jl") - include("chainrules.jl") - include("zygoterules.jl") + include("generic.jl") + include("chainrules.jl") + include("zygoterules.jl") - @testset "doctests" begin - DocMeta.setdocmeta!( - KernelFunctions, - :DocTestSetup, - quote - using KernelFunctions - using LinearAlgebra - using Random - using PDMats: PDMats - end; - recursive=true, - ) - doctest( - KernelFunctions; - doctestfilters=[ - r"{([a-zA-Z0-9]+,\s?)+[a-zA-Z0-9]+}", - r"(Array{[a-zA-Z0-9]+,\s?1}|Vector{[a-zA-Z0-9]+})", - r"(Array{[a-zA-Z0-9]+,\s?2}|Matrix{[a-zA-Z0-9]+})", - ], - ) + @testset "doctests" begin + DocMeta.setdocmeta!( + KernelFunctions, + :DocTestSetup, + quote + using KernelFunctions + using LinearAlgebra + using Random + using PDMats: PDMats + end; + recursive=true, + ) + doctest( + KernelFunctions; + doctestfilters=[ + r"{([a-zA-Z0-9]+,\s?)+[a-zA-Z0-9]+}", + r"(Array{[a-zA-Z0-9]+,\s?1}|Vector{[a-zA-Z0-9]+})", + r"(Array{[a-zA-Z0-9]+,\s?2}|Matrix{[a-zA-Z0-9]+})", + ], + ) + end end end From e4e81cb7d74c0b7408d4818c07859b495eb0c2f1 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Thu, 1 Jul 2021 14:20:06 +0200 Subject: [PATCH 2/5] Only test Ubuntu x64 --- .github/workflows/ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3b778a2f..17d137379 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: pull_request: jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.group }} + name: Julia ${{ matrix.version }} - ${{ matrix.group }} runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.version == 'nightly' }} strategy: @@ -18,8 +18,6 @@ jobs: - 'nightly' os: - ubuntu-latest - - macOS-latest - - windows-latest arch: - x64 group: @@ -48,13 +46,13 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 with: - coverage: ${{ matrix.coverage }} || false + coverage: ${{ matrix.version == '1' && matrix.os == 'ubuntu-latest' }} env: GROUP: ${{ matrix.group }} - uses: julia-actions/julia-processcoverage@v1 - if: matrix.coverage + if: matrix.version == '1' && matrix.os == 'ubuntu-latest' - name: Send coverage - if: matrix.coverage + if: matrix.version == '1' && matrix.os == 'ubuntu-latest' uses: coverallsapp/github-action@master with: github-token: ${{ secrets.GITHUB_TOKEN }} From 39664152987c67477f3fbcc3eb3abe316832d222 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Thu, 1 Jul 2021 16:34:23 +0200 Subject: [PATCH 3/5] Remove Matrix group --- .github/workflows/ci.yml | 1 - test/runtests.jl | 18 +++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17d137379..8d305b254 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,6 @@ jobs: - 'Transform' - 'BaseKernels' - 'Kernels' - - 'Matrix' - 'MultiOutput' - 'Others' steps: diff --git a/test/runtests.jl b/test/runtests.jl index a28eaa2be..8caa2a0b4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -126,15 +126,6 @@ include("test_utils.jl") @info "Ran tests on Kernel" end - if GROUP == "All" || GROUP == "Matrix" - @testset "matrix" begin - include(joinpath("matrix", "kernelmatrix.jl")) - include(joinpath("matrix", "kernelkroneckermat.jl")) - include(joinpath("matrix", "kernelpdmat.jl")) - end - @info "Ran tests on matrix" - end - if GROUP == "All" || GROUP == "MultiOutput" @testset "multi_output" begin include(joinpath("mokernels", "moinput.jl")) @@ -157,6 +148,15 @@ include("test_utils.jl") end @info "Ran tests on Distances" + if GROUP == "All" || GROUP == "Matrix" + @testset "matrix" begin + include(joinpath("matrix", "kernelmatrix.jl")) + include(joinpath("matrix", "kernelkroneckermat.jl")) + include(joinpath("matrix", "kernelpdmat.jl")) + end + @info "Ran tests on matrix" + end + @testset "approximations" begin include(joinpath("approximations", "nystrom.jl")) end From 2069135f4a4697214341c42cc48030986c7d6dd2 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Thu, 1 Jul 2021 22:47:11 +0200 Subject: [PATCH 4/5] Apply suggestions --- test/runtests.jl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 8caa2a0b4..067ef9a14 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,7 +19,9 @@ using KernelFunctions: SimpleKernel, metric, kappa, ColVecs, RowVecs, TestUtils using KernelFunctions.TestUtils: test_interface -const GROUP = get(ENV, "GROUP", "All") +# The GROUP is used to run different sets of tests in parallel on the GitHub Actions CI. +# If you want to introduce a new group, ensure you also add it to .github/workflows/ci.yml +const GROUP = get(ENV, "GROUP", "") # Writing tests: # 1. The file structure of the test should match precisely the file structure of src. @@ -56,7 +58,7 @@ const GROUP = get(ENV, "GROUP", "All") include("test_utils.jl") @testset "KernelFunctions" begin - if GROUP == "All" || GROUP == "Transform" + if GROUP == "" || GROUP == "Transform" @testset "transform" begin include(joinpath("transform", "transform.jl")) print(" ") @@ -78,7 +80,7 @@ include("test_utils.jl") @info "Ran tests on Transform" end - if GROUP == "All" || GROUP == "BaseKernels" + if GROUP == "" || GROUP == "BaseKernels" @testset "basekernels" begin include(joinpath("basekernels", "constant.jl")) print(" ") @@ -112,7 +114,7 @@ include("test_utils.jl") @info "Ran tests on BaseKernel" end - if GROUP == "All" || GROUP == "Kernels" + if GROUP == "" || GROUP == "Kernels" @testset "kernels" begin include(joinpath("kernels", "kernelproduct.jl")) include(joinpath("kernels", "kernelsum.jl")) @@ -126,7 +128,7 @@ include("test_utils.jl") @info "Ran tests on Kernel" end - if GROUP == "All" || GROUP == "MultiOutput" + if GROUP == "" || GROUP == "MultiOutput" @testset "multi_output" begin include(joinpath("mokernels", "moinput.jl")) include(joinpath("mokernels", "independent.jl")) @@ -137,7 +139,7 @@ include("test_utils.jl") @info "Ran tests on Multi-Output Kernels" end - if GROUP == "All" || GROUP == "Others" + if GROUP == "" || GROUP == "Others" include("utils.jl") @testset "distances" begin @@ -148,14 +150,12 @@ include("test_utils.jl") end @info "Ran tests on Distances" - if GROUP == "All" || GROUP == "Matrix" - @testset "matrix" begin - include(joinpath("matrix", "kernelmatrix.jl")) - include(joinpath("matrix", "kernelkroneckermat.jl")) - include(joinpath("matrix", "kernelpdmat.jl")) - end - @info "Ran tests on matrix" + @testset "matrix" begin + include(joinpath("matrix", "kernelmatrix.jl")) + include(joinpath("matrix", "kernelkroneckermat.jl")) + include(joinpath("matrix", "kernelpdmat.jl")) end + @info "Ran tests on matrix" @testset "approximations" begin include(joinpath("approximations", "nystrom.jl")) From 068bb4be7af35465ffb3d95505bd1d87b955cc94 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 2 Jul 2021 01:38:47 +0200 Subject: [PATCH 5/5] Fix Coveralls? --- .github/workflows/ci.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d305b254..8eeb12b4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,9 +50,20 @@ jobs: GROUP: ${{ matrix.group }} - uses: julia-actions/julia-processcoverage@v1 if: matrix.version == '1' && matrix.os == 'ubuntu-latest' - - name: Send coverage + - name: Coveralls parallel if: matrix.version == '1' && matrix.os == 'ubuntu-latest' uses: coverallsapp/github-action@master with: - github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: ./lcov.info + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./lcov.info + flag-name: run-${{ matrix.group }} + parallel: true + finish: + needs: test + runs-on: ubuntu-latest + steps: + - name: Send coverage + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.github_token }} + parallel-finished: true