From 87892fb4024270fc9161b1d3751e4ac290a0c10f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= <theo.galyfajou@gmail.com>
Date: Thu, 4 Nov 2021 13:50:46 +0100
Subject: [PATCH 1/6] Add benchmark file

---
 benchmark/benchmarks.jl | 60 ++++++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 17181b365..4d214072d 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -1,23 +1,51 @@
 using BenchmarkTools
-using Random
-using Distances, LinearAlgebra
+using KernelFunctions
 
-const SUITE = BenchmarkGroup()
+N1 = 10
+N2 = 20
 
-Random.seed!(1234)
+X = rand(N1, N2)
+Xc = ColVecs(X)
+Xr = RowVecs(X)
+Xv = collect.(eachcol(X))
+Y = rand(N1, N2)
+Yc = ColVecs(Y)
+Yr = RowVecs(Y)
+Yv = collect.(eachcol(Y))
 
-dim = 50
-N1 = 1000;
-N2 = 500;
-alpha = 2.0
+# Create the general suite of benchmarks
+SUITE = BenchmarkGroup()
 
-X = rand(Float64, N1, dim)
-Y = rand(Float64, N2, dim)
+kernels = Dict(
+    "SqExponential" => SqExponentialKernel(),
+    "Exponential" => ExponentialKernel(),
+)
 
-KXY = rand(Float64, N1, N2)
-KX = rand(Float64, N1, N1)
-sKX = Symmetric(rand(Float64, N1, N1))
-kX = rand(Float64, N1)
+inputtypes = Dict(
+    "ColVecs" => (Xc, Yc),
+    "RowVecs" => (Xr, Yr),
+    "Vecs" => (Xv, Yv),
+)
 
-include("kernelmatrix.jl")
-include("MLKernels.jl")
+functions = Dict(
+    "kernelmatrixX" => (kernel, X, Y) -> kernelmatrix(kernel, X),
+    "kernelmatrixXY" => (kernel, X, Y) -> kernelmatrix(kernel, X, Y),
+    "kernelmatrix_diagX" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X),
+    "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y)
+)
+
+for (kname, kernel) in kernels
+    SUITE[kname] = sk = BenchmarkGroup()
+    for (inputname, (X, Y)) in inputtypes
+        sk[inputname] = si = BenchmarkGroup()
+        for (fname, f) in functions
+            si[fname] = @benchmarkable $f($kernel, $X, $Y)
+        end
+    end
+end
+
+# Uncomment the following to run benchmark locally
+
+# tune!(SUITE)
+
+# results = run(SUITE, verbose=true)
\ No newline at end of file

From e74b85da75ad7895c8021496b4aa8a39db2656e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= <theo.galyfajou@gmail.com>
Date: Thu, 4 Nov 2021 13:51:01 +0100
Subject: [PATCH 2/6] delete old benchmarks

---
 benchmark/MLKernels.jl    | 22 ----------------------
 benchmark/kernelmatrix.jl | 39 ---------------------------------------
 2 files changed, 61 deletions(-)
 delete mode 100644 benchmark/MLKernels.jl
 delete mode 100644 benchmark/kernelmatrix.jl

diff --git a/benchmark/MLKernels.jl b/benchmark/MLKernels.jl
deleted file mode 100644
index 1b678bdd7..000000000
--- a/benchmark/MLKernels.jl
+++ /dev/null
@@ -1,22 +0,0 @@
-using MLKernels
-
-SUITE["MLKernels"] = BenchmarkGroup()
-
-mlkernelnames = ["SquaredExponentialKernel"]
-kernels = Dict{String,MLKernels.Kernel}()
-for k in mlkernelnames
-    SUITE["MLKernels"][k] = BenchmarkGroup()
-    kernels[k] = eval(Meta.parse("MLKernels." * k * "(alpha)"))
-end
-
-for k in mlkernelnames
-    SUITE["MLKernels"][k]["k(X,Y)"] = @benchmarkable MLKernels.kernelmatrix(
-        $(kernels[k]), $X, $Y
-    )
-    # SUITE["MLKernels"][k][kt]["k!(X,Y)"] = @benchmarkable MLKernels.kernelmatrix!(KXY,$(kernels[k][kt]),$X,$Y) setup=(KXY=copy($KXY))
-    SUITE["MLKernels"][k]["k(X)"] = @benchmarkable MLKernels.kernelmatrix($(kernels[k]), $X)
-    # SUITE["MLKernels"][k][kt]["k!(X)"] = @benchmarkable MLKernels.kernelmatrix!(KX,$(kernels[k][kt]),$X) setup=(KX=copy($KX))
-    # SUITE["MLKernels"][k][kt]["kdiag(X)"] = @benchmarkable MLKernels.kernelmatrix_diag($(kernels[k][kt]),$X)
-    # SUITE["MLKernels"][k][kt]["kdiag!(X)"] = @benchmarkable MLKernels.kernelmatrix_diag!(kX,$(kernels[k][kt]),$X) setup=(kX=copy($kX))
-end
-# results = run(SUITE)
diff --git a/benchmark/kernelmatrix.jl b/benchmark/kernelmatrix.jl
deleted file mode 100644
index e8e708012..000000000
--- a/benchmark/kernelmatrix.jl
+++ /dev/null
@@ -1,39 +0,0 @@
-using KernelFunctions
-
-SUITE["KernelFunctions"] = BenchmarkGroup()
-
-kernelnames = ["SqExponentialKernel"]
-kerneltypes = ["ARD", "ISO"]
-kernels = Dict{String,Dict{String,KernelFunctions.Kernel}}()
-for k in kernelnames
-    kernels[k] = Dict{String,KernelFunctions.Kernel}()
-    SUITE["KernelFunctions"][k] = BenchmarkGroup()
-    for kt in kerneltypes
-        SUITE["KernelFunctions"][k][kt] = BenchmarkGroup()
-        kernels[k][kt] = eval(
-            Meta.parse(
-                "KernelFunctions." *
-                k *
-                "(" *
-                (kt == "ARD" ? "alpha*ones(Float64,dim)" : "alpha") *
-                ")",
-            ),
-        )
-    end
-end
-
-for k in kernelnames
-    for kt in kerneltypes
-        SUITE["KernelFunctions"][k][kt]["k(X,Y)"] = @benchmarkable KernelFunctions.kernelmatrix(
-            $(kernels[k][kt]), $X, $Y; obsdim=1
-        )
-        # SUITE["KernelFunctions"][k][kt]["k!(X,Y)"] = @benchmarkable KernelFunctions.kernelmatrix!(KXY,$(kernels[k][kt]),$X,$Y) setup=(KXY=copy($KXY))
-        SUITE["KernelFunctions"][k][kt]["k(X)"] = @benchmarkable KernelFunctions.kernelmatrix(
-            $(kernels[k][kt]), $X; obsdim=1
-        )
-        # SUITE["KernelFunctions"][k][kt]["k!(X)"] = @benchmarkable KernelFunctions.kernelmatrix!(KX,$(kernels[k][kt]),$X) setup=(KX=copy($KX))
-        # SUITE["KernelFunctions"][k][kt]["kdiag(X)"] = @benchmarkable KernelFunctions.kernelmatrix_diag($(kernels[k][kt]),$X)
-        # SUITE["KernelFunctions"][k][kt]["kdiag!(X)"] = @benchmarkable KernelFunctions.kernelmatrix_diag!(kX,$(kernels[k][kt]),$X) setup=(kX=copy($kX))
-    end
-end
-# results = run(SUITE)

From 0fc266853efb12b07fa543645b965e4b594ae069 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= <theo.galyfajou@gmail.com>
Date: Thu, 4 Nov 2021 13:51:41 +0100
Subject: [PATCH 3/6] Add github action

---
 .github/workflows/benchmark.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 .github/workflows/benchmark.yml

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 000000000..3dfd39d4c
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,24 @@
+name: Run benchmarks
+
+on:
+  pull_request:
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'performance critical')
+    env:
+      JULIA_DEBUG: BenchmarkCI
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: 1.6
+      - name: Install dependencies
+        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI"'
+      - name: Run benchmarks
+        run: julia -e "using BenchmarkCI; BenchmarkCI.judge()"
+      - name: Post results
+        run: julia -e "using BenchmarkCI; BenchmarkCI.postjudge()"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file

From 0cf93797d1b66af6f0293d7696afb97a0c9174b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= <theo.galyfajou@gmail.com>
Date: Thu, 4 Nov 2021 13:54:14 +0100
Subject: [PATCH 4/6] Add Project

---
 .gitignore             | 1 +
 benchmark/Project.toml | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 benchmark/Project.toml

diff --git a/.gitignore b/.gitignore
index eefb4afce..ba19ae215 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
 /test/Manifest.toml
 coverage/
 .DS_store
+benchmark/Manifest.toml
diff --git a/benchmark/Project.toml b/benchmark/Project.toml
new file mode 100644
index 000000000..cf9c7b9c1
--- /dev/null
+++ b/benchmark/Project.toml
@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"

From f4cd28a6290caaa243ab84329f64eb3ea61fb910 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= <theo.galyfajou@gmail.com>
Date: Thu, 4 Nov 2021 14:00:35 +0100
Subject: [PATCH 5/6] Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 benchmark/benchmarks.jl | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 4d214072d..d51f3c020 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -17,21 +17,16 @@ Yv = collect.(eachcol(Y))
 SUITE = BenchmarkGroup()
 
 kernels = Dict(
-    "SqExponential" => SqExponentialKernel(),
-    "Exponential" => ExponentialKernel(),
+    "SqExponential" => SqExponentialKernel(), "Exponential" => ExponentialKernel()
 )
 
-inputtypes = Dict(
-    "ColVecs" => (Xc, Yc),
-    "RowVecs" => (Xr, Yr),
-    "Vecs" => (Xv, Yv),
-)
+inputtypes = Dict("ColVecs" => (Xc, Yc), "RowVecs" => (Xr, Yr), "Vecs" => (Xv, Yv))
 
 functions = Dict(
     "kernelmatrixX" => (kernel, X, Y) -> kernelmatrix(kernel, X),
     "kernelmatrixXY" => (kernel, X, Y) -> kernelmatrix(kernel, X, Y),
     "kernelmatrix_diagX" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X),
-    "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y)
+    "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y),
 )
 
 for (kname, kernel) in kernels

From d41b6e08b24ce9f32ef15636cc166286d4f9df9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= <theo.galyfajou@gmail.com>
Date: Thu, 4 Nov 2021 14:00:55 +0100
Subject: [PATCH 6/6] Missing end of line

---
 benchmark/benchmarks.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index d51f3c020..0cb80e5ce 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -43,4 +43,4 @@ end
 
 # tune!(SUITE)
 
-# results = run(SUITE, verbose=true)
\ No newline at end of file
+# results = run(SUITE, verbose=true)