From cacb9788f92a9cdcb1d95ff9b6c5af394f17c7ce Mon Sep 17 00:00:00 2001 From: ST John Date: Fri, 21 Jan 2022 11:36:41 +0200 Subject: [PATCH 01/13] make nystrom work with AbstractVector --- src/approximations/nystrom.jl | 54 ++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index a5cb1c035..151781383 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -1,23 +1,29 @@ # Following the algorithm by William and Seeger, 2001 # Cs is equivalent to X_mm and C to X_mn -function sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) +function sampleindex(X::AbstractVector, r::Real) 0 < r <= 1 || throw(ArgumentError("Sample rate `r` must be in range (0,1]")) - n = size(X, obsdim) + n = length(X) m = ceil(Int, n * r) S = StatsBase.sample(1:n, m; replace=false, ordered=true) return S end +function sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) + return sampleindex(vec_of_vecs(X; obsdim), r) +end + +function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) + Xₘ = X[S] + C = kernelmatrix(k, Xₘ, X) + Cs = C[:, S] + return (C, Cs) +end + function nystrom_sample( k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs ) - obsdim ∈ [1, 2] || - throw(ArgumentError("`obsdim` should be 1 or 2 (see docs of kernelmatrix))")) - Xₘ = obsdim == 1 ? X[S, :] : X[:, S] - C = kernelmatrix(k, Xₘ, X; obsdim=obsdim) - Cs = C[:, S] - return (C, Cs) + return nystrom_sample(k, vec_of_vecs(X; obsdim), S) end function nystrom_pinv!(Cs::Matrix{T}, tol::T=eps(T) * size(Cs, 1)) where {T<:Real} @@ -63,38 +69,46 @@ function NystromFact(W::Matrix{<:Real}, C::Matrix{<:Real}) end @doc raw""" - nystrom(k::Kernel, X::Matrix, S::Vector; obsdim::Int=defaultobs) + nystrom(k::Kernel, X::Vector, S::Vector) -Computes a factorization of Nystrom approximation of the square kernel matrix of data -matrix `X` with respect to kernel `k`. Returns a `NystromFact` struct which stores a -Nystrom factorization satisfying: +Computes a factorization of Nystrom approximation of the square kernel matrix +of data vector `X` with respect to kernel `k`, using indices `S`. +Returns a `NystromFact` struct which stores a Nystrom factorization satisfying: ```math \mathbf{K} \approx \mathbf{C}^{\intercal}\mathbf{W}\mathbf{C} ``` """ -function nystrom(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Int=defaultobs) - C, Cs = nystrom_sample(k, X, S; obsdim=obsdim) +function nystrom(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) + C, Cs = nystrom_sample(k, X, S) W = nystrom_pinv!(Cs) return NystromFact(W, C) end @doc raw""" - nystrom(k::Kernel, X::Matrix, r::Real; obsdim::Int=defaultobs) + nystrom(k::Kernel, X::Vector, r::Real) -Computes a factorization of Nystrom approximation of the square kernel matrix of data -matrix `X` with respect to kernel `k` using a sample ratio of `r`. +Computes a factorization of Nystrom approximation of the square kernel matrix +of data vector `X` with respect to kernel `k` using a sample ratio of `r`. Returns a `NystromFact` struct which stores a Nystrom factorization satisfying: ```math \mathbf{K} \approx \mathbf{C}^{\intercal}\mathbf{W}\mathbf{C} ``` """ +function nystrom(k::Kernel, X::AbstractVector, r::Real) + S = sampleindex(X, r) + return nystrom(k, X, S) +end + +function nystrom(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Int=defaultobs) + return nystrom(k, vec_of_vecs(X; obsdim), S) +end + function nystrom(k::Kernel, X::AbstractMatrix, r::Real; obsdim::Int=defaultobs) - S = sampleindex(X, r; obsdim=obsdim) - return nystrom(k, X, S; obsdim=obsdim) + return nystrom(k, vec_of_vecs(X; obsdim), r) end """ - nystrom(CᵀWC::NystromFact) + kernelmatrix(CᵀWC::NystromFact) Compute the approximate kernel matrix based on the Nystrom factorization. """ From ab061381ebcc37198b8727a7573bb901e34feda4 Mon Sep 17 00:00:00 2001 From: ST John Date: Fri, 21 Jan 2022 11:40:36 +0200 Subject: [PATCH 02/13] add test --- test/approximations/nystrom.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/approximations/nystrom.jl b/test/approximations/nystrom.jl index 5e9c6773d..41b6d7c08 100644 --- a/test/approximations/nystrom.jl +++ b/test/approximations/nystrom.jl @@ -2,6 +2,12 @@ dims = [10, 5] X = rand(dims...) k = SqExponentialKernel() + for obsdim in [1, 2] + Xv = vec_of_vecs(X; obsdim) + @assert Xv isa Union{ColVecs,RowVecs} + @test kernelmatrix(k, Xv) ≈ kernelmatrix(nystrom(k, Xv, 1.0)) + @test kernelmatrix(k, Xv) ≈ kernelmatrix(nystrom(k, Xv, collect(1:dims[obsdim]))) + end for obsdim in [1, 2] @test kernelmatrix(k, X; obsdim=obsdim) ≈ kernelmatrix(nystrom(k, X, 1.0; obsdim=obsdim)) From 5697e73255eaf831c21268a7b9c484c237b48074 Mon Sep 17 00:00:00 2001 From: st-- Date: Fri, 21 Jan 2022 10:43:17 +0100 Subject: [PATCH 03/13] Update test/approximations/nystrom.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/approximations/nystrom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/approximations/nystrom.jl b/test/approximations/nystrom.jl index 41b6d7c08..335e05431 100644 --- a/test/approximations/nystrom.jl +++ b/test/approximations/nystrom.jl @@ -10,7 +10,7 @@ end for obsdim in [1, 2] @test kernelmatrix(k, X; obsdim=obsdim) ≈ - kernelmatrix(nystrom(k, X, 1.0; obsdim=obsdim)) + kernelmatrix(nystrom(k, X, 1.0; obsdim=obsdim)) @test kernelmatrix(k, X; obsdim=obsdim) ≈ kernelmatrix(nystrom(k, X, collect(1:dims[obsdim]); obsdim=obsdim)) end From dcd0addd3460875bf7d6d59a6167ace3ea83fed3 Mon Sep 17 00:00:00 2001 From: ST John Date: Fri, 21 Jan 2022 11:44:37 +0200 Subject: [PATCH 04/13] patch bump --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 3169e3777..822c5ee2d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "KernelFunctions" uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392" -version = "0.10.27" +version = "0.10.28" [deps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" From e85a977edc458013bbd3529c018a01e65d65a01f Mon Sep 17 00:00:00 2001 From: st-- Date: Fri, 21 Jan 2022 11:01:53 +0100 Subject: [PATCH 05/13] Update test/approximations/nystrom.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/approximations/nystrom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/approximations/nystrom.jl b/test/approximations/nystrom.jl index 335e05431..3ef830f96 100644 --- a/test/approximations/nystrom.jl +++ b/test/approximations/nystrom.jl @@ -12,6 +12,6 @@ @test kernelmatrix(k, X; obsdim=obsdim) ≈ kernelmatrix(nystrom(k, X, 1.0; obsdim=obsdim)) @test kernelmatrix(k, X; obsdim=obsdim) ≈ - kernelmatrix(nystrom(k, X, collect(1:dims[obsdim]); obsdim=obsdim)) + kernelmatrix(nystrom(k, X, collect(1:dims[obsdim]); obsdim=obsdim)) end end From 7cd1366d6ac87abd27158371c5cd3a456f306872 Mon Sep 17 00:00:00 2001 From: st-- Date: Fri, 21 Jan 2022 12:16:45 +0100 Subject: [PATCH 06/13] Apply suggestions from code review Co-authored-by: David Widmann --- src/approximations/nystrom.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 151781383..208785990 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -10,7 +10,7 @@ function sampleindex(X::AbstractVector, r::Real) end function sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) - return sampleindex(vec_of_vecs(X; obsdim), r) + return sampleindex(vec_of_vecs(X; obsdim=obsdim), r) end function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) @@ -23,7 +23,7 @@ end function nystrom_sample( k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs ) - return nystrom_sample(k, vec_of_vecs(X; obsdim), S) + return nystrom_sample(k, vec_of_vecs(X; obsdim=obsdim), S) end function nystrom_pinv!(Cs::Matrix{T}, tol::T=eps(T) * size(Cs, 1)) where {T<:Real} @@ -78,7 +78,7 @@ Returns a `NystromFact` struct which stores a Nystrom factorization satisfying: \mathbf{K} \approx \mathbf{C}^{\intercal}\mathbf{W}\mathbf{C} ``` """ -function nystrom(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) +function nystrom(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer}) C, Cs = nystrom_sample(k, X, S) W = nystrom_pinv!(Cs) return NystromFact(W, C) @@ -100,7 +100,7 @@ function nystrom(k::Kernel, X::AbstractVector, r::Real) end function nystrom(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Int=defaultobs) - return nystrom(k, vec_of_vecs(X; obsdim), S) + return nystrom(k, vec_of_vecs(X; obsdim=obsdim), S) end function nystrom(k::Kernel, X::AbstractMatrix, r::Real; obsdim::Int=defaultobs) From 27b005f86cce770c13a56fd3d5842ab1fa96ed84 Mon Sep 17 00:00:00 2001 From: st-- Date: Fri, 21 Jan 2022 12:20:47 +0100 Subject: [PATCH 07/13] Apply suggestions from code review --- src/approximations/nystrom.jl | 2 +- test/approximations/nystrom.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 208785990..65af0939b 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -104,7 +104,7 @@ function nystrom(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Int end function nystrom(k::Kernel, X::AbstractMatrix, r::Real; obsdim::Int=defaultobs) - return nystrom(k, vec_of_vecs(X; obsdim), r) + return nystrom(k, vec_of_vecs(X; obsdim=obsdim), r) end """ diff --git a/test/approximations/nystrom.jl b/test/approximations/nystrom.jl index 3ef830f96..14476340f 100644 --- a/test/approximations/nystrom.jl +++ b/test/approximations/nystrom.jl @@ -3,7 +3,7 @@ X = rand(dims...) k = SqExponentialKernel() for obsdim in [1, 2] - Xv = vec_of_vecs(X; obsdim) + Xv = vec_of_vecs(X; obsdim=obsdim) @assert Xv isa Union{ColVecs,RowVecs} @test kernelmatrix(k, Xv) ≈ kernelmatrix(nystrom(k, Xv, 1.0)) @test kernelmatrix(k, Xv) ≈ kernelmatrix(nystrom(k, Xv, collect(1:dims[obsdim]))) From 4ef3481355a38df46961f1d6a8fc722a18f88606 Mon Sep 17 00:00:00 2001 From: ST John Date: Fri, 21 Jan 2022 14:31:42 +0200 Subject: [PATCH 08/13] deprecate --- src/approximations/nystrom.jl | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 65af0939b..5bc6f4a4c 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -9,9 +9,7 @@ function sampleindex(X::AbstractVector, r::Real) return S end -function sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) - return sampleindex(vec_of_vecs(X; obsdim=obsdim), r) -end +@deprecate sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) sampleindex(vec_of_vecs(X; obsdim=obsdim), r) false function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) Xₘ = X[S] @@ -20,11 +18,7 @@ function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) return (C, Cs) end -function nystrom_sample( - k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs -) - return nystrom_sample(k, vec_of_vecs(X; obsdim=obsdim), S) -end +@deprecate nystrom_sample(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs) nystrom_sample(k, vec_of_vecs(X; obsdim=obsdim), S) false function nystrom_pinv!(Cs::Matrix{T}, tol::T=eps(T) * size(Cs, 1)) where {T<:Real} # Compute eigendecomposition of sampled component of K From b9cb3dfc544bdab931a17996d6699666dff7013e Mon Sep 17 00:00:00 2001 From: st-- Date: Fri, 21 Jan 2022 13:33:36 +0100 Subject: [PATCH 09/13] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/approximations/nystrom.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 5bc6f4a4c..09a015906 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -9,7 +9,9 @@ function sampleindex(X::AbstractVector, r::Real) return S end -@deprecate sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) sampleindex(vec_of_vecs(X; obsdim=obsdim), r) false +@deprecate sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) sampleindex( + vec_of_vecs(X; obsdim=obsdim), r +) false function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) Xₘ = X[S] @@ -18,7 +20,9 @@ function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) return (C, Cs) end -@deprecate nystrom_sample(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs) nystrom_sample(k, vec_of_vecs(X; obsdim=obsdim), S) false +@deprecate nystrom_sample( + k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs +) nystrom_sample(k, vec_of_vecs(X; obsdim=obsdim), S) false function nystrom_pinv!(Cs::Matrix{T}, tol::T=eps(T) * size(Cs, 1)) where {T<:Real} # Compute eigendecomposition of sampled component of K From 24c6ea723783d2bcb8ee06d8a6f21a3ad9c4f8b3 Mon Sep 17 00:00:00 2001 From: st-- Date: Sun, 23 Jan 2022 11:12:31 +0100 Subject: [PATCH 10/13] Apply suggestions from code review Co-authored-by: David Widmann --- src/approximations/nystrom.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 09a015906..6ecdd18e8 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -67,9 +67,9 @@ function NystromFact(W::Matrix{<:Real}, C::Matrix{<:Real}) end @doc raw""" - nystrom(k::Kernel, X::Vector, S::Vector) + nystrom(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer}) -Computes a factorization of Nystrom approximation of the square kernel matrix +Compute a factorization of a Nystrom approximation of the square kernel matrix of data vector `X` with respect to kernel `k`, using indices `S`. Returns a `NystromFact` struct which stores a Nystrom factorization satisfying: ```math @@ -83,9 +83,9 @@ function nystrom(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer}) end @doc raw""" - nystrom(k::Kernel, X::Vector, r::Real) + nystrom(k::Kernel, X::AbstractVector, r::Real) -Computes a factorization of Nystrom approximation of the square kernel matrix +Compute a factorization of a Nystrom approximation of the square kernel matrix of data vector `X` with respect to kernel `k` using a sample ratio of `r`. Returns a `NystromFact` struct which stores a Nystrom factorization satisfying: ```math From 41da4e6d594dd669a8e9a59464b76df18904f1dd Mon Sep 17 00:00:00 2001 From: st-- Date: Thu, 27 Jan 2022 17:07:33 +0100 Subject: [PATCH 11/13] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Galy-Fajou --- src/approximations/nystrom.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 6ecdd18e8..2e2d68d21 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -13,7 +13,7 @@ end vec_of_vecs(X; obsdim=obsdim), r ) false -function nystrom_sample(k::Kernel, X::AbstractVector, S::Vector{<:Integer}) +function nystrom_sample(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer}) Xₘ = X[S] C = kernelmatrix(k, Xₘ, X) Cs = C[:, S] @@ -97,7 +97,7 @@ function nystrom(k::Kernel, X::AbstractVector, r::Real) return nystrom(k, X, S) end -function nystrom(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Int=defaultobs) +function nystrom(k::Kernel, X::AbstractMatrix, S::AbstractVector{<:Integer}; obsdim::Int=defaultobs) return nystrom(k, vec_of_vecs(X; obsdim=obsdim), S) end From c34a4baaa1ab57d109d9f7611b1c5976df987b2c Mon Sep 17 00:00:00 2001 From: st-- Date: Thu, 27 Jan 2022 17:07:57 +0100 Subject: [PATCH 12/13] Update src/approximations/nystrom.jl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Galy-Fajou --- src/approximations/nystrom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 2e2d68d21..07bdd5006 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -14,7 +14,7 @@ end ) false function nystrom_sample(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer}) - Xₘ = X[S] + Xₘ = @view X[S] C = kernelmatrix(k, Xₘ, X) Cs = C[:, S] return (C, Cs) From 6d99f21245e63421d8374fba32c1e574bd4d1a7b Mon Sep 17 00:00:00 2001 From: st-- Date: Thu, 27 Jan 2022 17:13:21 +0100 Subject: [PATCH 13/13] Update src/approximations/nystrom.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/approximations/nystrom.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl index 07bdd5006..674d9c8a9 100644 --- a/src/approximations/nystrom.jl +++ b/src/approximations/nystrom.jl @@ -97,7 +97,9 @@ function nystrom(k::Kernel, X::AbstractVector, r::Real) return nystrom(k, X, S) end -function nystrom(k::Kernel, X::AbstractMatrix, S::AbstractVector{<:Integer}; obsdim::Int=defaultobs) +function nystrom( + k::Kernel, X::AbstractMatrix, S::AbstractVector{<:Integer}; obsdim::Int=defaultobs +) return nystrom(k, vec_of_vecs(X; obsdim=obsdim), S) end