diff --git a/docs/literate.jl b/docs/literate.jl index 0c8a0e2fe..452d1cdf2 100644 --- a/docs/literate.jl +++ b/docs/literate.jl @@ -13,8 +13,8 @@ Pkg.activate(EXAMPLEPATH) Pkg.instantiate() using Literate: Literate -# Add link to nbviewer below the first heading of level 1 function preprocess(content) + # Add link to nbviewer below the first heading of level 1 sub = SubstitutionString( """ #md # ```@meta @@ -35,7 +35,12 @@ function preprocess(content) # """, ) - return replace(content, r"^# # [^\n]*"m => sub; count=1) + content = replace(content, r"^# # [^\n]*"m => sub; count=1) + + # remove VSCode `##` block delimiter lines + content = replace(content, r"^##$."ms => "") + + return content end # Convert to markdown and notebook diff --git a/examples/kernel-ridge-regression/Manifest.toml b/examples/kernel-ridge-regression/Manifest.toml index c3a4f40f2..8b8a2d648 100644 --- a/examples/kernel-ridge-regression/Manifest.toml +++ b/examples/kernel-ridge-regression/Manifest.toml @@ -1,16 +1,5 @@ # This file is machine-generated - editing it directly is not advised -[[AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.0.1" - -[[AbstractTrees]] -git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5" -uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" -version = "0.3.4" - [[Adapt]] deps = ["LinearAlgebra"] git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7" @@ -23,12 +12,6 @@ uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -[[BFloat16s]] -deps = ["LinearAlgebra", "Test"] -git-tree-sha1 = "4af69e205efc343068dc8722b8dfec1ade89254a" -uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" -version = "0.1.0" - [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -38,41 +21,18 @@ git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e" uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" version = "1.0.6+5" -[[CEnum]] -git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9" -uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" -version = "0.4.1" - -[[CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Memoization", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"] -git-tree-sha1 = "f6f6d2fc7a80b7710b2db4ecb1f59a1b2c2a715a" -uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "3.3.0" - [[Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] git-tree-sha1 = "e2f47f6d8337369411569fd45ae5753ca10394c6" uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" version = "1.16.0+6" -[[ChainRules]] -deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "a41f9e72cffd789d5e19e75f1626b2786d640151" -uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "0.8.11" - [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] git-tree-sha1 = "dbc9aae1227cfddaa9d2552f3ecba5b641f6cce9" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" version = "0.10.5" -[[CodecZlib]] -deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" -uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.0" - [[ColorSchemes]] deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random", "StaticArrays"] git-tree-sha1 = "c8fd01e4b736013bc61b704871d20503b33ea402" @@ -91,12 +51,6 @@ git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" version = "0.12.8" -[[CommonSubexpressions]] -deps = ["MacroTools", "Test"] -git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" -uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" -version = "0.3.0" - [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] git-tree-sha1 = "e4e2b39db08f967cc1360951f01e8a75ec441cab" @@ -118,22 +72,11 @@ git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7" uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" version = "0.5.7" -[[Crayons]] -git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.0.4" - [[DataAPI]] git-tree-sha1 = "dfb3b7e89e395be1e25c2ad6d7690dc29cc53b1d" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.6.0" -[[DataFrames]] -deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "66ee4fe515a9294a8836ef18eea7239c6ac3db5e" -uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.1.1" - [[DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677" @@ -153,18 +96,6 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" deps = ["Mmap"] uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" -[[DiffResults]] -deps = ["StaticArrays"] -git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" -uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" -version = "1.0.3" - -[[DiffRules]] -deps = ["NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9" -uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.0.2" - [[Distances]] deps = ["LinearAlgebra", "Statistics", "StatsAPI"] git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8" @@ -203,11 +134,6 @@ git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" version = "2.2.10+0" -[[ExprTools]] -git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e" -uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.3" - [[FFMPEG]] deps = ["FFMPEG_jll"] git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" @@ -232,12 +158,6 @@ git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.4" -[[Flux]] -deps = ["AbstractTrees", "Adapt", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"] -git-tree-sha1 = "0b3c6d0ce57d3b793eabd346ccc8f605035ef079" -uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.12.4" - [[Fontconfig_jll]] deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] git-tree-sha1 = "35895cf184ceaab11fd778b4590144034a167a2f" @@ -250,12 +170,6 @@ git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" version = "0.4.2" -[[ForwardDiff]] -deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"] -git-tree-sha1 = "e2af66012e08966366a43251e1fd421522908be6" -uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.18" - [[FreeType2_jll]] deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] git-tree-sha1 = "cbd58c9deb1d304f5a245a0b7eb841a2560cfec6" @@ -274,28 +188,12 @@ git-tree-sha1 = "a7bb2af991c43dcf5c3455d276dd83976799634f" uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" version = "0.2.1" -[[Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - [[GLFW_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"] git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157" uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89" version = "3.3.5+0" -[[GPUArrays]] -deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"] -git-tree-sha1 = "3683030b5479249abaa18aa930fc02307fed05d3" -uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "7.0.0" - -[[GPUCompiler]] -deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "765d5b600d3177f1d422c9489525938dd8bd95d1" -uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.12.2" - [[GR]] deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"] git-tree-sha1 = "b83e3125048a9c3158cbb7ca423790c7b1b57bea" @@ -343,12 +241,6 @@ git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" version = "0.2.2" -[[IRTools]] -deps = ["InteractiveUtils", "MacroTools", "Test"] -git-tree-sha1 = "c67e7515a11f726f44083e74f218d134396d6510" -uuid = "7869d1d1-7146-5819-86e3-90919afe41df" -version = "0.4.2" - [[IniFile]] deps = ["Test"] git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" @@ -359,12 +251,6 @@ version = "0.5.0" deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -[[InvertedIndices]] -deps = ["Test"] -git-tree-sha1 = "15732c475062348b0165684ffe28e85ea8396afc" -uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.0.0" - [[IterTools]] git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" @@ -393,12 +279,6 @@ git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943" uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" version = "2.1.0+0" -[[Juno]] -deps = ["Base64", "Logging", "Media", "Profile"] -git-tree-sha1 = "07cb43290a840908a771552911a6274bc6c072c7" -uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d" -version = "0.8.4" - [[KernelFunctions]] deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "LinearAlgebra", "Random", "Requires", "SpecialFunctions", "StatsBase", "StatsFuns", "TensorCore", "Test", "ZygoteRules"] path = "../.." @@ -411,12 +291,6 @@ git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" version = "3.100.1+0" -[[LLVM]] -deps = ["CEnum", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "b3cd5971a37d3ac3c13ca805916b90878c699eaf" -uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "3.8.0" - [[LZO_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" @@ -434,16 +308,6 @@ git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92" uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" version = "0.15.6" -[[LazyArtifacts]] -deps = ["Artifacts", "Pkg"] -uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" - -[[LearnBase]] -deps = ["LinearAlgebra", "StatsBase"] -git-tree-sha1 = "47e6f4623c1db88570c7a7fa66c6528b92ba4725" -uuid = "7f8f8fb0-2700-5f03-b4bd-41f8cfc144b6" -version = "0.3.0" - [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" @@ -536,35 +400,12 @@ version = "0.2.4" [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -[[MLDataPattern]] -deps = ["LearnBase", "MLLabelUtils", "Random", "SparseArrays", "StatsBase"] -git-tree-sha1 = "e99514e96e8b8129bb333c69e063a56ab6402b5b" -uuid = "9920b226-0b2a-5f5f-9153-9aa70a013f8b" -version = "0.5.4" - -[[MLDataUtils]] -deps = ["DataFrames", "DelimitedFiles", "LearnBase", "MLDataPattern", "MLLabelUtils", "Statistics", "StatsBase"] -git-tree-sha1 = "ee54803aea12b9c8ee972e78ece11ac6023715e6" -uuid = "cc2ba9b6-d476-5e6d-8eaf-a92d5412d41d" -version = "0.5.4" - -[[MLLabelUtils]] -deps = ["LearnBase", "MappedArrays", "StatsBase"] -git-tree-sha1 = "3211c1fdd1efaefa692c8cf60e021fb007b76a08" -uuid = "66a33bbf-0c2b-5fc8-a008-9da813334f0a" -version = "0.5.6" - [[MacroTools]] deps = ["Markdown", "Random"] git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" version = "0.5.6" -[[MappedArrays]] -git-tree-sha1 = "18d3584eebc861e311a552cbb67723af8edff5de" -uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" -version = "0.4.0" - [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -584,18 +425,6 @@ git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f" uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e" version = "0.3.1" -[[Media]] -deps = ["MacroTools", "Test"] -git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58" -uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27" -version = "0.5.0" - -[[Memoization]] -deps = ["MacroTools"] -git-tree-sha1 = "a9175def295e0dc1f6da80e8e733a01dd0f36a56" -uuid = "6fafb56a-5788-4b4e-91ca-c0cea6611c73" -version = "0.1.11" - [[Missings]] deps = ["DataAPI"] git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7" @@ -608,18 +437,6 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -[[NNlib]] -deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] -git-tree-sha1 = "7461639cef384a2ad058005b49e32b318d844343" -uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.7.22" - -[[NNlibCUDA]] -deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] -git-tree-sha1 = "bd8b29bf75be7a6c2b288b4b9a4e8903d0376ac1" -uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" -version = "0.1.3" - [[NaNMath]] git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" @@ -703,32 +520,16 @@ git-tree-sha1 = "e995fa1821b6daff8b107a8eafbec234ae2263d0" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" version = "1.16.5" -[[PooledArrays]] -deps = ["DataAPI", "Future"] -git-tree-sha1 = "cde4ce9d6f33219465b55162811d8de8139c0414" -uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "1.2.1" - [[Preferences]] deps = ["TOML"] git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" uuid = "21216c6a-2e73-6563-6e65-726566657250" version = "1.2.2" -[[PrettyTables]] -deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"] -git-tree-sha1 = "0d1245a357cc61c8cd61934c07447aa569ff22e6" -uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "1.1.0" - [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -[[Profile]] -deps = ["Printf"] -uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" - [[Qt5Base_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"] git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8" @@ -749,18 +550,6 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -[[Random123]] -deps = ["Libdl", "Random", "RandomNumbers"] -git-tree-sha1 = "0e8b146557ad1c6deb1367655e052276690e71a3" -uuid = "74087812-796a-5b5d-8853-05524746bad3" -version = "1.4.2" - -[[RandomNumbers]] -deps = ["Random", "Requires"] -git-tree-sha1 = "441e6fc35597524ada7f85e13df1f4e10137d16f" -uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" -version = "1.4.0" - [[RecipesBase]] git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae" uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" @@ -903,18 +692,6 @@ version = "0.1.1" deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -[[TimerOutputs]] -deps = ["ExprTools", "Printf"] -git-tree-sha1 = "bf8aacc899a1bd16522d0350e1e2310510d77236" -uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.9" - -[[TranscodingStreams]] -deps = ["Random", "Test"] -git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.5" - [[URIs]] git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" @@ -1077,12 +854,6 @@ git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845" uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" version = "1.4.0+3" -[[ZipFile]] -deps = ["Libdl", "Printf", "Zlib_jll"] -git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7" -uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" -version = "0.9.3" - [[Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" @@ -1093,12 +864,6 @@ git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6" uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.0+0" -[[Zygote]] -deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "b1d95edd4e693066c38c13a10aab0a8f6a6e2f65" -uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.12" - [[ZygoteRules]] deps = ["MacroTools"] git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7" diff --git a/examples/kernel-ridge-regression/Project.toml b/examples/kernel-ridge-regression/Project.toml index f3b3a5b77..13219d95c 100644 --- a/examples/kernel-ridge-regression/Project.toml +++ b/examples/kernel-ridge-regression/Project.toml @@ -1,19 +1,13 @@ [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" -Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" -MLDataUtils = "cc2ba9b6-d476-5e6d-8eaf-a92d5412d41d" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Distributions = "0.25" -Flux = "0.12" KernelFunctions = "0.10" Literate = "2" -MLDataUtils = "0.5" Plots = "1" -Zygote = "0.6" julia = "1.3" diff --git a/examples/kernel-ridge-regression/script.jl b/examples/kernel-ridge-regression/script.jl index 5064a8a45..62afc5944 100644 --- a/examples/kernel-ridge-regression/script.jl +++ b/examples/kernel-ridge-regression/script.jl @@ -1,66 +1,162 @@ # # Kernel Ridge Regression # -# !!! warning -# This example is under construction - -# Setup +# Building on linear regression, we can fit non-linear data sets by introducing a feature space. In a higher-dimensional feature space, we can overfit the data; ridge regression introduces regularization to avoid this. In this notebook we show how we can use KernelFunctions.jl for *kernel* ridge regression. +## Loading and setup of required packages using KernelFunctions -using MLDataUtils -using Zygote -using Flux -using Distributions, LinearAlgebra -using Plots - -Flux.@functor SqExponentialKernel -Flux.@functor ScaleTransform -Flux.@functor KernelSum -Flux.@functor Matern32Kernel - -# Generate date - -xmin = -3; -xmax = 3; -x = range(xmin, xmax; length=100) -x_test = range(xmin, xmax; length=300) -x, y = noisy_function(sinc, x; noise=0.1) -X = RowVecs(reshape(x, :, 1)) -X_test = RowVecs(reshape(x_test, :, 1)) -#md nothing #hide - -# Set up kernel and regularisation parameter - -k = SqExponentialKernel() + Matern32Kernel() ∘ ScaleTransform(2.0) -λ = [-1.0] -#md nothing #hide +using LinearAlgebra +using Distributions -# +## Plotting +using Plots; +default(; lw=2.0, legendfontsize=11.0, ylims=(-150, 500)); -f(x, k, λ) = kernelmatrix(k, x, X) / (kernelmatrix(k, X) + exp(λ[1]) * I) * y -f(X, k, 1.0) +using Random: seed! +seed!(42); -# +## +# ## Toy data +# Here we use a one-dimensional toy problem. We generate data using the fourth-order polynomial $f(x) = (x+4)(x+1)(x-1)(x-3)$: + +f_truth(x) = (x + 4) * (x + 1) * (x - 1) * (x - 3) + +x_train = -5:0.5:5 +x_test = -7:0.1:7 + +noise = rand(Uniform(-20, 20), length(x_train)) +y_train = f_truth.(x_train) + noise +y_test = f_truth.(x_test) + +plot(x_test, y_test; label=raw"$f(x)$") +scatter!(x_train, y_train; seriescolor=1, label="observations") + +## +# ## Linear regression +# For training inputs $\mathrm{X}=(\mathbf{x}_n)_{n=1}^N$ and observations $\mathbf{y}=(y_n)_{n=1}^N$, the linear regression weights $\mathbf{w}$ using the least-squares estimator are given by +# ```math +# \mathbf{w} = (\mathrm{X}^\top \mathrm{X})^{-1} \mathrm{X}^\top \mathbf{y} +# ``` +# We predict at test inputs $\mathbf{x}_*$ using +# ```math +# \hat{y}_* = \mathbf{x}_*^\top \mathbf{w} +# ``` +# This is implemented by `linear_regression`: + +function linear_regression(X, y, Xstar) + weights = (X' * X) \ (X' * y) + return Xstar * weights +end; + +# A linear regression fit to the above data set: + +y_pred = linear_regression(x_train, y_train, x_test) +scatter(x_train, y_train; label="observations") +plot!(x_test, y_pred; label="linear fit") -loss(k, λ) = (ŷ -> sum(y - ŷ) / length(y) + exp(λ[1]) * norm(ŷ))(f(X, k, λ)) -loss(k, λ) +## +# ## Featurization +# We can improve the fit by including additional features, i.e. generalizing to $\tilde{\mathrm{X}} = (\phi(x_n))_{n=1}^N$, where $\phi(x)$ constructs a feature vector for each input $x$. Here we include powers of the input, $\phi(x) = (1, x, x^2, \dots, x^d)$: +function featurize_poly(x; degree=1) + return repeat(x, 1, degree + 1) .^ (0:degree)' +end + +function featurized_fit_and_plot(degree) + X = featurize_poly(x_train; degree=degree) + Xstar = featurize_poly(x_test; degree=degree) + y_pred = linear_regression(X, y_train, Xstar) + scatter(x_train, y_train; legend=false, title="fit of order $degree") + return plot!(x_test, y_pred) +end + +plot((featurized_fit_and_plot(degree) for degree in 1:4)...) + +## +# Note that the fit becomes perfect when we include exactly as many orders in the features as we have in the underlying polynomial (4). # +# However, when increasing the number of features, we can quickly overfit to noise in the data set: + +featurized_fit_and_plot(20) + +## +# ## Ridge regression +# To counteract this unwanted behaviour, we can introduce regularization. This leads to *ridge regression* with $L_2$ regularization of the weights ([Tikhonov regularization](https://en.wikipedia.org/wiki/Tikhonov_regularization)). +# Instead of the weights in linear regression, +# ```math +# \mathbf{w} = (\mathrm{X}^\top \mathrm{X})^{-1} \mathrm{X}^\top \mathbf{y} +# ``` +# we introduce the ridge parameter $\lambda$: +# ```math +# \mathbf{w} = (\mathrm{X}^\top \mathrm{X} + \lambda \mathbb{1})^{-1} \mathrm{X}^\top \mathbf{y} +# ``` +# As before, we predict at test inputs $\mathbf{x}_*$ using +# ```math +# \hat{y}_* = \mathbf{x}_*^\top \mathbf{w} +# ``` +# This is implemented by `ridge_regression`: + +function ridge_regression(X, y, Xstar, lambda) + weights = (X' * X + lambda * I) \ (X' * y) + return Xstar * weights +end -ps = Flux.params(k) -push!(ps, λ) -opt = Flux.Momentum(0.1) -#md nothing #hide - -plots = [] -for i in 1:10 - grads = Zygote.gradient(() -> loss(k, λ), ps) - Flux.Optimise.update!(opt, ps, grads) - p = Plots.scatter(x, y; lab="data", title="Loss = $(loss(k,λ))") - Plots.plot!(x_test, f(X_test, k, λ); lab="Prediction", lw=3.0) - push!(plots, p) +function regularized_fit_and_plot(degree, lambda) + X = featurize_poly(x_train; degree=degree) + Xstar = featurize_poly(x_test; degree=degree) + y_pred = ridge_regression(X, y_train, Xstar, lambda) + scatter(x_train, y_train; legend=false, title="\$\\lambda=$lambda\$") + return plot!(x_test, y_pred) end +plot((regularized_fit_and_plot(20, lambda) for lambda in (1e-3, 1e-2, 1e-1, 1))...) + +## +# ## Kernel ridge regression +# Instead of constructing the feature matrix explicitly, we can use *kernels* to replace inner products of feature vectors with a kernel evaluation: $\langle \phi(x), \phi(x') \rangle = k(x, x')$ or $\tilde{\mathrm{X}} \tilde{\mathrm{X}}^\top = \mathrm{K}$, where $\mathrm{K}_{ij} = k(x_i, x_j)$. +# +# To apply this "kernel trick" to ridge regression, we can rewrite the ridge estimate for the weights +# ```math +# \mathbf{w} = (\mathrm{X}^\top \mathrm{X} + \lambda \mathbb{1})^{-1} \mathrm{X}^\top \mathbf{y} +# ``` +# using the [matrix inversion lemma](https://tlienart.github.io/pub/csml/mtheory/matinvlem.html#basic_lemmas) +# as +# ```math +# \mathbf{w} = \mathrm{X}^\top (\mathrm{X} \mathrm{X}^\top + \lambda \mathbb{1})^{-1} \mathbf{y} +# ``` +# where we can now replace the inner product with the kernel matrix, +# ```math +# \mathbf{w} = \mathrm{X}^\top (\mathrm{K} + \lambda \mathbb{1})^{-1} \mathbf{y} +# ``` +# And the prediction yields another inner product, +# ```math +# \hat{y}_* = \mathbf{x}_*^\top \mathbf{w} = \langle \mathbf{x}_*, \mathbf{w} \rangle = \mathbf{k}_* (\mathrm{K} + \lambda \mathbb{1})^{-1} \mathbf{y} +# ``` +# where $(\mathbf{k}_*)_n = k(x_*, x_n)$. # +# This is implemented by `kernel_ridge_regression`: + +function kernel_ridge_regression(k, X, y, Xstar, lambda) + K = kernelmatrix(k, X) + kstar = kernelmatrix(k, Xstar, X) + return kstar * ((K + lambda * I) \ y) +end; + +# Now, instead of explicitly constructing features, we can simply pass in a `PolynomialKernel` object: + +function kernelized_fit_and_plot(kernel, lambda=1e-4) + y_pred = kernel_ridge_regression(kernel, x_train, y_train, x_test, lambda) + if kernel isa PolynomialKernel + title = string("order ", kernel.degree) + else + title = string(nameof(typeof(kernel))) + end + scatter(x_train, y_train; label=nothing) + return plot!(x_test, y_pred; label=nothing, title=title) +end + +plot((kernelized_fit_and_plot(PolynomialKernel(; degree=degree, c=1)) for degree in 1:4)...) + +## +# However, we can now also use kernels that would have an infinite-dimensional feature expansion, such as the squared exponential kernel: -l = @layout grid(10, 1) -plot(plots...; layout=l, size=(300, 1500)) +kernelized_fit_and_plot(SqExponentialKernel())