diff --git a/test/Project.toml b/test/Project.toml
index efb9fa08c3..6d7ba0b914 100644
--- a/test/Project.toml
+++ b/test/Project.toml
@@ -34,7 +34,7 @@ AdvancedPS = "0.2"
 AdvancedVI = "0.1"
 Clustering = "0.14"
 CmdStan = "6.0.8"
-Distributions = "0.23.8, 0.24, 0.25"
+Distributions = "< 0.25.11"
 DistributionsAD = "0.6.3"
 DynamicHMC = "2.1.6, 3.0"
 DynamicPPL = "0.12"
diff --git a/test/inference/Inference.jl b/test/inference/Inference.jl
index 52e2e1a76d..6d6955664e 100644
--- a/test/inference/Inference.jl
+++ b/test/inference/Inference.jl
@@ -1,4 +1,4 @@
-@testset "io.jl" begin
+@testset "inference.jl" begin
     # Only test threading if 1.3+.
     if VERSION > v"1.2"
         @testset "threaded sampling" begin
diff --git a/test/inference/gibbs.jl b/test/inference/gibbs.jl
index a9425b630b..9077aecbe8 100644
--- a/test/inference/gibbs.jl
+++ b/test/inference/gibbs.jl
@@ -50,19 +50,10 @@
         chain = sample(gdemo(1.5, 2.0), alg, 5_000)
         check_numerical(chain, [:s, :m], [49/24, 7/6], atol=0.1)
 
-        setadsafe(true)
-
         Random.seed!(200)
         gibbs = Gibbs(PG(15, :z1, :z2, :z3, :z4), HMC(0.15, 3, :mu1, :mu2))
         chain = sample(MoGtest_default, gibbs, 5_000)
         check_MoGtest_default(chain, atol=0.15)
-
-        setadsafe(false)
-
-        Random.seed!(200)
-        gibbs = Gibbs(PG(15, :z1, :z2, :z3, :z4), ESS(:mu1), ESS(:mu2))
-        chain = sample(MoGtest_default, gibbs, 5_000)
-        check_MoGtest_default(chain, atol=0.1)
     end
 
     @turing_testset "transitions" begin
diff --git a/test/runtests.jl b/test/runtests.jl
index f9fa2cb70b..0fa857dee3 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -46,22 +46,27 @@ include("test_utils/AllUtils.jl")
         include("core/ad.jl")
     end
 
+    @testset "samplers (without AD)" begin
+        include("inference/AdvancedSMC.jl")
+        include("inference/emcee.jl")
+        include("inference/ess.jl")
+        include("inference/is.jl")
+    end
+
     Turing.setrdcache(false)
     for adbackend in (:forwarddiff, :tracker, :reversediff)
         Turing.setadbackend(adbackend)
+        @info "Testing $(adbackend)"
+        start = time()
         @testset "inference: $adbackend" begin
             @testset "samplers" begin
                 include("inference/gibbs.jl")
                 include("inference/gibbs_conditional.jl")
                 include("inference/hmc.jl")
-                include("inference/is.jl")
-                include("inference/mh.jl")
-                include("inference/ess.jl")
-                include("inference/emcee.jl")
-                include("inference/AdvancedSMC.jl")
                 include("inference/Inference.jl")
                 include("contrib/inference/dynamichmc.jl")
                 include("contrib/inference/sghmc.jl")
+                include("inference/mh.jl")
             end
         end
 
@@ -72,6 +77,11 @@ include("test_utils/AllUtils.jl")
         @testset "modes" begin
             include("modes/ModeEstimation.jl")
         end
+
+        # Useful for
+        # a) discovering performance regressions,
+        # b) figuring out why CI is timing out.
+        @info "Tests for $(adbackend) took $(time() - start) seconds"
     end
     @testset "variational optimisers" begin
         include("variational/optimisers.jl")