diff --git a/README.md b/README.md index 97979bc9..78b16cce 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +Note: Some type definitions were changed between BenchmarkTools v0.0.2 and v0.0.3. To deserialize old JLD-formatted data using a newer version of BenchmarkTools, use `BenchmarkTools.loadold(args...)` instead of `JLD.load(args...)`. + # BenchmarkTools.jl [![Build Status](https://travis-ci.org/JuliaCI/BenchmarkTools.jl.svg?branch=master)](https://travis-ci.org/JuliaCI/BenchmarkTools.jl) diff --git a/REQUIRE b/REQUIRE index 2d1525cc..b4fb5426 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,2 +1,3 @@ julia 0.4 Compat 0.7.20 +JLD 0.6.1 diff --git a/doc/manual.md b/doc/manual.md index cd71a120..9308c6db 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -841,7 +841,7 @@ Caching parameters in this manner leads to a far shorter turnaround time, and mo # Miscellaneous tips and info -- Times reported by BenchmarkTools are limited to nanosecond resolution, though derived estimates might report fractions of nanoseconds. +- BenchmarkTools restricts the minimum measurable benchmark execution time to one picosecond. - If you use `rand` or something similar to generate the values that are used in your benchmarks, you should seed the RNG (or provide a seeded RNG) so that the values are consistent between trials/samples/evaluations. - BenchmarkTools attempts to be robust against machine noise occurring between *samples*, but BenchmarkTools can't do very much about machine noise occurring between *trials*. To cut down on the latter kind of noise, it is advised that you dedicate CPUs and memory to the benchmarking Julia process by using a shielding tool such as [cset](http://manpages.ubuntu.com/manpages/precise/man1/cset.1.html). - On some machines, for some versions of BLAS and Julia, the number of BLAS worker threads can exceed the number of available cores. This can occasionally result in scheduling issues and inconsistent performance for BLAS-heavy benchmarks. To fix this issue, you can use `BLAS.set_num_threads(i::Int)` in the Julia REPL to ensure that the number of BLAS threads is equal to or less than the number of available cores. diff --git a/src/BenchmarkTools.jl b/src/BenchmarkTools.jl index c4fef316..5cfaa70a 100644 --- a/src/BenchmarkTools.jl +++ b/src/BenchmarkTools.jl @@ -1,6 +1,7 @@ module BenchmarkTools using Compat +import JLD # `show` compatibility for pre-JuliaLang/julia#16354 builds if VERSION < v"0.5.0-dev+4305" @@ -59,6 +60,12 @@ export tune!, @benchmark, @benchmarkable +########################### +# Backwards Compatibility # +########################### + +include("compat.jl") + ########################################## # Plotting Facilities (loaded on demand) # ########################################## diff --git a/src/compat.jl b/src/compat.jl new file mode 100644 index 00000000..7e8b5d1e --- /dev/null +++ b/src/compat.jl @@ -0,0 +1,42 @@ +############################################ +# Backwards-compatible JLD Deserialization # +############################################ + +type OldParameters + seconds::Float64 + samples::Int + evals::Int + overhead::Int + gctrial::Bool + gcsample::Bool + time_tolerance::Float64 + memory_tolerance::Float64 +end + +type OldTrial + params::Parameters + times::Vector{Float64} + gctimes::Vector{Float64} + memory::Int + allocs::Int +end + +function JLD.readas(p::OldParameters) + return Parameters(p.seconds, p.samples, p.evals, Float64(p.overhead), p.gctrial, + p.gcsample, p.time_tolerance, p.memory_tolerance) +end + +function JLD.readas(t::OldTrial) + new_times = convert(Vector{Float64}, t.times) + new_gctimes = convert(Vector{Float64}, t.gctimes) + return Trial(t.params, new_times, new_gctimes, t.memory, t.allocs) +end + +function loadold(args...) + JLD.translate("BenchmarkTools.Parameters", "BenchmarkTools.OldParameters") + JLD.translate("BenchmarkTools.Trial", "BenchmarkTools.OldTrial") + result = JLD.load(args...) + JLD.translate("BenchmarkTools.Parameters", "BenchmarkTools.Parameters") + JLD.translate("BenchmarkTools.Trial", "BenchmarkTools.Trial") + return result +end diff --git a/src/execution.jl b/src/execution.jl index 8d3703c8..9f499a3b 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -41,7 +41,7 @@ end function _lineartrial(b::Benchmark, p::Parameters = b.params; maxevals = RESOLUTION, kwargs...) params = Parameters(p; kwargs...) - estimates = zeros(Int, maxevals) + estimates = zeros(maxevals) completed = 0 params.gctrial && gc() start_time = time() @@ -111,7 +111,7 @@ end function tune!(b::Benchmark, p::Parameters = b.params; verbose::Bool = false, pad = "", kwargs...) - estimate = minimum(lineartrial(b, p; kwargs...)) + estimate = ceil(Int, minimum(lineartrial(b, p; kwargs...))) b.params.evals = guessevals(estimate) return b end @@ -234,8 +234,8 @@ macro benchmarkable(args...) __sample_time = time_ns() - __start_time __gcdiff = Base.GC_Diff(Base.gc_num(), __gc_start) $($(Expr(:quote, teardown))) - __time = max(Int(cld(__sample_time, __evals)) - __params.overhead, 1) - __gctime = max(Int(cld(__gcdiff.total_time, __evals)) - __params.overhead, 0) + __time = max((__sample_time / __evals) - __params.overhead, 0.001) + __gctime = max((__gcdiff.total_time / __evals) - __params.overhead, 0.0) __memory = Int(fld(__gcdiff.allocd, __evals)) __allocs = Int(fld(__gcdiff.malloc + __gcdiff.realloc + __gcdiff.poolalloc + __gcdiff.bigalloc, diff --git a/src/parameters.jl b/src/parameters.jl index 6ba04834..23e88a38 100644 --- a/src/parameters.jl +++ b/src/parameters.jl @@ -9,7 +9,7 @@ type Parameters seconds::Float64 samples::Int evals::Int - overhead::Int + overhead::Float64 gctrial::Bool gcsample::Bool time_tolerance::Float64 @@ -80,16 +80,13 @@ end nullfunc() end sample_time = time_ns() - start_time - return Int(cld(sample_time, evals)) + return sample_time / evals end function estimate_overhead() x = typemax(Int) for _ in 1:10000 - y = overhead_sample(RESOLUTION) - if y < x - x = y - end + x = min(x, overhead_sample(RESOLUTION)) end return x end diff --git a/src/trials.jl b/src/trials.jl index 64ad5d7c..f83f2d70 100644 --- a/src/trials.jl +++ b/src/trials.jl @@ -4,13 +4,13 @@ type Trial params::Parameters - times::Vector{Int} - gctimes::Vector{Int} + times::Vector{Float64} + gctimes::Vector{Float64} memory::Int allocs::Int end -Trial(params::Parameters) = Trial(params, Int[], Int[], typemax(Int), typemax(Int)) +Trial(params::Parameters) = Trial(params, Float64[], Float64[], typemax(Int), typemax(Int)) @compat function Base.:(==)(a::Trial, b::Trial) return a.params == b.params && @@ -246,7 +246,7 @@ function prettytime(t) else value, units = t / 1e9, "s" end - return string(@sprintf("%.2f", value), " ", units) + return string(@sprintf("%.3f", value), " ", units) end function prettymemory(b) diff --git a/test/CompatTests.jl b/test/CompatTests.jl new file mode 100644 index 00000000..3d6536a9 --- /dev/null +++ b/test/CompatTests.jl @@ -0,0 +1,16 @@ +module CompatTests + +using Base.Test +using BenchmarkTools +using JLD + +old_data = BenchmarkTools.loadold(joinpath(dirname(@__FILE__), "old_data.jld")) +new_data = JLD.load(joinpath(dirname(@__FILE__), "new_data.jld")) + +@test old_data["params"] == old_data["trial"].params +@test new_data["params"] == new_data["trial"].params + +@test old_data["params"] == new_data["params"] +@test old_data["trial"] == new_data["trial"] + +end # module diff --git a/test/TrialsTests.jl b/test/TrialsTests.jl index 387f0b94..3c235dae 100644 --- a/test/TrialsTests.jl +++ b/test/TrialsTests.jl @@ -154,12 +154,12 @@ tj_r_2 = judge(tr; time_tolerance = 2.0, memory_tolerance = 2.0) @test BenchmarkTools.prettydiff(1.0) == "+0.00%" @test BenchmarkTools.prettydiff(2.0) == "+100.00%" -@test BenchmarkTools.prettytime(999) == "999.00 ns" -@test BenchmarkTools.prettytime(1000) == "1.00 μs" -@test BenchmarkTools.prettytime(999_999) == "1000.00 μs" -@test BenchmarkTools.prettytime(1_000_000) == "1.00 ms" -@test BenchmarkTools.prettytime(999_999_999) == "1000.00 ms" -@test BenchmarkTools.prettytime(1_000_000_000) == "1.00 s" +@test BenchmarkTools.prettytime(999) == "999.000 ns" +@test BenchmarkTools.prettytime(1000) == "1.000 μs" +@test BenchmarkTools.prettytime(999_999) == "999.999 μs" +@test BenchmarkTools.prettytime(1_000_000) == "1.000 ms" +@test BenchmarkTools.prettytime(999_999_999) == "1000.000 ms" +@test BenchmarkTools.prettytime(1_000_000_000) == "1.000 s" @test BenchmarkTools.prettymemory(1023) == "1023.00 bytes" @test BenchmarkTools.prettymemory(1024) == "1.00 kb" diff --git a/test/new_data.jld b/test/new_data.jld new file mode 100644 index 00000000..3ef4f076 Binary files /dev/null and b/test/new_data.jld differ diff --git a/test/old_data.jld b/test/old_data.jld new file mode 100644 index 00000000..c686f878 Binary files /dev/null and b/test/old_data.jld differ diff --git a/test/runtests.jl b/test/runtests.jl index 69d7b7a2..d5fbf58d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,3 +13,8 @@ println("done (took ", toq(), " seconds)") print("Testing execution..."); tic() include("ExecutionTests.jl") println("done (took ", toq(), " seconds)") + + +print("Testing backwards compatibility..."); tic() +include("CompatTests.jl") +println("done (took ", toq(), " seconds)")