From 138b9b2c19a04bd21ffd48909d57c6993f98abd7 Mon Sep 17 00:00:00 2001 From: Jarrett Revels Date: Sat, 18 Jun 2016 13:11:06 -0400 Subject: [PATCH 1/2] loosen nanosecond resolution restriction, add compat layer for deserializing old types --- README.md | 2 ++ REQUIRE | 1 + doc/manual.md | 2 +- src/BenchmarkTools.jl | 7 +++++++ src/compat.jl | 42 ++++++++++++++++++++++++++++++++++++++++++ src/execution.jl | 8 ++++---- src/parameters.jl | 9 +++------ src/trials.jl | 8 ++++---- test/CompatTests.jl | 16 ++++++++++++++++ test/TrialsTests.jl | 12 ++++++------ test/new_data.jld | Bin 0 -> 174936 bytes test/old_data.jld | Bin 0 -> 174928 bytes test/runtests.jl | 5 +++++ 13 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 src/compat.jl create mode 100644 test/CompatTests.jl create mode 100644 test/new_data.jld create mode 100644 test/old_data.jld diff --git a/README.md b/README.md index 97979bc9..78b16cce 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +Note: Some type definitions were changed between BenchmarkTools v0.0.2 and v0.0.3. To deserialize old JLD-formatted data using a newer version of BenchmarkTools, use `BenchmarkTools.loadold(args...)` instead of `JLD.load(args...)`. + # BenchmarkTools.jl [![Build Status](https://travis-ci.org/JuliaCI/BenchmarkTools.jl.svg?branch=master)](https://travis-ci.org/JuliaCI/BenchmarkTools.jl) diff --git a/REQUIRE b/REQUIRE index 2d1525cc..b4fb5426 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,2 +1,3 @@ julia 0.4 Compat 0.7.20 +JLD 0.6.1 diff --git a/doc/manual.md b/doc/manual.md index cd71a120..9308c6db 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -841,7 +841,7 @@ Caching parameters in this manner leads to a far shorter turnaround time, and mo # Miscellaneous tips and info -- Times reported by BenchmarkTools are limited to nanosecond resolution, though derived estimates might report fractions of nanoseconds. +- BenchmarkTools restricts the minimum measurable benchmark execution time to one picosecond. - If you use `rand` or something similar to generate the values that are used in your benchmarks, you should seed the RNG (or provide a seeded RNG) so that the values are consistent between trials/samples/evaluations. - BenchmarkTools attempts to be robust against machine noise occurring between *samples*, but BenchmarkTools can't do very much about machine noise occurring between *trials*. To cut down on the latter kind of noise, it is advised that you dedicate CPUs and memory to the benchmarking Julia process by using a shielding tool such as [cset](http://manpages.ubuntu.com/manpages/precise/man1/cset.1.html). - On some machines, for some versions of BLAS and Julia, the number of BLAS worker threads can exceed the number of available cores. This can occasionally result in scheduling issues and inconsistent performance for BLAS-heavy benchmarks. To fix this issue, you can use `BLAS.set_num_threads(i::Int)` in the Julia REPL to ensure that the number of BLAS threads is equal to or less than the number of available cores. diff --git a/src/BenchmarkTools.jl b/src/BenchmarkTools.jl index c4fef316..5cfaa70a 100644 --- a/src/BenchmarkTools.jl +++ b/src/BenchmarkTools.jl @@ -1,6 +1,7 @@ module BenchmarkTools using Compat +import JLD # `show` compatibility for pre-JuliaLang/julia#16354 builds if VERSION < v"0.5.0-dev+4305" @@ -59,6 +60,12 @@ export tune!, @benchmark, @benchmarkable +########################### +# Backwards Compatibility # +########################### + +include("compat.jl") + ########################################## # Plotting Facilities (loaded on demand) # ########################################## diff --git a/src/compat.jl b/src/compat.jl new file mode 100644 index 00000000..7e8b5d1e --- /dev/null +++ b/src/compat.jl @@ -0,0 +1,42 @@ +############################################ +# Backwards-compatible JLD Deserialization # +############################################ + +type OldParameters + seconds::Float64 + samples::Int + evals::Int + overhead::Int + gctrial::Bool + gcsample::Bool + time_tolerance::Float64 + memory_tolerance::Float64 +end + +type OldTrial + params::Parameters + times::Vector{Float64} + gctimes::Vector{Float64} + memory::Int + allocs::Int +end + +function JLD.readas(p::OldParameters) + return Parameters(p.seconds, p.samples, p.evals, Float64(p.overhead), p.gctrial, + p.gcsample, p.time_tolerance, p.memory_tolerance) +end + +function JLD.readas(t::OldTrial) + new_times = convert(Vector{Float64}, t.times) + new_gctimes = convert(Vector{Float64}, t.gctimes) + return Trial(t.params, new_times, new_gctimes, t.memory, t.allocs) +end + +function loadold(args...) + JLD.translate("BenchmarkTools.Parameters", "BenchmarkTools.OldParameters") + JLD.translate("BenchmarkTools.Trial", "BenchmarkTools.OldTrial") + result = JLD.load(args...) + JLD.translate("BenchmarkTools.Parameters", "BenchmarkTools.Parameters") + JLD.translate("BenchmarkTools.Trial", "BenchmarkTools.Trial") + return result +end diff --git a/src/execution.jl b/src/execution.jl index 8d3703c8..9f499a3b 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -41,7 +41,7 @@ end function _lineartrial(b::Benchmark, p::Parameters = b.params; maxevals = RESOLUTION, kwargs...) params = Parameters(p; kwargs...) - estimates = zeros(Int, maxevals) + estimates = zeros(maxevals) completed = 0 params.gctrial && gc() start_time = time() @@ -111,7 +111,7 @@ end function tune!(b::Benchmark, p::Parameters = b.params; verbose::Bool = false, pad = "", kwargs...) - estimate = minimum(lineartrial(b, p; kwargs...)) + estimate = ceil(Int, minimum(lineartrial(b, p; kwargs...))) b.params.evals = guessevals(estimate) return b end @@ -234,8 +234,8 @@ macro benchmarkable(args...) __sample_time = time_ns() - __start_time __gcdiff = Base.GC_Diff(Base.gc_num(), __gc_start) $($(Expr(:quote, teardown))) - __time = max(Int(cld(__sample_time, __evals)) - __params.overhead, 1) - __gctime = max(Int(cld(__gcdiff.total_time, __evals)) - __params.overhead, 0) + __time = max((__sample_time / __evals) - __params.overhead, 0.001) + __gctime = max((__gcdiff.total_time / __evals) - __params.overhead, 0.0) __memory = Int(fld(__gcdiff.allocd, __evals)) __allocs = Int(fld(__gcdiff.malloc + __gcdiff.realloc + __gcdiff.poolalloc + __gcdiff.bigalloc, diff --git a/src/parameters.jl b/src/parameters.jl index 6ba04834..23e88a38 100644 --- a/src/parameters.jl +++ b/src/parameters.jl @@ -9,7 +9,7 @@ type Parameters seconds::Float64 samples::Int evals::Int - overhead::Int + overhead::Float64 gctrial::Bool gcsample::Bool time_tolerance::Float64 @@ -80,16 +80,13 @@ end nullfunc() end sample_time = time_ns() - start_time - return Int(cld(sample_time, evals)) + return sample_time / evals end function estimate_overhead() x = typemax(Int) for _ in 1:10000 - y = overhead_sample(RESOLUTION) - if y < x - x = y - end + x = min(x, overhead_sample(RESOLUTION)) end return x end diff --git a/src/trials.jl b/src/trials.jl index 64ad5d7c..f83f2d70 100644 --- a/src/trials.jl +++ b/src/trials.jl @@ -4,13 +4,13 @@ type Trial params::Parameters - times::Vector{Int} - gctimes::Vector{Int} + times::Vector{Float64} + gctimes::Vector{Float64} memory::Int allocs::Int end -Trial(params::Parameters) = Trial(params, Int[], Int[], typemax(Int), typemax(Int)) +Trial(params::Parameters) = Trial(params, Float64[], Float64[], typemax(Int), typemax(Int)) @compat function Base.:(==)(a::Trial, b::Trial) return a.params == b.params && @@ -246,7 +246,7 @@ function prettytime(t) else value, units = t / 1e9, "s" end - return string(@sprintf("%.2f", value), " ", units) + return string(@sprintf("%.3f", value), " ", units) end function prettymemory(b) diff --git a/test/CompatTests.jl b/test/CompatTests.jl new file mode 100644 index 00000000..3d6536a9 --- /dev/null +++ b/test/CompatTests.jl @@ -0,0 +1,16 @@ +module CompatTests + +using Base.Test +using BenchmarkTools +using JLD + +old_data = BenchmarkTools.loadold(joinpath(dirname(@__FILE__), "old_data.jld")) +new_data = JLD.load(joinpath(dirname(@__FILE__), "new_data.jld")) + +@test old_data["params"] == old_data["trial"].params +@test new_data["params"] == new_data["trial"].params + +@test old_data["params"] == new_data["params"] +@test old_data["trial"] == new_data["trial"] + +end # module diff --git a/test/TrialsTests.jl b/test/TrialsTests.jl index 387f0b94..6715e261 100644 --- a/test/TrialsTests.jl +++ b/test/TrialsTests.jl @@ -154,12 +154,12 @@ tj_r_2 = judge(tr; time_tolerance = 2.0, memory_tolerance = 2.0) @test BenchmarkTools.prettydiff(1.0) == "+0.00%" @test BenchmarkTools.prettydiff(2.0) == "+100.00%" -@test BenchmarkTools.prettytime(999) == "999.00 ns" -@test BenchmarkTools.prettytime(1000) == "1.00 μs" -@test BenchmarkTools.prettytime(999_999) == "1000.00 μs" -@test BenchmarkTools.prettytime(1_000_000) == "1.00 ms" -@test BenchmarkTools.prettytime(999_999_999) == "1000.00 ms" -@test BenchmarkTools.prettytime(1_000_000_000) == "1.00 s" +@test BenchmarkTools.prettytime(999) == "999.000 ns" +@test BenchmarkTools.prettytime(1000) == "1.000 μs" +@test BenchmarkTools.prettytime(999_999) == "1000.000 μs" +@test BenchmarkTools.prettytime(1_000_000) == "1.000 ms" +@test BenchmarkTools.prettytime(999_999_999) == "1000.000 ms" +@test BenchmarkTools.prettytime(1_000_000_000) == "1.000 s" @test BenchmarkTools.prettymemory(1023) == "1023.00 bytes" @test BenchmarkTools.prettymemory(1024) == "1.00 kb" diff --git a/test/new_data.jld b/test/new_data.jld new file mode 100644 index 0000000000000000000000000000000000000000..3ef4f0766a24651ebdb787ea8592bb0d3a221220 GIT binary patch literal 174936 zcmeI&&2tpT8G!MgT_7@$ig1X7Q~B7$Nm-^SEHF+;a$vzI2!cTcxKQOHZ;7=b7Dy!& z7q%}Pe84v!W1oDKD^;!>eata`0!MSQk3RZFW~RHvYV<`Y5(NB)+I_csx_i2x>3K%X z^r^qiTOeiSL*N^vpOFGBS%ggq&LS8Bgm?!vjTu~3%G z?dO=Pg|D{8Un4m~e5weHj~$(;20WQYeK^H`r+Jo=VQPB5-&>fSUrH~$H`l)&!qj~K ztulm#`76DdGS2npdsoYhJa0R>LL(O8xn=WrIL+VgP%Kz?s{9$}Z!muw{3IiR=KM|5 zC)`S2%bfCj%=0SeKWIFE=IrqzYz_H%opx*~A7?zJFQs!Pd%k1m7(2c_jXw_~ z`*ZP`nA6QWQv6?V-ZF5E$CY#D7IwtJbO**%{oIz$Lors5ghubn+zkGTR?Zh8Fa_7&#l`*BD5zWsw`NeKiHKmY**5I_I{ z1Q0*~0R#|u3<%`sEcc$j#wXCln$KDO`{%LwnL>qQ<#UyfHzlR~d}YrsV)wd@Uaow8 zb?l9kr;jiGG?iP)H9m*REFR>!_Fx}Ya{R2(!_Aq@T<1Emf89nm2e>|S@$9+dQ|C|q zWi0%BsnmUQEo;Q*)1A8Rz2|@VeL*}Q?XpzSYHwM|b8+dIkHe1vZuvR&(Nky7jeIY4 zeqjIhu`|a{9z8R4?Ck54X?9o7{Z?O3(0uQ-(RzZ9e;p4v@;E@#$A?a!`93zUC)o4C z@vg&fe_gj7BWV>$qC7klz<%&)oh^EI64P2;VkpJ{`&UzWUzV z(YeO?Xi~`Up}$S#`X7Oe3)q68Dv(shgI%^lN_YA&^ z-x<1fUQxb49V-9*wO;pFEe?u}{ICqW%TK=ij`goiU%uL#f9JyN?9BE37t*q!HSa&o z^${E0f5QHF+&nL1%J2#5>`7*Gcv6 zJ6|WS&Cd7tA1kksGOyZ-^6pl>lbuZK^2*<>+C!yCs+;EZ{`Bm%OO;vfZ`;*5k{r|x zhjxTRcgqH^_pZ*()OLz(Y~-jx5A@&d%~VI<$WMcHX3NFUm;1d-xzqWcv(^Y!>vQSH zN+DDE4~NcLl~aB`A8${`(dLT3yw7@-ji(l7XZrKKYt#Lykd|YuS$nlUCpQnY602`? z3tg?PongGLyxzQezATaFRl~m%)pE`LLb)hCjUfO3>Ts&C+Ip;N9=2M`@AxbKe--C@ zyPlm@!R{|k+x?+^N@XEe_P!{6Cw2Z+*FM z{XXZpKmPLi%meelb-)8WzymzM13bV3Jir4yzymzM z13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3 zJir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4y zzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM z13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3 zJir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4y zzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM z13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3 zJir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4yzymzM13bV3Jir4y zzymzM13a*qJg~1D&&Q*R`_u8ocpjc~4x{OOKWIFV>gRYcr8kPNq;k!8DCMseUrqae z`;g-AQag_-9!|%9QseK__#XH8hcpf`)Ef>sR0JM$9{jNz`E8bXD)#5*pHsiibc?Vp z6jdkduPW@RQlThf#_)@EGUV7U_g8y%b&iCgdRI3b+7S-jZO3w3%iL;ztJSI-7)s^a z#qGJ+;?2V`#`4dmR;s)^AN4PWa3k(`MBkamNa|!U(7&on?J!(^s`1UeFxbVVaTj^q zqxJ4~7ztnQizUZjj`8;8K>~xVgs_swXUhw0(p2V_Pwv{>CiV^vVI{Z2_3m~!n2PQ0 z=DGKEnq#?s5kCE-+CD$>!R{|k+63q+ywdAj>do~!?_QbgcSepM{ljlx-P8G?KYQ)U^wrM3y>IM&BiNo)0)H*9_p_T{ z+FXxsXeew5!*Qss7`{*2*xzGDPG=tPq@5zg8!67G`bDVzg|I8-Z%ger%bh_nES7^i{>|8W|htvG+4#k3Xr^?+pe}nnk;3pXgH0N)c zKH*04TIQ7JW1d$z|3Txq(`SwrVPnYW>$GD_`8>x{`f}<=C!J{DD~rorOzFS88uPQy zZ>I{my{}UG?k{4weKCYx@8=RJqY<*#zrM11xkk1BTe1k7Y7Y#h?iGn>?rppDLbX3! zf{v`ab>AxTReq~z)kTMrKiUgdq%Ib=5488s8~qC*+=>U*`Dbh4&YthsImV7}PUFwR z$o`yvCgybWffWB2oHq=d<8kGjxrHrpFx`PMRlA$gbts05EhpbTb>h(YTPIGRIeRmI z?{RGaOUcajovktBc%#@-u49MJA36T-G=0nOcJ#t<52`U%%G+Ij z441L`iDlxIy620SJ(Sw3{8FiUKJ_!VyL3L^C`+xME%W`yPhy|CjjnA9ojRwSx5IJX zj-4GnJ9=vL(7DlWn(I}8V@J-Ms>fUbE4MuUbo&Z3v;BA=eSY=aqND@@2q1s}0tg_0 z00IagfB*srJO%{vbC!G0U*i+#V#Vhy|NZmW{7j+3vGlph&2>pBKVRAPv)H|EqnAsc zUmbq?#Hpk6KThSAa*fZSGK&X!u07a?r5tZHdbmE5nd?j^_OILM<^b1cE}S`gbo|_j zKaPfdCo7K^KsZ2;Fh0LA3AyF?8pyN z=Le2&9zA{Z#G%vUhtIrqG0pDkx!>~Z37YSn)>=<+^Oy02BaZ_#eSGKyn(t%tdV*aq zFFw`2xGgM!O6fOqsg(XrPG@g+az-N-<@dN+vXI{wWY2u{t61=2ZXkTusQGjxO zz5CAJlOxl!{k@0F??{bqA@oT-QnaSEtv5mDHHRyr<2ffMa>}&aHu+DV382WO*cPV!|-*Z+P!C>9e zi!7Kh@;FEe}13!DhrR#O;7e`dsip=;~_1>TCw(We3O$0T8Y)y zx^*ts*4A*muDsfO+Fh2&^Qqz8v1*xSf394No<@-Wt~!t^EVmA;nt!d<;yeA)zpLUr zZ`ZT3YJ5j`IIVH}wrXHtE5c`=R@>(XKi=`hv2Q;5QW(bda8?nm={%oDh|{wL3=9M^oF(#n7L$8|#OVr*YzbUjb` zIyv|pAkM$*>jLGosTX1dfddW|0rS8-Fb{kWcz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7 zcz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3Z zfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh z2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7 zcz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3Z zfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh z2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7 zcz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3Z zfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSh2Y7%7cz_3ZfCqSB zeR$x9u^Zu0#plxb#R%d_=P;bE_k+f5seX>zQ+loVd@9$BJ5v5i@r87J=R=Awrgk1x z+?CG%q{iK8e2;s4DUCynAaKB;BJiN|;71W(STAvZ>d)^|{C$c=*c6Itzy7E~G<8wL zjNw;e!5rIRxGp+86hisG+`9GW#xhmaXti2(#i6pDYNuV?nTai3KM-SGJZw+3tGq97 z^e=>PJsx<(pP9!<>TEI4&#G(H_}gV{wPs!(?CQe!i#+~)_2G6H3AgvelHJ#1ymNVw z#9%9787{XL7g(pM%q?H!mAQ?*HaLW(+z!--+u_YrY)3cGy>Bb<16vV3`?T6VKlt&E zFOGfl(I*F&<2OH0!!7~{Ab Date: Thu, 23 Jun 2016 11:21:20 -0400 Subject: [PATCH 2/2] fix pretty printing test --- test/TrialsTests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TrialsTests.jl b/test/TrialsTests.jl index 6715e261..3c235dae 100644 --- a/test/TrialsTests.jl +++ b/test/TrialsTests.jl @@ -156,7 +156,7 @@ tj_r_2 = judge(tr; time_tolerance = 2.0, memory_tolerance = 2.0) @test BenchmarkTools.prettytime(999) == "999.000 ns" @test BenchmarkTools.prettytime(1000) == "1.000 μs" -@test BenchmarkTools.prettytime(999_999) == "1000.000 μs" +@test BenchmarkTools.prettytime(999_999) == "999.999 μs" @test BenchmarkTools.prettytime(1_000_000) == "1.000 ms" @test BenchmarkTools.prettytime(999_999_999) == "1000.000 ms" @test BenchmarkTools.prettytime(1_000_000_000) == "1.000 s"