Skip to content

Commit 8cd9d22

Browse files
vtjnashvtjnash
committed
use a non-stack-based algorithm to compute summarysize
in addition to not using stack space, this doesn't need as many special cases
1 parent 95cb94c commit 8cd9d22

File tree

4 files changed

+165
-150
lines changed

4 files changed

+165
-150
lines changed

‎base/interactiveutil.jl‎

Lines changed: 0 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -716,151 +716,3 @@ function whos(io::IO=STDOUT, m::Module=current_module(), pattern::Regex=r"")
716716
end
717717
whos(m::Module, pat::Regex=r"") = whos(STDOUT, m, pat)
718718
whos(pat::Regex) = whos(STDOUT, current_module(), pat)
719-
720-
#################################################################################
721-
722-
"""
723-
Base.summarysize(obj; exclude=Union{Module,DataType,TypeName}) -> Int
724-
725-
Compute the amount of memory used by all unique objects reachable from the argument.
726-
Keyword argument `exclude` specifies a type of objects to exclude from the traversal.
727-
"""
728-
summarysize(obj; exclude = Union{Module,DataType,TypeName}) =
729-
summarysize(obj, ObjectIdDict(), exclude)
730-
731-
summarysize(obj::Symbol, seen, excl) = 0
732-
733-
function summarysize(obj::UnionAll, seen, excl)
734-
return 2*sizeof(Int) + summarysize(obj.body, seen, excl) + summarysize(obj.var, seen, excl)
735-
end
736-
737-
function summarysize(obj::DataType, seen, excl)
738-
key = pointer_from_objref(obj)
739-
haskey(seen, key) ? (return 0) : (seen[key] = true)
740-
size = 7*sizeof(Int) + 6*sizeof(Int32) + 4*nfields(obj) + ifelse(Sys.WORD_SIZE == 64, 4, 0)
741-
size += summarysize(obj.parameters, seen, excl)::Int
742-
size += summarysize(obj.types, seen, excl)::Int
743-
return size
744-
end
745-
746-
function summarysize(obj::TypeName, seen, excl)
747-
key = pointer_from_objref(obj)
748-
haskey(seen, key) ? (return 0) : (seen[key] = true)
749-
return Core.sizeof(obj) + (isdefined(obj,:mt) ? summarysize(obj.mt, seen, excl) : 0)
750-
end
751-
752-
summarysize(obj::ANY, seen, excl) = _summarysize(obj, seen, excl)
753-
# define the general case separately to make sure it is not specialized for every type
754-
function _summarysize(obj::ANY, seen, excl)
755-
key = pointer_from_objref(obj)
756-
haskey(seen, key) ? (return 0) : (seen[key] = true)
757-
size = Core.sizeof(obj)
758-
ft = typeof(obj).types
759-
for i in 1:nfields(obj)
760-
if !isbits(ft[i]) && isdefined(obj,i)
761-
val = getfield(obj, i)
762-
if !isa(val,excl)
763-
size += summarysize(val, seen, excl)::Int
764-
end
765-
end
766-
end
767-
return size
768-
end
769-
770-
function summarysize(obj::Array, seen, excl)
771-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
772-
size = Core.sizeof(obj)
773-
# TODO: add size of jl_array_t
774-
if !isbits(eltype(obj))
775-
for i in 1:length(obj)
776-
if ccall(:jl_array_isassigned, Cint, (Any, UInt), obj, i-1) == 1
777-
val = obj[i]
778-
if !isa(val, excl)
779-
size += summarysize(val, seen, excl)::Int
780-
end
781-
end
782-
end
783-
end
784-
return size
785-
end
786-
787-
summarysize(s::String, seen, excl) = sizeof(Int) + sizeof(s)
788-
789-
function summarysize(obj::SimpleVector, seen, excl)
790-
key = pointer_from_objref(obj)
791-
haskey(seen, key) ? (return 0) : (seen[key] = true)
792-
size = Core.sizeof(obj)
793-
for i in 1:length(obj)
794-
if isassigned(obj, i)
795-
val = obj[i]
796-
if !isa(val, excl)
797-
size += summarysize(val, seen, excl)::Int
798-
end
799-
end
800-
end
801-
return size
802-
end
803-
804-
function summarysize(obj::Module, seen, excl)
805-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
806-
size::Int = Core.sizeof(obj)
807-
for binding in names(obj, true)
808-
if isdefined(obj, binding) && !isdeprecated(obj, binding)
809-
value = getfield(obj, binding)
810-
if !isa(value, Module) || module_parent(value) === obj
811-
size += summarysize(value, seen, excl)::Int
812-
vt = isa(value,DataType) ? value : typeof(value)
813-
if vt.name.module === obj
814-
if vt !== value
815-
size += summarysize(vt, seen, excl)::Int
816-
end
817-
# charge a TypeName to its module
818-
size += summarysize(vt.name, seen, excl)::Int
819-
end
820-
end
821-
end
822-
end
823-
return size
824-
end
825-
826-
function summarysize(obj::Task, seen, excl)
827-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
828-
size::Int = Core.sizeof(obj)
829-
if isdefined(obj, :code)
830-
size += summarysize(obj.code, seen, excl)::Int
831-
end
832-
size += summarysize(obj.storage, seen, excl)::Int
833-
size += summarysize(obj.backtrace, seen, excl)::Int
834-
size += summarysize(obj.donenotify, seen, excl)::Int
835-
size += summarysize(obj.exception, seen, excl)::Int
836-
size += summarysize(obj.result, seen, excl)::Int
837-
# TODO: add stack size, and possibly traverse stack roots
838-
return size
839-
end
840-
841-
function summarysize(obj::MethodTable, seen, excl)
842-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
843-
size::Int = Core.sizeof(obj)
844-
size += summarysize(obj.defs, seen, excl)::Int
845-
size += summarysize(obj.cache, seen, excl)::Int
846-
if isdefined(obj, :kwsorter)
847-
size += summarysize(obj.kwsorter, seen, excl)::Int
848-
end
849-
return size
850-
end
851-
852-
function summarysize(m::TypeMapEntry, seen, excl)
853-
size::Int = 0
854-
while true # specialized to prevent stack overflow while following this linked list
855-
haskey(seen, m) ? (return size) : (seen[m] = true)
856-
size += Core.sizeof(m)
857-
if isdefined(m, :func)
858-
size += summarysize(m.func, seen, excl)::Int
859-
end
860-
size += summarysize(m.sig, seen, excl)::Int
861-
size += summarysize(m.tvars, seen, excl)::Int
862-
m.next === nothing && break
863-
m = m.next::TypeMapEntry
864-
end
865-
return size
866-
end

‎base/summarysize.jl‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
immutable SummarySize
2+
seen::ObjectIdDict
3+
frontier_x::Vector{Any}
4+
frontier_i::Vector{Int}
5+
exclude::Any
6+
chargeall::Any
7+
end
8+
9+
10+
"""
11+
Base.summarysize(obj; exclude=Union{...}, chargeall=Union{...}) -> Int
12+
13+
Compute the amount of memory used by all unique objects reachable from the argument.
14+
15+
# Keyword Arguments
16+
* `exclude`: specifies the types of objects to exclude from the traversal.
17+
* `chargeall`: specifies the types of objects to always charge the size of all of their fields,
18+
even if those fields would normally be excluded.
19+
"""
20+
function summarysize(obj::ANY;
21+
exclude::ANY = Union{DataType, TypeName, Method},
22+
chargeall::ANY = Union{TypeMapEntry, Core.MethodInstance})
23+
ss = SummarySize(ObjectIdDict(), Any[], Int[], exclude, chargeall)
24+
size::Int = ss(obj)
25+
while !isempty(ss.frontier_x)
26+
# DFS heap traversal of everything without a specialization
27+
# BFS heap traversal of anything with a specialization
28+
x = ss.frontier_x[end]
29+
i = ss.frontier_i[end]
30+
val = nothing
31+
if isa(x, SimpleVector)
32+
nf = length(x)
33+
if isassigned(x, i)
34+
val = x[i]
35+
end
36+
elseif isa(x, Array)
37+
nf = length(x)
38+
if ccall(:jl_array_isassigned, Cint, (Any, UInt), x, i - 1) != 0
39+
val = x[i]
40+
end
41+
else
42+
nf = nfields(x)
43+
ft = typeof(x).types
44+
if !isbits(ft[i]) && isdefined(x, i)
45+
val = getfield(x, i)
46+
end
47+
end
48+
if nf > i
49+
ss.frontier_i[end] = i + 1
50+
else
51+
pop!(ss.frontier_x)
52+
pop!(ss.frontier_i)
53+
end
54+
if val !== nothing && !isa(val, Module) && (!isa(val, ss.exclude) || isa(x, ss.chargeall))
55+
size += ss(val)::Int
56+
end
57+
end
58+
return size
59+
end
60+
61+
(ss::SummarySize)(obj::ANY) = _summarysize(ss, obj)
62+
# define the general case separately to make sure it is not specialized for every type
63+
@noinline function _summarysize(ss::SummarySize, obj::ANY)
64+
key = pointer_from_objref(obj)
65+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
66+
if nfields(obj) > 0
67+
push!(ss.frontier_x, obj)
68+
push!(ss.frontier_i, 1)
69+
end
70+
if isa(obj, UnionAll)
71+
# black-list of items that don't have a Core.sizeof
72+
return 2 * sizeof(Int)
73+
end
74+
return Core.sizeof(obj)
75+
end
76+
77+
(::SummarySize)(obj::Symbol) = 0
78+
(::SummarySize)(obj::SummarySize) = 0
79+
(::SummarySize)(obj::String) = Core.sizeof(Int) + Core.sizeof(obj)
80+
81+
function (ss::SummarySize)(obj::DataType)
82+
key = pointer_from_objref(obj)
83+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
84+
size::Int = 7 * Core.sizeof(Int) + 6 * Core.sizeof(Int32)
85+
size += 4 * nfields(obj) + ifelse(Sys.WORD_SIZE == 64, 4, 0)
86+
size += ss(obj.parameters)::Int
87+
size += ss(obj.types)::Int
88+
return size
89+
end
90+
91+
function (ss::SummarySize)(obj::TypeName)
92+
key = pointer_from_objref(obj)
93+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
94+
return Core.sizeof(obj) + (isdefined(obj, :mt) ? ss(obj.mt) : 0)
95+
end
96+
97+
function (ss::SummarySize)(obj::Array)
98+
haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true)
99+
size::Int = Core.sizeof(obj)
100+
# TODO: add size of jl_array_t
101+
if !isbits(eltype(obj)) && !isempty(obj)
102+
push!(ss.frontier_x, obj)
103+
push!(ss.frontier_i, 1)
104+
end
105+
return size
106+
end
107+
108+
function (ss::SummarySize)(obj::SimpleVector)
109+
key = pointer_from_objref(obj)
110+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
111+
size::Int = Core.sizeof(obj)
112+
if !isempty(obj)
113+
push!(ss.frontier_x, obj)
114+
push!(ss.frontier_i, 1)
115+
end
116+
return size
117+
end
118+
119+
function (ss::SummarySize)(obj::Module)
120+
haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true)
121+
size::Int = Core.sizeof(obj)
122+
for binding in names(obj, true)
123+
if isdefined(obj, binding) && !isdeprecated(obj, binding)
124+
value = getfield(obj, binding)
125+
if !isa(value, Module) || module_parent(value) === obj
126+
size += ss(value)::Int
127+
if isa(value, UnionAll)
128+
value = unwrap_unionall(value)
129+
end
130+
if isa(value, DataType) && value.name.module === obj && value.name.name === binding
131+
# charge a TypeName to its module (but not to the type)
132+
size += ss(value.name)::Int
133+
end
134+
end
135+
end
136+
end
137+
return size
138+
end
139+
140+
function (ss::SummarySize)(obj::Task)
141+
haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true)
142+
size::Int = Core.sizeof(obj)
143+
if isdefined(obj, :code)
144+
size += ss(obj.code)::Int
145+
end
146+
size += ss(obj.storage)::Int
147+
size += ss(obj.backtrace)::Int
148+
size += ss(obj.donenotify)::Int
149+
size += ss(obj.exception)::Int
150+
size += ss(obj.result)::Int
151+
# TODO: add stack size, and possibly traverse stack roots
152+
return size
153+
end

‎base/sysimg.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,7 @@ include("datafmt.jl")
300300
importall .DataFmt
301301
include("deepcopy.jl")
302302
include("interactiveutil.jl")
303+
include("summarysize.jl")
303304
include("replutil.jl")
304305
include("test.jl")
305306
include("i18n.jl")

‎test/misc.jl‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,8 +293,17 @@ v11801, t11801 = @timed sin(1)
293293
# interactive utilities
294294

295295
import Base.summarysize
296-
@test summarysize(Core) > summarysize(Core.Inference) > Core.sizeof(Core)
297-
@test summarysize(Base) > 10_000*sizeof(Int)
296+
@test summarysize(Core) > (summarysize(Core.Inference) + Base.summarysize(Core.Intrinsics)) > Core.sizeof(Core)
297+
@test summarysize(Base) > 100_000 * sizeof(Ptr)
298+
299+
let R = Ref{Any}(nothing), depth = 10^6
300+
for i = 1:depth
301+
R = Ref{Any}(R)
302+
end
303+
R = Core.svec(R, R)
304+
@test summarysize(R) == (depth + 4) * sizeof(Ptr)
305+
end
306+
298307
module _test_whos_
299308
export x
300309
x = 1.0

0 commit comments

Comments
 (0)