Skip to content

Commit a62e179

Browse files
vtjnashvtjnash
committed
use a non-stack-based algorithm to compute summarysize
in addition to not using stack space, this doesn't need as many special cases
1 parent 95cb94c commit a62e179

File tree

4 files changed

+151
-150
lines changed

4 files changed

+151
-150
lines changed

‎base/interactiveutil.jl‎

Lines changed: 0 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -716,151 +716,3 @@ function whos(io::IO=STDOUT, m::Module=current_module(), pattern::Regex=r"")
716716
end
717717
whos(m::Module, pat::Regex=r"") = whos(STDOUT, m, pat)
718718
whos(pat::Regex) = whos(STDOUT, current_module(), pat)
719-
720-
#################################################################################
721-
722-
"""
723-
Base.summarysize(obj; exclude=Union{Module,DataType,TypeName}) -> Int
724-
725-
Compute the amount of memory used by all unique objects reachable from the argument.
726-
Keyword argument `exclude` specifies a type of objects to exclude from the traversal.
727-
"""
728-
summarysize(obj; exclude = Union{Module,DataType,TypeName}) =
729-
summarysize(obj, ObjectIdDict(), exclude)
730-
731-
summarysize(obj::Symbol, seen, excl) = 0
732-
733-
function summarysize(obj::UnionAll, seen, excl)
734-
return 2*sizeof(Int) + summarysize(obj.body, seen, excl) + summarysize(obj.var, seen, excl)
735-
end
736-
737-
function summarysize(obj::DataType, seen, excl)
738-
key = pointer_from_objref(obj)
739-
haskey(seen, key) ? (return 0) : (seen[key] = true)
740-
size = 7*sizeof(Int) + 6*sizeof(Int32) + 4*nfields(obj) + ifelse(Sys.WORD_SIZE == 64, 4, 0)
741-
size += summarysize(obj.parameters, seen, excl)::Int
742-
size += summarysize(obj.types, seen, excl)::Int
743-
return size
744-
end
745-
746-
function summarysize(obj::TypeName, seen, excl)
747-
key = pointer_from_objref(obj)
748-
haskey(seen, key) ? (return 0) : (seen[key] = true)
749-
return Core.sizeof(obj) + (isdefined(obj,:mt) ? summarysize(obj.mt, seen, excl) : 0)
750-
end
751-
752-
summarysize(obj::ANY, seen, excl) = _summarysize(obj, seen, excl)
753-
# define the general case separately to make sure it is not specialized for every type
754-
function _summarysize(obj::ANY, seen, excl)
755-
key = pointer_from_objref(obj)
756-
haskey(seen, key) ? (return 0) : (seen[key] = true)
757-
size = Core.sizeof(obj)
758-
ft = typeof(obj).types
759-
for i in 1:nfields(obj)
760-
if !isbits(ft[i]) && isdefined(obj,i)
761-
val = getfield(obj, i)
762-
if !isa(val,excl)
763-
size += summarysize(val, seen, excl)::Int
764-
end
765-
end
766-
end
767-
return size
768-
end
769-
770-
function summarysize(obj::Array, seen, excl)
771-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
772-
size = Core.sizeof(obj)
773-
# TODO: add size of jl_array_t
774-
if !isbits(eltype(obj))
775-
for i in 1:length(obj)
776-
if ccall(:jl_array_isassigned, Cint, (Any, UInt), obj, i-1) == 1
777-
val = obj[i]
778-
if !isa(val, excl)
779-
size += summarysize(val, seen, excl)::Int
780-
end
781-
end
782-
end
783-
end
784-
return size
785-
end
786-
787-
summarysize(s::String, seen, excl) = sizeof(Int) + sizeof(s)
788-
789-
function summarysize(obj::SimpleVector, seen, excl)
790-
key = pointer_from_objref(obj)
791-
haskey(seen, key) ? (return 0) : (seen[key] = true)
792-
size = Core.sizeof(obj)
793-
for i in 1:length(obj)
794-
if isassigned(obj, i)
795-
val = obj[i]
796-
if !isa(val, excl)
797-
size += summarysize(val, seen, excl)::Int
798-
end
799-
end
800-
end
801-
return size
802-
end
803-
804-
function summarysize(obj::Module, seen, excl)
805-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
806-
size::Int = Core.sizeof(obj)
807-
for binding in names(obj, true)
808-
if isdefined(obj, binding) && !isdeprecated(obj, binding)
809-
value = getfield(obj, binding)
810-
if !isa(value, Module) || module_parent(value) === obj
811-
size += summarysize(value, seen, excl)::Int
812-
vt = isa(value,DataType) ? value : typeof(value)
813-
if vt.name.module === obj
814-
if vt !== value
815-
size += summarysize(vt, seen, excl)::Int
816-
end
817-
# charge a TypeName to its module
818-
size += summarysize(vt.name, seen, excl)::Int
819-
end
820-
end
821-
end
822-
end
823-
return size
824-
end
825-
826-
function summarysize(obj::Task, seen, excl)
827-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
828-
size::Int = Core.sizeof(obj)
829-
if isdefined(obj, :code)
830-
size += summarysize(obj.code, seen, excl)::Int
831-
end
832-
size += summarysize(obj.storage, seen, excl)::Int
833-
size += summarysize(obj.backtrace, seen, excl)::Int
834-
size += summarysize(obj.donenotify, seen, excl)::Int
835-
size += summarysize(obj.exception, seen, excl)::Int
836-
size += summarysize(obj.result, seen, excl)::Int
837-
# TODO: add stack size, and possibly traverse stack roots
838-
return size
839-
end
840-
841-
function summarysize(obj::MethodTable, seen, excl)
842-
haskey(seen, obj) ? (return 0) : (seen[obj] = true)
843-
size::Int = Core.sizeof(obj)
844-
size += summarysize(obj.defs, seen, excl)::Int
845-
size += summarysize(obj.cache, seen, excl)::Int
846-
if isdefined(obj, :kwsorter)
847-
size += summarysize(obj.kwsorter, seen, excl)::Int
848-
end
849-
return size
850-
end
851-
852-
function summarysize(m::TypeMapEntry, seen, excl)
853-
size::Int = 0
854-
while true # specialized to prevent stack overflow while following this linked list
855-
haskey(seen, m) ? (return size) : (seen[m] = true)
856-
size += Core.sizeof(m)
857-
if isdefined(m, :func)
858-
size += summarysize(m.func, seen, excl)::Int
859-
end
860-
size += summarysize(m.sig, seen, excl)::Int
861-
size += summarysize(m.tvars, seen, excl)::Int
862-
m.next === nothing && break
863-
m = m.next::TypeMapEntry
864-
end
865-
return size
866-
end

‎base/summarysize.jl‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
immutable SummarySize
2+
seen::ObjectIdDict
3+
frontier_x::Vector{Any}
4+
frontier_i::Vector{Int}
5+
exclude::Any
6+
chargeall::Any
7+
end
8+
9+
10+
"""
11+
Base.summarysize(obj; exclude=Union{...}, chargeall=Union{...}) -> Int
12+
13+
Compute the amount of memory used by all unique objects reachable from the argument.
14+
Keyword argument `exclude` specifies the types of objects to exclude from the traversal.
15+
Keyword argument `chargeall` specifies the types of objects to always charge the size of all of their fields, even if those fields would normally be excluded.
16+
"""
17+
function summarysize(obj::ANY; exclude::ANY = Union{DataType, TypeName}, chargeall::ANY = Union{TypeMapEntry, Method, Core.MethodInstance})
18+
ss = SummarySize(ObjectIdDict(), Any[], Int[], exclude, chargeall)
19+
size::Int = ss(obj)
20+
while !isempty(ss.frontier_x)
21+
# DFS heap traversal of everything without a specialization
22+
# BFS heap traversal of anything with a specialization
23+
x = ss.frontier_x[end]
24+
i = ss.frontier_i[end]
25+
val = nothing
26+
if isa(x, SimpleVector)
27+
nf = length(x)
28+
if isassigned(x, i)
29+
val = x[i]
30+
end
31+
elseif isa(x, Array)
32+
nf = length(x)
33+
if ccall(:jl_array_isassigned, Cint, (Any, UInt), x, i - 1) != 0
34+
val = x[i]
35+
end
36+
else
37+
nf = nfields(x)
38+
ft = typeof(x).types
39+
if !isbits(ft[i]) && isdefined(x, i)
40+
val = getfield(x, i)
41+
end
42+
end
43+
if nfields(x) > i
44+
ss.frontier_i[end] = i + 1
45+
else
46+
pop!(ss.frontier_x)
47+
pop!(ss.frontier_i)
48+
end
49+
if val !== nothing && !isa(val, Module) && (!isa(val, ss.exclude) || isa(x, ss.chargeall))
50+
size += ss(val)::Int
51+
end
52+
end
53+
return size
54+
end
55+
56+
(ss::SummarySize)(obj::ANY) = _summarysize(ss, obj)
57+
# define the general case separately to make sure it is not specialized for every type
58+
@noinline function _summarysize(ss::SummarySize, obj::ANY)
59+
key = pointer_from_objref(obj)
60+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
61+
if nfields(obj) > 0
62+
push!(ss.frontier_x, obj)
63+
push!(ss.frontier_i, 1)
64+
end
65+
if isa(obj, UnionAll)
66+
# black-list of items that don't have a Core.sizeof
67+
return 2 * sizeof(Int)
68+
end
69+
return Core.sizeof(obj)
70+
end
71+
72+
(::SummarySize)(obj::Symbol) = 0
73+
(::SummarySize)(obj::SummarySize) = 0
74+
(::SummarySize)(obj::String) = Core.sizeof(Int) + Core.sizeof(obj)
75+
76+
function (ss::SummarySize)(obj::DataType)
77+
key = pointer_from_objref(obj)
78+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
79+
size::Int = 7 * Core.sizeof(Int) + 6 * Core.sizeof(Int32)
80+
size += 4 * nfields(obj) + ifelse(Sys.WORD_SIZE == 64, 4, 0)
81+
size += ss(obj.parameters)::Int
82+
size += ss(obj.types)::Int
83+
return size
84+
end
85+
86+
function (ss::SummarySize)(obj::TypeName)
87+
key = pointer_from_objref(obj)
88+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
89+
return Core.sizeof(obj) + (isdefined(obj, :mt) ? ss(obj.mt) : 0)
90+
end
91+
92+
function (ss::SummarySize)(obj::Array)
93+
haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true)
94+
size::Int = Core.sizeof(obj)
95+
# TODO: add size of jl_array_t
96+
if !isbits(eltype(obj)) && !isempty(obj)
97+
push!(ss.frontier_x, obj)
98+
push!(ss.frontier_i, 1)
99+
end
100+
return size
101+
end
102+
103+
function (ss::SummarySize)(obj::SimpleVector)
104+
key = pointer_from_objref(obj)
105+
haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true)
106+
size::Int = Core.sizeof(obj)
107+
if !isempty(obj)
108+
push!(ss.frontier_x, obj)
109+
push!(ss.frontier_i, 1)
110+
end
111+
return size
112+
end
113+
114+
function (ss::SummarySize)(obj::Module)
115+
haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true)
116+
size::Int = Core.sizeof(obj)
117+
for binding in names(obj, true)
118+
if isdefined(obj, binding) && !isdeprecated(obj, binding)
119+
value = getfield(obj, binding)
120+
if !isa(value, Module) || module_parent(value) === obj
121+
size += ss(value)::Int
122+
if isa(value, UnionAll)
123+
value = unwrap_unionall(value)
124+
end
125+
if isa(value, DataType) && value.name.module === obj && value.name.name === binding
126+
# charge a TypeName to its module (but not to the type)
127+
size += ss(value.name)::Int
128+
end
129+
end
130+
end
131+
end
132+
return size
133+
end
134+
135+
function (ss::SummarySize)(obj::Task)
136+
haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true)
137+
size::Int = Core.sizeof(obj)
138+
if isdefined(obj, :code)
139+
size += ss(obj.code)::Int
140+
end
141+
size += ss(obj.storage)::Int
142+
size += ss(obj.backtrace)::Int
143+
size += ss(obj.donenotify)::Int
144+
size += ss(obj.exception)::Int
145+
size += ss(obj.result)::Int
146+
# TODO: add stack size, and possibly traverse stack roots
147+
return size
148+
end

‎base/sysimg.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,7 @@ include("datafmt.jl")
300300
importall .DataFmt
301301
include("deepcopy.jl")
302302
include("interactiveutil.jl")
303+
include("summarysize.jl")
303304
include("replutil.jl")
304305
include("test.jl")
305306
include("i18n.jl")

‎test/misc.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,8 +293,8 @@ v11801, t11801 = @timed sin(1)
293293
# interactive utilities
294294

295295
import Base.summarysize
296-
@test summarysize(Core) > summarysize(Core.Inference) > Core.sizeof(Core)
297-
@test summarysize(Base) > 10_000*sizeof(Int)
296+
@test summarysize(Core) > (summarysize(Core.Inference) + Base.summarysize(Core.Intrinsics)) > Core.sizeof(Core)
297+
@test summarysize(Base) > 100_000 * sizeof(Int)
298298
module _test_whos_
299299
export x
300300
x = 1.0

0 commit comments

Comments
 (0)