print memory usage and a truncated version of the object in whos()

fix #3393. close #7603. close #11461.

Author: vtjnash

base/interactiveutil.jl

@@ -409,3 +409,242 @@ function runtests(tests = ["all"], numcores = ceil(Int,CPU_CORES/2))
               "including error messages above and the output of versioninfo():\n$(readall(buf))")
     end
 end
+
+# testing
+
+function whos(io::IO=STDOUT, m::Module=current_module(), pattern::Regex=r"")
+    maxline = tty_size()[2]
+    line = zeros(UInt8, maxline)
+    head = PipeBuffer(maxline + 1)
+    for v in sort!(names(m))
+        s = string(v)
+        if isdefined(m, v) && ismatch(pattern, s)
+            value = getfield(m, v)
+            bytes = summarysize(value, true)
+            @printf head "%30s " s
+            if bytes < 10_000
+                @printf(head, "%6d bytes  ", bytes)
+            else
+                @printf(head, "%6d KB     ", bytes ÷ (1024))
+            end
+            print(head, summary(value))
+            print(head, " : ")
+            show(head, value)
+
+            newline = search(head, UInt8('\n')) - 1
+            if newline < 0
+                newline = nb_available(head)
+            end
+            if newline > maxline
+                newline = maxline - 1 # make space for ...
+            end
+            line = resize!(line, newline)
+            line = read!(head, line)
+
+            write(io, line)
+            if nb_available(head) > 0 # more to read? replace with ...
+                print(io, '\u2026') # hdots
+            end
+            println(io)
+            seekend(head) # skip the rest of the text
+        end
+    end
+end
+whos(m::Module, pat::Regex=r"") = whos(STDOUT, m, pat)
+whos(pat::Regex) = whos(STDOUT, current_module(), pat)
+
+# summarysize is an estimate of the size of the object
+# as if all iterables were allocated inline
+# in general, this forms a conservative lower bound
+# on the memory "controlled" by the object
+# if recurse is true, then simply reachable memory
+# should also be included, otherwise, only
+# directly used memory should be included
+# you should never ignore recurse in cases where recursion is possible
+
+summarysize(obj::ANY, recurse::Bool) = try convert(Int, sizeof(obj)); catch; Core.sizeof(obj); end
+
+# these three cases override the exception that would be thrown by Core.sizeof
+summarysize(obj::Symbol, recurse::Bool) = 0
+summarysize(obj::DataType, recurse::Bool) = 0
+function summarysize(obj::Module, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse
+        for binding in names(obj, true)
+            if isdefined(obj, binding)
+                value = getfield(obj, binding)
+                if (value !== obj) # skip the self-recursive definition
+                    recurseok = !isa(value, Module) || module_parent(value) === obj
+                    size += summarysize(value, recurseok)::Int # recurse on anything that isn't a module
+                end
+            end
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Task, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse
+        size += summarysize(obj.code, true)::Int
+        size += summarysize(obj.storage, true)::Int
+
+        size += summarysize(obj.backtrace, false)::Int
+        size += summarysize(obj.donenotify, false)::Int
+        size += summarysize(obj.exception, false)::Int
+        size += summarysize(obj.result, false)::Int
+    end
+    return size
+end
+
+function summarysize(obj::SimpleVector, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse
+        for val in obj
+            if val !== obj
+                size += summarysize(val, false)::Int
+            end
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Tuple, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse
+        for val in obj
+            if val !== obj && !isbits(val)
+                size += summarysize(val, recurse)::Int
+            end
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Array, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse && !isbits(eltype(obj))
+        for i in 1:length(obj)
+            if isdefined(obj, i) && (val = obj[i]) !== obj
+                size += summarysize(val, false)::Int
+            end
+        end
+    end
+    return size
+end
+
+function summarysize(obj::AbstractArray, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse && !isbits(eltype(obj))
+        for val in obj
+            if val !== obj
+                size += summarysize(val, false)::Int
+            end
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Associative, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse
+        for (key, val) in obj
+            if key !== obj
+                size += summarysize(key, false)::Int
+            end
+            if val !== obj
+                size += summarysize(val, false)::Int
+            end
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Dict, recurse::Bool)
+    size::Int = sizeof(obj)
+    size += summarysize(obj.keys, recurse)::Int
+    size += summarysize(obj.vals, recurse)::Int
+    size += summarysize(obj.slots, recurse)::Int
+    return size
+end
+
+summarysize(obj::Set, recurse::Bool) =
+    sizeof(obj) + summarysize(obj.dict, recurse)
+
+function summarysize(obj::Function, recurse::Bool)
+    size::Int = sizeof(obj)
+    size += summarysize(obj.env, recurse)::Int
+    if isdefined(obj, :code)
+        size += summarysize(obj.code, recurse)::Int
+    end
+    return size
+end
+
+function summarysize(obj::MethodTable, recurse::Bool)
+    size::Int = sizeof(obj)
+    size += summarysize(obj.defs, recurse)::Int
+    size += summarysize(obj.cache, recurse)::Int
+    size += summarysize(obj.cache_arg1, recurse)::Int
+    size += summarysize(obj.cache_targ, recurse)::Int
+    if isdefined(obj, :kwsorter)
+        size += summarysize(obj.kwsorter, recurse)::Int
+    end
+    return size
+end
+
+function summarysize(obj::Method, recurse::Bool)
+    size::Int = sizeof(obj)
+    size += summarysize(obj.func, recurse)::Int
+    size += summarysize(obj.next, recurse)::Int
+    return size
+end
+
+function summarysize(obj::LambdaStaticData, recurse::Bool)
+    size::Int = sizeof(obj)
+    size += summarysize(obj.ast, true)::Int # always include the AST
+    size += summarysize(obj.sparams, true)::Int
+    if isdefined(obj, :roots)
+        size += summarysize(obj.roots, recurse)::Int
+    end
+    if isdefined(obj, :capt)
+        size += summarysize(obj.capt, false)::Int
+    end
+    return size
+end
+
+function summarysize(obj::Expr, recurse::Bool)
+    size::Int = sizeof(obj) + sizeof(obj.args)
+    if recurse
+        for arg in obj.args
+            size += summarysize(arg, isa(arg, Expr))::Int
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Box, recurse::Bool)
+    size::Int = sizeof(obj)
+    # ignore the recurse parameter for Box,
+    # even though it could in theory recurse
+    # since it is an internal construct
+    # used by codegen with very limited usage
+    if isdefined(obj, :contents)
+        if obj.contents !== obj
+            size += summarysize(obj.contents, false)::Int
+        end
+    end
+    return size
+end
+
+function summarysize(obj::Ref, recurse::Bool)
+    size::Int = sizeof(obj)
+    if recurse && !isbits(eltype(obj))
+        try
+            val = obj[]
+            if val !== obj
+                size += summarysize(val, false)::Int
+            end
+        end
+    end
+    return size
+end

base/iobuffer.jl

@@ -343,7 +343,8 @@ readbytes(io::AbstractIOBuffer, nb) = read!(io, Array(UInt8, min(nb, nb_availabl
 function search(buf::IOBuffer, delim::UInt8)
     p = pointer(buf.data, buf.ptr)
     q = ccall(:memchr,Ptr{UInt8},(Ptr{UInt8},Int32,Csize_t),p,delim,nb_available(buf))
-    nb = (q == C_NULL ? 0 : q-p+1)
+    nb::Int = (q == C_NULL ? 0 : q-p+1)
+    return nb
 end
 
 function search(buf::AbstractIOBuffer, delim::UInt8)

base/show.jl

@@ -1166,18 +1166,6 @@ function show_nd(io::IO, a::AbstractArray, limit, print_matrix, label_slices)
     cartesianmap(print_slice, tail)
 end
 
-function whos(m::Module, pattern::Regex)
-    for v in sort(names(m))
-        s = string(v)
-        if isdefined(m,v) && ismatch(pattern, s)
-            println(rpad(s, 30), summary(eval(m,v)))
-        end
-    end
-end
-whos() = whos(r"")
-whos(m::Module) = whos(m, r"")
-whos(pat::Regex) = whos(current_module(), pat)
-
 # global flag for limiting output
 # TODO: this should be replaced with a better mechanism. currently it is only
 # for internal use in showing arrays.

test/misc.jl

@@ -156,3 +156,21 @@ v11801, t11801 = @timed sin(1)
 @test isa(t11801,Real) && t11801 >= 0
 
 @test names(current_module(), true) == names_before_timing
+
+# interactive utilities
+
+import Base.summarysize
+@test summarysize(Core, true) > summarysize(Core.Inference, true) > summarysize(Core, false) == summarysize(Core.Inference, false) == Core.sizeof(Core)
+@test summarysize(Main, true) > 10_000*summarysize(Main, false) > 10_000*sizeof(Int)
+@test 0 == summarysize(Int, true) == summarysize(Int, false) == summarysize(DataType, true) == summarysize(Ptr, true) == summarysize(Any, true)
+@test sprint(whos, Main, r"^$") == ""
+let v = sprint(whos, Main)
+    @test contains(v, " KB     Module : Main")
+    @test contains(v, " KB     Module : Base")
+    @test contains(v, " KB     Module : Core")
+    @test contains(v, "  0 bytes  DataType : NoMethodHasThisType")
+    @test contains(v, "\u2026\n")
+    @test match(r".\u2026$"m, v) !== nothing
+    @test match(r"\u2026."m, v) === nothing
+    @test !contains(v, "Core.Inference")
+end