From 87c559c8bd4585903dff839c24ef5b7a1c78d4ec Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 11 Nov 2016 21:25:20 -0500 Subject: [PATCH 1/3] ulimit the max open pipes during testing fix #19132 --- .travis.yml | 4 +++- test/Makefile | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cacc3f81b3827..ca71afa6626dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,7 +125,9 @@ script: - /tmp/julia/bin/julia --precompiled=no -e 'true' && /tmp/julia/bin/julia-debug --precompiled=no -e 'true' - /tmp/julia/bin/julia -e 'versioninfo()' - - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test && + - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && + ulimit -n 128 && + cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && diff --git a/test/Makefile b/test/Makefile index 66156a74c7a61..12569ea960267 100644 --- a/test/Makefile +++ b/test/Makefile @@ -8,8 +8,17 @@ TESTS = all linalg sparse unicode strings dates $(filter-out TestHelpers runtest default: all +# set hard and soft limits for spawn fd-exhaustion test +ifeq ($(OS),WINNT) +ULIMIT_TEST= +else +ULIMIT_TEST=ulimit -n 128 +endif + $(TESTS): - @cd $(SRCDIR) && $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no ./runtests.jl $@) + @cd $(SRCDIR) && \ + $(ULIMIT_TEST) && \ + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no ./runtests.jl $@) perf: @$(MAKE) -C $(SRCDIR)/perf all From adaad6228776b7b0ee442787d0c3f662c298db42 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Nov 2016 11:05:02 -0500 Subject: [PATCH 2/3] make Libuv stream printing more useful and add error-handling and tests to Base._fd --- base/stream.jl | 24 +++++++++++++++++------- test/spawn.jl | 8 ++++++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/base/stream.jl b/base/stream.jl index acc7c02096c33..c8d39e443943a 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -187,8 +187,12 @@ function TTY(fd::RawFD; readable::Bool = false) return tty end -show(io::IO,stream::LibuvServer) = print(io, typeof(stream), "(", uv_status_string(stream), ")") -show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(", uv_status_string(stream), ", ", +show(io::IO, stream::LibuvServer) = print(io, typeof(stream), "(", + _fd(stream), " ", + uv_status_string(stream), ")") +show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(", + _fd(stream), " ", + uv_status_string(stream), ", ", nb_available(stream.buffer)," bytes waiting)") # Shared LibuvStream object interface @@ -521,7 +525,9 @@ end show(io::IO, stream::Pipe) = print(io, "Pipe(", + _fd(stream.in), " ", uv_status_string(stream.in), " => ", + _fd(stream.out), " ", uv_status_string(stream.out), ", ", nb_available(stream), " bytes waiting)") @@ -925,12 +931,16 @@ Connect to the named pipe / UNIX domain socket at `path`. """ connect(path::AbstractString) = connect(init_pipe!(PipeEndpoint(); readable=false, writable=false, julia_only=true),path) +const OS_HANDLE = is_windows() ? WindowsRawSocket : RawFD +const INVALID_OS_HANDLE = is_windows() ? WindowsRawSocket(Ptr{Void}(-1)) : RawFD(-1) _fd(x::IOStream) = RawFD(fd(x)) -if is_windows() - _fd(x::LibuvStream) = WindowsRawSocket( - ccall(:jl_uv_handle, Ptr{Void}, (Ptr{Void},), x.handle)) -else - _fd(x::LibuvStream) = RawFD(ccall(:jl_uv_handle, Int32, (Ptr{Void},), x.handle)) +function _fd(x::Union{LibuvStream, LibuvServer}) + fd = Ref{OS_HANDLE}(INVALID_OS_HANDLE) + if x.status != StatusUninit && x.status != StatusClosed + err = ccall(:uv_fileno, Int32, (Ptr{Void}, Ptr{OS_HANDLE}), x.handle, fd) + # handle errors by returning INVALID_OS_HANDLE + end + return fd[] end for (x, writable, unix_fd, c_symbol) in diff --git a/test/spawn.jl b/test/spawn.jl index 4ddce0a6ef56c..6f5d76746d091 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -283,7 +283,7 @@ let bad = "bad\0name" end # issue #12829 -let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition(), t +let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition(), t, infd, outfd @test_throws ArgumentError write(out, "not open error") t = @async begin # spawn writer task open(echo, "w", out) do in1 @@ -295,6 +295,8 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r write(in2, "orld\n") end end + infd = Base._fd(out.in) + outfd = Base._fd(out.out) show(out, out) notify(ready) @test isreadable(out) @@ -327,13 +329,15 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test !isreadable(out) @test !iswritable(out) @test !isopen(out) + @test infd != Base._fd(out.in) == Base.INVALID_OS_HANDLE + @test outfd != Base._fd(out.out) == Base.INVALID_OS_HANDLE @test nb_available(out) == 0 @test c == UInt8['w'] @test lstrip(ln2) == "1\thello\n" @test ln1 == "orld\n" @test isempty(read(out)) @test eof(out) - @test desc == "Pipe(open => active, 0 bytes waiting)" + @test desc == "Pipe($infd open => $outfd active, 0 bytes waiting)" wait(t) end From 13537276b61cb851937cc53a22e5e153f20a7d62 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Nov 2016 16:59:43 -0500 Subject: [PATCH 3/3] permit serializing MAP_FAILED / INVALID_HANDLE constants into system image --- src/dump.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dump.c b/src/dump.c index 6583e75718848..d9e5414163a2e 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1042,7 +1042,9 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) } size_t nf = jl_datatype_nfields(t); if (nf == 0 && jl_datatype_size(t)>0) { - if (t->name == jl_pointer_type->name) { + if (t->name == jl_pointer_type->name && jl_unbox_voidpointer(v) != (void*)-1) { + // normalize most pointers to NULL, to help catch memory errors + // but permit MAP_FAILED / INVALID_HANDLE to be stored unchanged write_int32(s->s, 0); #ifdef _P64 write_int32(s->s, 0);