Merge pull request #19307 from JuliaLang/jn/pipe-ulimit

ulimit the max open pipes during testing

Author: vtjnash

.travis.yml

@@ -125,7 +125,9 @@ script:
     - /tmp/julia/bin/julia --precompiled=no -e 'true' &&
         /tmp/julia/bin/julia-debug --precompiled=no -e 'true'
     - /tmp/julia/bin/julia -e 'versioninfo()'
-    - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test &&
+    - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 &&
+        ulimit -n 128 &&
+        cd /tmp/julia/share/julia/test &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg
     - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia &&

base/stream.jl

@@ -187,8 +187,12 @@ function TTY(fd::RawFD; readable::Bool = false)
     return tty
 end
 
-show(io::IO,stream::LibuvServer) = print(io, typeof(stream), "(", uv_status_string(stream), ")")
-show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(", uv_status_string(stream), ", ",
+show(io::IO, stream::LibuvServer) = print(io, typeof(stream), "(",
+    _fd(stream), " ",
+    uv_status_string(stream), ")")
+show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(",
+    _fd(stream), " ",
+    uv_status_string(stream), ", ",
     nb_available(stream.buffer)," bytes waiting)")
 
 # Shared LibuvStream object interface
@@ -521,7 +525,9 @@ end
 
 show(io::IO, stream::Pipe) = print(io,
     "Pipe(",
+    _fd(stream.in), " ",
     uv_status_string(stream.in), " => ",
+    _fd(stream.out), " ",
     uv_status_string(stream.out), ", ",
     nb_available(stream), " bytes waiting)")
 
@@ -925,12 +931,16 @@ Connect to the named pipe / UNIX domain socket at `path`.
 """
 connect(path::AbstractString) = connect(init_pipe!(PipeEndpoint(); readable=false, writable=false, julia_only=true),path)
 
+const OS_HANDLE = is_windows() ? WindowsRawSocket : RawFD
+const INVALID_OS_HANDLE = is_windows() ? WindowsRawSocket(Ptr{Void}(-1)) : RawFD(-1)
 _fd(x::IOStream) = RawFD(fd(x))
-if is_windows()
-    _fd(x::LibuvStream) = WindowsRawSocket(
-        ccall(:jl_uv_handle, Ptr{Void}, (Ptr{Void},), x.handle))
-else
-    _fd(x::LibuvStream) = RawFD(ccall(:jl_uv_handle, Int32, (Ptr{Void},), x.handle))
+function _fd(x::Union{LibuvStream, LibuvServer})
+    fd = Ref{OS_HANDLE}(INVALID_OS_HANDLE)
+    if x.status != StatusUninit && x.status != StatusClosed
+        err = ccall(:uv_fileno, Int32, (Ptr{Void}, Ptr{OS_HANDLE}), x.handle, fd)
+        # handle errors by returning INVALID_OS_HANDLE
+    end
+    return fd[]
 end
 
 for (x, writable, unix_fd, c_symbol) in

src/dump.c

@@ -1042,7 +1042,9 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
             }
             size_t nf = jl_datatype_nfields(t);
             if (nf == 0 && jl_datatype_size(t)>0) {
-                if (t->name == jl_pointer_type->name) {
+                if (t->name == jl_pointer_type->name && jl_unbox_voidpointer(v) != (void*)-1) {
+                    // normalize most pointers to NULL, to help catch memory errors
+                    // but permit MAP_FAILED / INVALID_HANDLE to be stored unchanged
                     write_int32(s->s, 0);
 #ifdef _P64
                     write_int32(s->s, 0);

test/Makefile

@@ -8,8 +8,17 @@ TESTS = all linalg sparse unicode strings dates $(filter-out TestHelpers runtest
 
 default: all
 
+# set hard and soft limits for spawn fd-exhaustion test
+ifeq ($(OS),WINNT)
+ULIMIT_TEST=
+else
+ULIMIT_TEST=ulimit -n 128
+endif
+
 $(TESTS):
-	@cd $(SRCDIR) && $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no ./runtests.jl $@)
+	@cd $(SRCDIR) && \
+	$(ULIMIT_TEST) && \
+	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no ./runtests.jl $@)
 
 perf:
 	@$(MAKE) -C $(SRCDIR)/perf all

test/spawn.jl

@@ -283,7 +283,7 @@ let bad = "bad\0name"
 end
 
 # issue #12829
-let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition(), t
+let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition(), t, infd, outfd
     @test_throws ArgumentError write(out, "not open error")
     t = @async begin # spawn writer task
         open(echo, "w", out) do in1
@@ -295,6 +295,8 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r
                 write(in2, "orld\n")
             end
         end
+        infd = Base._fd(out.in)
+        outfd = Base._fd(out.out)
         show(out, out)
         notify(ready)
         @test isreadable(out)
@@ -327,13 +329,15 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r
     @test !isreadable(out)
     @test !iswritable(out)
     @test !isopen(out)
+    @test infd != Base._fd(out.in) == Base.INVALID_OS_HANDLE
+    @test outfd != Base._fd(out.out) == Base.INVALID_OS_HANDLE
     @test nb_available(out) == 0
     @test c == UInt8['w']
     @test lstrip(ln2) == "1\thello\n"
     @test ln1 == "orld\n"
     @test isempty(read(out))
     @test eof(out)
-    @test desc == "Pipe(open => active, 0 bytes waiting)"
+    @test desc == "Pipe($infd open => $outfd active, 0 bytes waiting)"
     wait(t)
 end